diff --git a/benchmarks/single_node/agentic/apply_dsv4_container_patches.sh b/benchmarks/single_node/agentic/apply_dsv4_container_patches.sh new file mode 100755 index 000000000..ef9f5c1b2 --- /dev/null +++ b/benchmarks/single_node/agentic/apply_dsv4_container_patches.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# ============================================================================= +# Container patch stack for the 08-09 nightly base +# vllm/vllm-openai-rocm:nightly-f8d03e77416bf90c49acbe50e233275722f02c4b +# (vllm 0.26.1rc1.dev528+gf8d03e774) +# +# Everything here comes from an UPSTREAM ref -- either a vllm PR diff or a +# pinned aiter/mori commit. Nothing is sourced from a private measurement +# image. Idempotent: every step is marker-gated, re-running is a no-op. +# +# Run INSIDE a fresh container of the pinned image: +# docker exec -i bash /path/to/apply_dsv4_container_patches.sh +# +# WHAT THIS ENABLES +# Three knobs that the base cannot run unpatched, all three verified to +# import cleanly afterwards (see the checks at the end): +# * sparse gluon decode (VLLM_ROCM_DSV4_SPARSE_GLUON=1) +# * FSE (aiter fhmoe) +# * MegaMoE / DEP8 (--moe-backend flydsl_mega_moe) +# +# The gluon knob in particular was silently dead before this: the base's +# aiter facade pa_decode_sparse() predates the extra_cache/extra_indices/ +# extra_indptr keywords that vllm #51714's call site passes, so every worker +# raised +# TypeError: pa_decode_sparse() got an unexpected keyword argument +# 'extra_cache' +# on its first decode and latched onto the Triton fallback for the rest of +# the process. Copying only the gfx950 gluon kernel is NOT enough; the facade +# that dispatches to it has to come along. +# +# WHAT IS NOT DONE +# * aiter core (jit/core.py) is not modified. The base already exposes +# _set_current_hip_stream, which the nightly vllm calls for +# module_rmsnorm_quant; an earlier wholesale post2-python overlay regressed +# it and crashed rmsnorm_quant warmup. +# * aiter #4417 is NOT grafted separately. It is already contained in the +# aiter@$AITER_SHA moe_kernels.py this script installs, so the old +# anchor-graft helper is gone. +# ============================================================================= +set -uo pipefail +AITER_SHA="97d0c6e4cb7a0919c12291c7c7d560ad412f15c1" +AITER_REPO="https://github.com/ROCm/aiter" +MORI_SHA="84a33cc0f15f019c78c995728973b70ea3d10bb7" +MORI_REPO="https://github.com/ROCm/mori" +VLLM_REPO="https://github.com/vllm-project/vllm" +ROOT="$(python -c 'import importlib.util as u, os; print(os.path.dirname(os.path.dirname(u.find_spec("vllm").origin)))')" +[ -d "$ROOT/vllm" ] && [ -d "$ROOT/aiter" ] || { echo "ERROR ROOT=$ROOT"; exit 1; } +echo "[patch] ROOT=$ROOT" +echo "[patch] vllm = $(python -c 'import vllm;print(vllm.__version__)' 2>/dev/null)" +WS=/tmp/dsv4_patch; mkdir -p "$WS" + +# --- 1/4 aiter files from the pinned SHA -------------------------------------- +# Sourced from aiter@$AITER_SHA, not from any prebuilt image. +# +# A note on why the facade is in this list. An earlier revision of this script +# copied only the ONE file the base lacked (the gfx950 gluon kernel) and +# deliberately skipped the three pa_decode_sparse variants that ship in the +# base, to avoid a cross-version transplant. That was the right instinct but +# the wrong call for the facade specifically: keeping the base's older facade +# is what produced the extra_cache TypeError above. The facade and the kernel +# it dispatches to are one unit and have to move together. +# +# moe_kernels.py is replaced rather than patched. Symbol-diffed base vs +# upstream before doing so: 58 base symbols, 66 upstream, ZERO lost, +8 gained +# (including _flydsl_moe_stage1_impl / _flydsl_moe_stage2_impl, which FSE's +# aiter/ops/flydsl/fhmoe.py imports and the base simply does not define). +AITER_FILES=( + # sparse gluon decode: the kernel AND the facade that dispatches to it + aiter/ops/triton/attention/pa_decode_sparse.py + aiter/ops/triton/_gluon_kernels/gfx950/attention/pa_decode_sparse.py + # FSE (fused heterogeneous MoE) + aiter/fhmoe.py + aiter/ops/flydsl/fhmoe.py + aiter/ops/flydsl/kernels/fhmoe.py + aiter/aot/flydsl/fhmoe.py + # carries FSE's stage1/stage2 impls, and aiter #4417's two guards + aiter/ops/flydsl/moe_kernels.py +) +ASRC="$WS/aiter_src" +if [ ! -d "$ASRC/.git" ]; then git clone -q --filter=blob:none --no-checkout "$AITER_REPO" "$ASRC" 2>&1 | tail -1; fi +( cd "$ASRC" && git fetch -q --depth 1 origin "$AITER_SHA" 2>&1 | tail -1 && git checkout -q "$AITER_SHA" -- "${AITER_FILES[@]}" ) \ + || { echo " aiter: CHECKOUT FAILED"; exit 1; } +for p in "${AITER_FILES[@]}"; do + [ -e "$ASRC/$p" ] || { echo " MISSING in src: $p"; continue; } + if [ -f "$ROOT/$p" ] && cmp -s "$ASRC/$p" "$ROOT/$p"; then echo " same $p"; continue; fi + mkdir -p "$(dirname "$ROOT/$p")"; cp -a "$ASRC/$p" "$ROOT/$p"; echo " installed $p" +done + +# --- 2/4 mori: mori.ir.flydsl + the cov cascade ------------------------------- +# MegaMoE's intranode dispatch/combine kernel does `import mori.ir.flydsl`, and +# the base ships mori WITHOUT that subpackage. Three .py files -- no compiled +# artifact, contrary to a first reading of the ModuleNotFoundError. +# +# They cannot be dropped in alone. mori/ir/flydsl/runtime.py calls +# find_bitcode(cov=6) (FlyDSL needs ABI 600), and `cov` is a parameter that +# cascades through three more files the base predates: ir/bitcode.py, +# jit/cache.py, jit/core.py (37 call sites upstream, 0 in the base) plus +# jit/config.py for is_debuginfo_enabled. Installing a subset yields, in order, +# TypeError: find_bitcode() got an unexpected keyword argument 'cov' +# ImportError: cannot import name 'is_debuginfo_enabled' +# so the whole cascade goes or none of it does. +# +# libmori_shmem_device.bc is NOT shipped here. It is a 485 KB LLVM IR blob and +# a build artifact; find_bitcode() falls back to mori.jit.core.ensure_bitcode(), +# which compiles it in-container with hipcc on first import (~1 min). That +# keeps this script free of binary payloads. +MORI_FILES=( + mori/ir/flydsl/__init__.py + mori/ir/flydsl/ops.py + mori/ir/flydsl/runtime.py + mori/ir/bitcode.py + mori/jit/cache.py + mori/jit/config.py + mori/jit/core.py +) +if [ -d "$ROOT/mori" ]; then + MSRC="$WS/mori_src" + if [ ! -d "$MSRC/.git" ]; then git clone -q --filter=blob:none --no-checkout "$MORI_REPO" "$MSRC" 2>&1 | tail -1; fi + ( cd "$MSRC" && git fetch -q --depth 1 origin "$MORI_SHA" 2>&1 | tail -1 && git checkout -q "$MORI_SHA" -- python/mori ) \ + || { echo " mori: CHECKOUT FAILED"; } + for p in "${MORI_FILES[@]}"; do + src="$MSRC/python/$p" + [ -e "$src" ] || { echo " MISSING in src: $p"; continue; } + if [ -f "$ROOT/$p" ] && cmp -s "$src" "$ROOT/$p"; then echo " same $p"; continue; fi + mkdir -p "$(dirname "$ROOT/$p")"; cp -a "$src" "$ROOT/$p"; echo " installed $p" + done + find "$ROOT/mori" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null +else + echo " mori: not installed in this image, skipping MegaMoE deps" +fi +find "$ROOT/aiter" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null + +# --- 3/4 vllm PRs ------------------------------------------------------------- +fetch_diff(){ local pr="$1"; local d="$WS/vllm_$pr.diff" + [ -s "$d" ] || curl -ksSL -o "$d" "$VLLM_REPO/pull/$pr.diff" || return 1 + [ -s "$d" ] || return 1; echo "$d" +} + +# Split a PR diff down to the hunks for ONE file. `patch -i ` +# does NOT do this -- it applies every hunk in the diff to the named file. Doing +# that once wrote dspark.py's and mtp.py's hunks into model.py and left it with +# a SyntaxError (an import landed mid-docstring). Always split first. +split_diff(){ local d="$1" f="$2" out="$3" + python - "$d" "$f" "$out" <<'PYEOF' +import re, sys +src, target, out = sys.argv[1], sys.argv[2], sys.argv[3] +parts = re.split(r"(?m)^(?=diff --git )", open(src).read()) +keep = [p for p in parts if p.startswith("diff --git a/%s " % target)] +open(out, "w").write("".join(keep)) +sys.exit(0 if keep else 1) +PYEOF +} + +# Apply one file's hunks. git apply first (exact); fall back to patch --fuzz, +# which tolerates the context drift from #51918 being written against a tree a +# few days newer than this base. +apply_file(){ local pr="$1" f="$2" mk="$3" fuzz="${4:-0}" + if [ -n "$mk" ] && grep -qF "$mk" "$ROOT/$f" 2>/dev/null; then echo " #$pr $f: already present"; return 0; fi + local d; d="$(fetch_diff "$pr")" || { echo " #$pr: FETCH FAIL"; return 1; } + local one="$WS/${pr}_$(echo "$f" | tr / _).diff" + split_diff "$d" "$f" "$one" || { echo " #$pr $f: no hunks in diff"; return 1; } + if ( cd "$ROOT" && git apply -p1 "$one" ) 2>/dev/null; then + echo " #$pr $f: APPLIED"; return 0 + fi + if [ "$fuzz" != "0" ]; then + if ( cd "$ROOT" && patch -p1 --forward --fuzz="$fuzz" --no-backup-if-mismatch -s -i "$one" ) 2>/dev/null; then + echo " #$pr $f: APPLIED (fuzz=$fuzz)" + ( cd "$ROOT" && rm -f "$f.rej" "$f.orig" ) + return 0 + fi + ( cd "$ROOT" && rm -f "$f.rej" "$f.orig" ) + fi + echo " #$pr $f: FAILED"; return 1 +} + +# #51473 -- native MXFP4 TP8 shard allocation. MERGED 2026-08-11, i.e. AFTER +# this 08-09 base, so unlike the 08-12 line it must be applied here. Its tests/ +# hunk has no counterpart in an installed wheel, hence per-file application. +apply_file 51473 "vllm/model_executor/layers/fused_moe/oracle/mxfp4.py" \ + "AITER_MXFP4_BF16 and activation == MoEActivation.SILU" || true + +# #51714 -- opt-in gluon sparse-MLA decode for gfx950. Dormant unless +# VLLM_ROCM_DSV4_SPARSE_GLUON=1. Needs the aiter facade from step 1 to work. +apply_file 51714 "vllm/v1/attention/ops/rocm_aiter_mla_sparse.py" "_DSV4_SPARSE_GLUON" || true + +# #51918 -- FlyDSL mega-MoE backend, now applied IN FULL. +# A previous revision took only config/kernel.py (the backend name) and skipped +# the model side, on the grounds that mori.ir.flydsl was missing so the backend +# could never run. Step 2 removes that blocker, so the model hunks go in too. +# model.py needs fuzz=3: #51918 targets a tree a few days newer and the earlier +# hunks shift line numbering. All 6 of its hunks land; the result is +# py_compile-clean and imports (checked below). +# +# Each entry carries its own presence marker. Without one the step re-runs on an +# already-patched container, git apply correctly refuses, and the log says +# FAILED -- functionally a no-op, but it reads like a real failure in CI output. +apply_file 51918 "vllm/config/kernel.py" "flydsl_mega_moe" || true +apply_file 51918 "vllm/models/deepseek_v4/amd/dspark.py" "finalize_mega_moe_layers" || true +apply_file 51918 "vllm/models/deepseek_v4/amd/mega_moe_experts.py" "MegaMoE expert layer" || true +apply_file 51918 "vllm/models/deepseek_v4/amd/mega_moe_runtime.py" "MegaMoEV2 runtime" || true +apply_file 51918 "vllm/models/deepseek_v4/amd/mtp.py" "finalize_mega_moe_layers" || true +apply_file 51918 "vllm/models/deepseek_v4/amd/model.py" "use_mega_moe" 3 || true + +# --- 4/4 verify --------------------------------------------------------------- +echo "chk gluon gfx950 kernel = $([ -f "$ROOT/aiter/ops/triton/_gluon_kernels/gfx950/attention/pa_decode_sparse.py" ] && echo present || echo MISSING)" +echo "chk aiter #4417 guards = $(grep -c 'requires_flydsl_stage2_reduce\|resolve_flydsl_grid_y_persist_m' "$ROOT/aiter/ops/flydsl/moe_kernels.py" 2>/dev/null) (expect 5)" +echo "chk vllm #51473 = $(grep -c 'AITER_MXFP4_BF16 and activation == MoEActivation.SILU' "$ROOT/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py" 2>/dev/null) (expect 1)" +echo "chk vllm #51714 = $(grep -c '_DSV4_SPARSE_GLUON' "$ROOT/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py" 2>/dev/null)" +echo "chk vllm #51918 name = $(grep -c 'flydsl_mega_moe' "$ROOT/vllm/config/kernel.py" 2>/dev/null)" +echo "chk vllm #51918 model = $(grep -c 'use_mega_moe' "$ROOT/vllm/models/deepseek_v4/amd/model.py" 2>/dev/null) (expect >0)" + +python -m py_compile \ + "$ROOT/aiter/ops/triton/attention/pa_decode_sparse.py" \ + "$ROOT/aiter/ops/flydsl/moe_kernels.py" \ + "$ROOT/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py" \ + "$ROOT/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py" \ + "$ROOT/vllm/models/deepseek_v4/amd/model.py" \ + "$ROOT/vllm/models/deepseek_v4/amd/mtp.py" \ + "$ROOT/vllm/models/deepseek_v4/amd/dspark.py" \ + "$ROOT/vllm/models/deepseek_v4/amd/mega_moe_experts.py" \ + "$ROOT/vllm/models/deepseek_v4/amd/mega_moe_runtime.py" \ + "$ROOT/vllm/config/kernel.py" && echo PY_COMPILE_OK || { echo PY_COMPILE_FAIL; exit 1; } + +python - <<'PYEOF' +import importlib, inspect, re, sys + +fail = False + +# The facade signature is the check that actually matters for the gluon knob: +# the kernel file being present says nothing about whether the call site's +# keywords are accepted. +try: + from aiter.ops.triton.attention.pa_decode_sparse import pa_decode_sparse as f + p = inspect.signature(f).parameters + missing = [k for k in ("extra_cache", "extra_indices", "extra_indptr") if k not in p] + if missing: + fail = True + print("FACADE_MISSING", missing, "-- gluon would fall back to Triton at runtime") + else: + print("FACADE_OK extra_cache/extra_indices/extra_indptr accepted") +except Exception as e: + fail = True + print("FACADE_ERR", type(e).__name__, e) + +for m in ["aiter.jit.core", + "aiter.ops.triton.attention.pa_decode_sparse", + "aiter.fhmoe", # FSE + "aiter.ops.flydsl.fhmoe", # FSE + "mori.ir.flydsl", # MegaMoE + "aiter.ops.flydsl.kernels.flydsl_dispatch_combine_intranode_kernel", + "vllm.model_executor.layers.fused_moe.oracle.mxfp4", + "vllm.v1.attention.ops.rocm_aiter_mla_sparse", + "vllm.models.deepseek_v4.amd.model", + "vllm._aiter_ops"]: + try: + importlib.import_module(m) + print("IMPORT_OK", m) + except Exception as e: + fail = True + print("IMPORT_ERR", m, type(e).__name__, (str(e).splitlines() or [""])[-1]) + +# _set_current_hip_stream is NOT a top-level attribute of aiter.jit.core -- it +# is called there as `module._set_current_hip_stream(...)`, i.e. it lives on the +# compiled .so that core.py loads. hasattr(core, ...) therefore returns False on +# a perfectly healthy tree; an earlier version of this probe read that False as +# a missing symbol. Grep the call site instead. +try: + import aiter.jit.core as c + n = len(re.findall(r"_set_current_hip_stream", inspect.getsource(c))) + print(f"core.py _set_current_hip_stream call sites: {n} (expect >=1)") +except Exception as e: + print("core probe err", e) + +print("VERIFY:", "FAILURES_PRESENT" if fail else "ALL_OK") +sys.exit(1 if fail else 0) +PYEOF +rc=$? +echo "[patch] DONE (verify rc=$rc)" +exit $rc diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 1b9311eda..d585de12e 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -387,11 +387,22 @@ else SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" fi +# Patch the pinned image before serving. Covered by docs/waiver/.md per +# docs/PR_REVIEW_CHECKLIST.md. The script is strictly additive and idempotent; +# see its header for what is carried and, just as importantly, what is not. +bash "$(dirname "$0")/apply_dsv4_container_patches.sh" + echo "Starting vllm server..." set -x export VLLM_ROCM_USE_AITER=1 #export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export VLLM_ROCM_USE_AITER_MOE=1 +# vllm #51714's opt-in gluon sparse-MLA decode kernel for gfx950. Read straight +# from os.environ by the call site (rocm_aiter_mla_sparse.py), not through +# envs.py, so vLLM logs it as an "Unknown vLLM environment variable" -- that +# warning is expected and does not mean the knob was ignored. Grep the server +# log for the kernel's own line to confirm the path was taken. +export VLLM_ROCM_DSV4_SPARSE_GLUON=${VLLM_ROCM_DSV4_SPARSE_GLUON:-1} sleep 180 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..adf73c6b3 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1301,7 +1301,19 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: # is no longer available from Docker Hub. Throughput uses the three-token golden # synthetic acceptance length, while eval-only runs retain real verification. dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c + # Pinned to the 08-09 nightly. This is the base the local bring-up actually + # serves on: with benchmarks/single_node/agentic/apply_dsv4_container_patches.sh + # it completes weight load, the profile run and cudagraph capture, and answers + # an 8k-in/1k-out sweep at concurrency 32. + # + # Do NOT bump this to a newer nightly without re-running that bring-up. The + # 08-12 nightly (3ee2df30) memory-faults during the profile run on this + # config, and it does so on the PRISTINE image as well as the patched one -- + # eight workers serialize on /tmp/aiter_configs/bf16_tuned_gemm.csv.lock and + # the one holding the baton dies in the post-MoE bf16 GEMM. That is a + # regression in the base, not something this recipe's patches introduce, and + # it is under separate investigation. + image: vllm/vllm-openai-rocm:nightly-f8d03e77416bf90c49acbe50e233275722f02c4b model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/docs/waiver/2589.md b/docs/waiver/2589.md new file mode 100644 index 000000000..df9892a20 --- /dev/null +++ b/docs/waiver/2589.md @@ -0,0 +1,151 @@ +# Inference-engine patch waiver — PR #2589 + +Filed per [`docs/PR_REVIEW_CHECKLIST.md`](../PR_REVIEW_CHECKLIST.md) (image-provenance / patch item): a +benchmark script in this PR patches the pinned serving-stack image before serving, which the checklist +prohibits unless covered by a filled-out waiver. + +## Config covered + +- **Master config entry:** `dsv4-fp4-mi355x-vllm-agentic-mtp` in [`configs/amd-master.yaml`](../../configs/amd-master.yaml) +- **Pinned image:** `vllm/vllm-openai-rocm:nightly-f8d03e77416bf90c49acbe50e233275722f02c4b` + (vllm `0.26.1rc1.dev528+gf8d03e774`) +- **Patch entrypoint:** `bash "$(dirname "$0")/apply_dsv4_container_patches.sh"` invoked from + [`benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh`](../../benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh) +- **Patch script:** [`benchmarks/single_node/agentic/apply_dsv4_container_patches.sh`](../../benchmarks/single_node/agentic/apply_dsv4_container_patches.sh) + +## What is patched + +Every change is sourced from an **upstream ref** — a vLLM PR `.diff`, or a pinned `ROCm/aiter` / +`ROCm/mori` commit. Nothing comes from a private or prebuilt measurement image. + +The script is **idempotent**: every step is marker-gated (file-content markers for the PR diffs, byte +comparison for the file syncs), and re-running it on an already-patched container is a verified no-op — +confirmed by running it twice in a row on a container created from the pinned image, with the second run +reporting `same` / `already present` for all 21 steps and `VERIFY: ALL_OK`. + +| Upstream ref | What it changes | Notes | +|---|---|---| +| vllm #51473 | native MXFP4 TP8 shard allocation (`vllm/model_executor/layers/fused_moe/oracle/mxfp4.py`) | **merged 2026-08-11, i.e. after this base.** Applied per-file: the PR carries a `tests/` hunk with no counterpart in an installed wheel | +| vllm #51714 | opt-in AITER gluon sparse-MLA decode for gfx950 (`vllm/v1/attention/ops/rocm_aiter_mla_sparse.py`) | open upstream. Dormant unless `VLLM_ROCM_DSV4_SPARSE_GLUON=1`. Requires the aiter facade below to be functional — see "The gluon facade" | +| vllm #51918 (**in full**, 6 files) | FlyDSL mega-MoE backend: `config/kernel.py` (backend name) plus `models/deepseek_v4/amd/{model,mtp,dspark,mega_moe_experts,mega_moe_runtime}.py` | open upstream. `model.py` needs `patch --fuzz=3`; the diff targets a tree a few days newer and the earlier hunks shift its line numbering. All 6 of its hunks land | +| `aiter@97d0c6e4` (7 files) | sparse gluon decode kernel **and its facade**; FSE (`fhmoe`); `ops/flydsl/moe_kernels.py` | see below | +| `mori@84a33cc` (7 files) | `mori/ir/flydsl/` (3 files) plus the `cov` cascade `ir/bitcode.py`, `jit/{cache,config,core}.py` | see below | + +### The gluon facade + +Copying only the gfx950 gluon kernel is **not** enough, and an earlier revision of this script made exactly +that mistake. The base's `aiter/ops/triton/attention/pa_decode_sparse.py` facade predates the +`extra_cache` / `extra_indices` / `extra_indptr` keywords that #51714's call site passes, so with +`VLLM_ROCM_DSV4_SPARSE_GLUON=1` every worker raised + +``` +TypeError: pa_decode_sparse() got an unexpected keyword argument 'extra_cache' +VLLM_ROCM_DSV4_SPARSE_GLUON: aiter gluon sparse decode failed; falling back to +the Triton path for the rest of this process. +``` + +on its first decode and latched onto the Triton fallback for the life of the process — i.e. the knob was +accepted, logged as on, and silently inert. The facade and the kernel it dispatches to are one unit and +move together. The script now asserts the accepted keywords by signature introspection, not by file +presence, precisely because file presence is what gave the false positive. + +### `mori.ir.flydsl` and the `cov` cascade + +MegaMoE's intranode dispatch/combine kernel does `import mori.ir.flydsl`, and the base ships mori without +that subpackage. It is three plain `.py` files — no compiled artifact. + +They cannot be dropped in alone. `mori/ir/flydsl/runtime.py` calls `find_bitcode(cov=6)` (FlyDSL needs +AMDGPU code-object ABI 600), and `cov` cascades through three more files the base predates — +`ir/bitcode.py`, `jit/cache.py`, `jit/core.py` (37 call sites upstream, **0** in the base) — plus +`jit/config.py` for `is_debuginfo_enabled`. Installing a subset yields, in order, +`TypeError: find_bitcode() got an unexpected keyword argument 'cov'` and then +`ImportError: cannot import name 'is_debuginfo_enabled'`. The whole cascade goes, or none of it does. + +`libmori_shmem_device.bc` is deliberately **not** shipped: it is a 485 KB LLVM IR build artifact. +`find_bitcode()` falls back to `mori.jit.core.ensure_bitcode()`, which compiles it in-container with hipcc +on first import (~1 min, logged as `[mori-jit] Compiling shmem device bitcode for gfx950 (nic=ionic, +cov=6)`). This keeps the patch stack free of binary payloads. + +### `moe_kernels.py` is replaced, not patched + +`aiter/ops/flydsl/moe_kernels.py` is the one file swapped wholesale (2409 → 3003 lines). FSE's +`aiter/ops/flydsl/fhmoe.py` imports `_flydsl_moe_stage1_impl` / `_flydsl_moe_stage2_impl`, which the base +simply does not define. Before replacing, base and upstream were symbol-diffed at the AST level: + +``` +base symbols: 58 upstream symbols: 66 +LOST if replaced: 0 +GAINED: 8 (incl. _flydsl_moe_stage1_impl, _flydsl_moe_stage2_impl) +``` + +Zero symbols lost, so no in-tree caller can break on the swap. + +### aiter #4417 is *not* carried separately + +An earlier revision grafted aiter #4417 (large-token FlyDSL MoE launch/output limits) by anchor, via a +`graft_aiter_4417.py` helper. That helper is **gone** — both for a correctness reason and a scope reason: + +- Its two guards cannot fire on this config. `requires_flydsl_stage2_reduce` first fires at 299594 tokens; + this recipe's ceiling is 65536 (`max_num_batched_tokens` 16384 × the MTP fan-out of 4), 4.6× below the + threshold. `resolve_flydsl_grid_y_persist_m` guards `grid.y > 65535`; at 65536 tokens `num_m_blocks` is + 8192 and it returns `persist_m=1`, i.e. inert, with 8× of headroom. The sweep grid varies concurrency + only, never `max_num_batched_tokens`. +- It is redundant anyway: the `aiter@97d0c6e4` `moe_kernels.py` installed above already contains both + guards (verified, `grep -c` = 5). + +### What is *not* touched + +`aiter/jit/core.py` is left alone. The base already exposes `_set_current_hip_stream`, which the nightly +vLLM calls for `module_rmsnorm_quant`; an earlier wholesale post2-python overlay regressed it and crashed +rmsnorm_quant warmup. + +## Why the unmodified upstream image cannot run this benchmark + +Two separate problems, at two different pins. + +**1. The previous pin (`nightly-821717118f…`, and the 08-12 `nightly-3ee2df30…` line) memory-faults before +serving a single request.** During the profile run the eight TP workers serialize on +`/tmp/aiter_configs/bf16_tuned_gemm.csv.lock`; the worker holding the baton dies in the post-MoE bf16 GEMM +(`Memory access fault by GPU node-N … Reason: Unknown`) while the other seven survive parked on the lock. +This reproduces on a **pristine, completely unpatched** 08-12 nightly, so it is a base regression, not +something the patches here introduce or fix. Two standalone repros clear the loudest suspects: the +`inter_dim=384` MoE returns finite results at the exact faulting shape with preshuffled weights and the +same kernel pair, and the bf16 GEMM passes on all four paths (`gemm_a16w16_opus`, asm, triton, torch) at +both M=65536 and M=16384 × N=7168 × K=7168. It is non-deterministic across runs (different worker, +different address, different kernel selection) and is tracked separately upstream. + +**2. The 08-09 base this PR repins to is stable, but predates vllm #51473.** That PR merged 2026-08-11 and +supplies the native MXFP4 TP8 shard allocation this DSv4-Pro FP4 config needs; the 08-12 line already +carried it, the 08-09 line does not. Without it the config cannot serve on the older, working base. + +So neither pin serves this config unmodified: the newer one faults, and the older one is missing a +merged-upstream fix. Patching the older base is the only combination that reaches +`Application startup complete` today. + +Verified on the patched 08-09 image (hold job 6707, node crsuse2-m2m-100): weight load, profile run and +cudagraph capture all complete with fault=0, `compilation_config` correctly retains +`mode: VLLM_COMPILE: 3` / `cudagraph_mode: FULL_AND_PIECEWISE`, and an 8k-in/1k-out `vllm bench serve` run +at concurrency 32 completes 320/320 requests with 0 failures at an MTP acceptance length of 2.49 — +matching the pinned `synthetic_acceptance_length` of 2.49. + +## Upstream PR / issue links + +- vLLM: https://github.com/vllm-project/vllm/pull/51473 (merged 2026-08-11), + https://github.com/vllm-project/vllm/pull/51714 (open), + https://github.com/vllm-project/vllm/pull/51918 (open) +- AITER: `aiter@97d0c6e4cb7a0919c12291c7c7d560ad412f15c1` — + https://github.com/ROCm/aiter/commit/97d0c6e4cb7a0919c12291c7c7d560ad412f15c1 + (includes https://github.com/ROCm/aiter/pull/4417, merged 2026-07-30) +- mori: `mori@84a33cc0f15f019c78c995728973b70ea3d10bb7` — + https://github.com/ROCm/mori/commit/84a33cc0f15f019c78c995728973b70ea3d10bb7 + +## Removal plan + +Retire `apply_dsv4_container_patches.sh` and its invocation from `dsv4_fp4_mi355x_vllm_mtp.sh` once a ROCm +vLLM nightly satisfies all of: it carries merged vllm #51473, it ships the gfx950 `pa_decode_sparse` gluon +kernel **together with a facade that accepts the `extra_*` keywords**, its vendored aiter is at or past +`97d0c6e4` and its mori ships `mori/ir/flydsl/` with the `cov` cascade, and it does **not** carry the +profile-run bf16-GEMM memory fault described above. At that point bump the pinned `image:` for +`dsv4-fp4-mi355x-vllm-agentic-mtp` to that nightly, drop the patch script and the +`bash …/apply_dsv4_container_patches.sh` call, and delete this waiver in the same PR. The vllm #51714 and +#51918 hunks can be dropped as soon as those PRs land upstream, independently of the fault. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..9f960569f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,18 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin the image to the 08-09 nightly (f8d03e77) and patch it before serving via benchmarks/single_node/agentic/apply_dsv4_container_patches.sh, so the config can actually start: the previously pinned nightly memory-faults during the profile run on this recipe" + - "The fault is in the base, not in the patches -- it reproduces identically on a pristine 08-12 nightly (3ee2df30) with no patches applied at all: eight TP workers serialize on /tmp/aiter_configs/bf16_tuned_gemm.csv.lock and the worker holding the baton dies in the post-MoE bf16 GEMM" + - "Patch stack is idempotent and entirely upstream-sourced: vllm #51473 (native MXFP4 TP8 shard allocation, merged 2026-08-11 i.e. after this base), vllm #51714 (opt-in AITER gluon sparse-MLA decode for gfx950), vllm #51918 in full across 6 files (the flydsl_mega_moe backend name plus the MegaMoE model side), 7 files from aiter@97d0c6e4 and 7 from mori@84a33cc" + - "The gluon knob was silently dead before this: copying only the gfx950 kernel left the base's older aiter pa_decode_sparse facade in place, which rejects the extra_cache/extra_indices/extra_indptr keywords #51714's call site passes, so every worker hit a TypeError on first decode and latched onto the Triton fallback for the life of the process. The facade now moves with the kernel, and the script asserts the accepted keywords by signature introspection rather than by file presence" + - "MegaMoE and FSE are now reachable on this base: mori/ir/flydsl (3 .py files) plus the cov cascade it needs (ir/bitcode.py, jit/{cache,config,core}.py -- 37 cov call sites upstream, 0 in the base), and aiter fhmoe plus the moe_kernels.py that defines its stage1/stage2 impls. moe_kernels.py is the one wholesale replacement, AST symbol-diffed first: 0 symbols lost, 8 gained" + - "aiter #4417 is no longer grafted separately -- it is already contained in the aiter@97d0c6e4 moe_kernels.py, and neither of its guards can fire on this recipe anyway (stage2 reduce first fires at 299594 tokens vs a 65536 ceiling; grid.y persist_m is inert with 8x headroom)" + - "aiter core is not modified: the base already exposes _set_current_hip_stream, which the nightly vLLM needs for module_rmsnorm_quant. No binary payloads are shipped -- libmori_shmem_device.bc is JIT-compiled in-container with hipcc on first import" + - "Enable VLLM_ROCM_DSV4_SPARSE_GLUON=1 by default (overridable); the knob is read via os.environ rather than envs.py, so vLLM logs it as an unknown environment variable" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2589