[AMD] [AGENTX] GLM-5.2 FP4 MI355X ATOM Agentic MTP #2576
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| #!/usr/bin/env bash | ||
| set -eo pipefail | ||
| set -x | ||
|
|
||
| # Agentic trace replay benchmark for GLM5.2 FP4 on MI355X using ATOM MTP | ||
| # | ||
| # Required env vars: | ||
| # MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, | ||
| # TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION | ||
|
|
||
| source "$(dirname "$0")/../../benchmark_lib.sh" | ||
|
|
||
| check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION | ||
|
|
||
| echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" | ||
|
|
||
| if [[ -v SLURM_JOB_ID ]]; then | ||
| echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" | ||
| fi | ||
|
|
||
| # ROCR/HIP visibility for vLLM 0.14+ | ||
| if [[ -v ROCR_VISIBLE_DEVICES ]]; then | ||
| export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" | ||
| fi | ||
|
|
||
| if [[ -n "$MODEL_PATH" ]]; then | ||
| if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then | ||
| hf download "$MODEL" --local-dir "$MODEL_PATH" | ||
| fi | ||
| else | ||
| hf download "$MODEL" | ||
| export MODEL_PATH="$MODEL" | ||
| fi | ||
|
|
||
| rocm-smi || true | ||
| amd-smi || true | ||
|
|
||
| resolve_trace_source | ||
| install_agentic_deps | ||
|
|
||
| # Require the vLLM Prometheus stream in every official result. AIPerf | ||
| # deduplicates this endpoint against its automatic localhost discovery. | ||
| export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" | ||
| export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" | ||
|
|
||
| # VRAM space check | ||
| wait_for_amd_gpu_clean | ||
|
|
||
| # ---- Server config ---------------------------------------------------------- | ||
| SERVER_LOG="$RESULT_DIR/server.log" | ||
| LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" | ||
| mkdir -p "$RESULT_DIR" | ||
|
|
||
| SERVER_PID="" | ||
| LMCACHE_PIDS=() | ||
| cleanup_agentic_services() { | ||
| local exit_code=$? | ||
| trap - EXIT INT TERM | ||
| set +e | ||
| stop_background_process_tree "$SERVER_PID" "ATOM server" 60 | ||
| local i | ||
| for i in "${!LMCACHE_PIDS[@]}"; do | ||
| stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" | ||
| done | ||
|
Comment on lines
+51
to
+64
Contributor
There was a problem hiding this comment. 🟡 Dead LMCache server-process scaffolding was copy-pasted from the vLLM script this recipe was adapted from: Extended reasoning...This script was adapted from This GLM-5.2 ATOM recipe took a different architectural path for KV offload — it configures the in-process
Step-by-step proof: (1) This has no functional/runtime impact — an empty-array loop is harmless — but it is misleading dead code: a future maintainer reading this script would reasonably assume an external LMCache server process is started and tracked here, when in fact KV offload is entirely in-process. Recommend removing |
||
| exit "$exit_code" | ||
| } | ||
| trap cleanup_agentic_services EXIT | ||
| trap 'exit 130' INT | ||
| trap 'exit 143' TERM | ||
|
|
||
| OFFLOAD_ARGS=() | ||
|
|
||
| case "$KV_OFFLOAD_BACKEND" in | ||
| "") | ||
| require_agentic_kv_offload_none | ||
| ;; | ||
| lmcache) | ||
| require_agentic_kv_offload_backend lmcache | ||
|
|
||
| # LMCache settings | ||
| export PYTHONHASHSEED=0 | ||
| export LMCACHE_LOCAL_CPU=True | ||
| export LMCACHE_MAX_LOCAL_CPU_SIZE="$TOTAL_CPU_DRAM_GB" | ||
| export LMCACHE_CHUNK_SIZE=256 | ||
| export OFFLOAD_MIN_LOAD_TOKENS=8192 | ||
|
seungrokj marked this conversation as resolved.
|
||
|
|
||
| OFFLOAD_ARGS=( | ||
| --kv-transfer-config | ||
| "{\"kv_connector\":\"lmcache_offload\",\"kv_role\":\"offload\"}" | ||
| ) | ||
| ;; | ||
| *) | ||
| echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 | ||
| exit 1 | ||
| ;; | ||
| esac | ||
|
|
||
| # ---- LLM server config ---------------------------------------------------------- | ||
|
|
||
| echo "Starting atom server..." | ||
| export PYTHONNOUSERSITE=1 | ||
|
|
||
| # ---- ATOM env ---- | ||
| export AITER_QUICK_REDUCE_QUANTIZATION=INT4 | ||
| export AITER_USE_FLYDSL_MOE_SORTING=1 | ||
|
|
||
| # CUDA/HIPGRAPH settings | ||
| case "$CONC" in | ||
| 1) CUDAGRAPH_CAPTURE_SIZES='[1,2]' ;; | ||
| 2) CUDAGRAPH_CAPTURE_SIZES='[1,2,4]' ;; | ||
| 4) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8]' ;; | ||
| 8) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16]' ;; | ||
| 10) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16,20]' ;; | ||
| 12) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16,20,24]' ;; | ||
| 16) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16,20,24,28,32]' ;; | ||
| *) | ||
| echo "Unsupported CONC=$CONC" >&2 | ||
| exit 2 | ||
| ;; | ||
| esac | ||
|
|
||
| # PARALLEL settings | ||
| PARALLEL_ARGS=(--tensor-parallel-size "$TP") #TP | ||
| if [ "$DP_ATTENTION" = "true" ]; then | ||
| # DPA+EP | ||
| if [ "$EP_SIZE" -gt 1 ]; then #DP+EP | ||
| PARALLEL_ARGS=(--tensor-parallel-size "$TP" --enable-dp-attention --enable-expert-parallel) | ||
| # DPA+TP | ||
| else | ||
| PARALLEL_ARGS=(--tensor-parallel-size "$TP" --enable-dp-attention ) | ||
| fi | ||
| fi | ||
|
|
||
| # SPEC settings | ||
| # SIMULATE_ACC_LEN and NUM_SPEC_TOKENS reference: | ||
| # https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/glm5.2_mtp.yaml | ||
| SIMULATE_ACC_LEN=2.99 | ||
| NUM_SPEC_TOKENS=3 | ||
| # spec-decode-acceptance-rate = (SIMULATE_ACC_LEN - 1) / NUM_SPEC_TOKENS | ||
| SPEC_ACCEPTANCE_RATE=$(awk "BEGIN{print ($SIMULATE_ACC_LEN-1)/$NUM_SPEC_TOKENS}") | ||
|
Comment on lines
+137
to
+140
Collaborator
There was a problem hiding this comment. can u edit to have the atom way of setting fair mtp
Collaborator
Author
There was a problem hiding this comment. @functionstackx and this will reject based on the synthetic decay factor and they will behave the same as sgl's match-expected behavior but will cross check with the one who implemented this to make sure |
||
| if [ "${EVAL_ONLY}" = "true" ]; then | ||
| SPEC_ARGS=( | ||
| --method mtp | ||
| --num-speculative-tokens "$NUM_SPEC_TOKENS" | ||
| ) | ||
| else | ||
| SPEC_ARGS=( | ||
| --method mtp | ||
| --num-speculative-tokens "$NUM_SPEC_TOKENS" | ||
| --spec-decode-acceptance-rate "$SPEC_ACCEPTANCE_RATE" | ||
| ) | ||
| fi | ||
| echo "SIMULATE_ACC_LEN=$SIMULATE_ACC_LEN NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS SPEC_ACCEPTANCE_RATE=$SPEC_ACCEPTANCE_RATE" | ||
|
|
||
| ATOM_CMD=( | ||
| python -m atom.entrypoints.openai_server | ||
| --model "$MODEL_PATH" | ||
| --host 0.0.0.0 | ||
| --server-port "$PORT" | ||
| "${PARALLEL_ARGS[@]}" | ||
| --online_quant_config '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' | ||
| --max-num-seqs "$((2 * CONC))" | ||
| --cudagraph-capture-sizes "$CUDAGRAPH_CAPTURE_SIZES" | ||
| --max-num-batched-tokens 16384 | ||
| --kv_cache_dtype fp8 | ||
| "${SPEC_ARGS[@]}" | ||
| "${OFFLOAD_ARGS[@]}" | ||
| ) | ||
| write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" | ||
| "${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & | ||
| SERVER_PID=$! | ||
| echo "Server PID: $SERVER_PID" | ||
|
|
||
| wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" | ||
|
|
||
| # ---- Run benchmark ---------------------------------------------------------- | ||
| if [ "${EVAL_ONLY}" = "true" ]; then | ||
| run_eval --port "$PORT" | ||
| else | ||
| build_replay_cmd "$RESULT_DIR" | ||
| REPLAY_CMD+=" --apply-chat-template" | ||
| run_agentic_replay_and_write_outputs "$RESULT_DIR" | ||
| fi | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| """Native ATOM server metric adapter.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from typing import Any | ||
|
|
||
| from ..aggregation_common import gauge_stat, normalize_fraction, rate, sum_stat | ||
| from .base import ServerMetricsBackend, counter_int | ||
|
|
||
|
|
||
| def _atom_names(suffix: str) -> list[str]: | ||
| """Accept direct worker metrics and Atomesh's colon-normalized form.""" | ||
| return [f"atom:{suffix}", f"atom_{suffix}"] | ||
|
|
||
|
|
||
| def _atom_counter_names(stem: str) -> list[str]: | ||
| """Accept Prometheus raw names and AIPerf's counter-family names.""" | ||
| return [*_atom_names(stem), *_atom_names(f"{stem}_total")] | ||
|
|
||
|
|
||
| class AtomBackend(ServerMetricsBackend): | ||
| name = "atom" | ||
|
|
||
| def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: | ||
| metric_names = set(metrics) | ||
| return any(name.startswith(("atom:", "atom_")) for name in metric_names) or ( | ||
| not metrics and framework.lower() == "atom" | ||
| ) | ||
|
|
||
| def populate( | ||
| self, | ||
| metrics: dict[str, dict[str, Any]], | ||
| flat: dict[str, Any], | ||
| nested: dict[str, Any], | ||
| ) -> None: | ||
| prompt_total = sum_stat( | ||
| metrics, | ||
| _atom_counter_names("prompt_tokens"), | ||
| preferred_keys=("total", "sum", "max", "avg"), | ||
| ) | ||
| generation_total = sum_stat( | ||
| metrics, | ||
| _atom_counter_names("generation_tokens"), | ||
| preferred_keys=("total", "sum", "max", "avg"), | ||
| ) | ||
| flat["total_prompt_tokens"] = counter_int(prompt_total) | ||
| flat["total_generation_tokens"] = counter_int(generation_total) | ||
|
|
||
| cached_tokens = sum_stat( | ||
| metrics, | ||
| _atom_counter_names("prefix_cache_cached_tokens"), | ||
| preferred_keys=("total", "sum", "max", "avg"), | ||
| ) | ||
| full_tokens = sum_stat( | ||
| metrics, | ||
| _atom_counter_names("prefix_cache_full_tokens"), | ||
| preferred_keys=("total", "sum", "max", "avg"), | ||
| ) | ||
| cache_hit_rate = rate(cached_tokens, full_tokens) | ||
| external_tokens = sum_stat( | ||
| metrics, | ||
| _atom_counter_names("lmcache_loaded_tokens"), | ||
| preferred_keys=("total", "sum", "max", "avg"), | ||
| ) | ||
| external_hit_rate = rate(external_tokens, full_tokens) | ||
| # ATOM's admitted cache counter may already include a completed | ||
| # LMCache load, so do not add external tokens a second time. | ||
| overall_hit_rate = cache_hit_rate | ||
|
|
||
| flat["server_gpu_cache_hit_rate"] = cache_hit_rate | ||
| flat["server_cpu_cache_hit_rate"] = external_hit_rate | ||
| flat["server_external_cache_hit_rate"] = external_hit_rate | ||
| flat["server_overall_cache_hit_rate"] = overall_hit_rate | ||
| flat["gpu_kv_cache_usage_pct"] = normalize_fraction( | ||
| gauge_stat( | ||
| metrics, | ||
| _atom_names("kv_cache_usage_ratio"), | ||
| preferred_keys=("max", "avg", "total"), | ||
| combine="max", | ||
| ) | ||
| ) | ||
|
|
||
| nested["cache"].update( | ||
| { | ||
| "gpu_cache_hit_rate": cache_hit_rate, | ||
| "cpu_cache_hit_rate": external_hit_rate, | ||
| "external_cache_hit_rate": external_hit_rate, | ||
| "overall_cache_hit_rate": overall_hit_rate, | ||
| "prefix_cache_hits": cached_tokens, | ||
| "prefix_cache_queries": full_tokens, | ||
| "external_prefix_cache_hits": external_tokens, | ||
| "external_prefix_cache_queries": full_tokens, | ||
| } | ||
| ) | ||
| nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] | ||
| nested["tokens"].update( | ||
| { | ||
| "prompt_total": flat["total_prompt_tokens"], | ||
| "generation_total": flat["total_generation_tokens"], | ||
| } | ||
| ) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔴 AGENTS.md declares a non-negotiable invariant: every recipe addition requires a new
perf-changelog.yamlentry, but this PR adds the brand-newglm5.2-fp4-mi355x-atom-agentic-mtprecipe (script +configs/amd-master.yamlentry) without one. The directly analogous sibling PR (glm5.2-fp4-mi355x-sglang-agentic-mtp) added the required 7-line entry alongside its script/config changes — this PR needs the equivalent append at the tail ofperf-changelog.yamlbefore merge.Extended reasoning...
The bug:
AGENTS.mdlines 19-21 list "Non-negotiable benchmark invariants," the first of which states: "Every change that can affect benchmark performance and every recipe addition or modification requires a newperf-changelog.yamlentry. The file is append-only and byte-sensitive. Preserve all existing bytes and separator whitespace, and append only at the tail."This PR adds a brand-new recipe key,
glm5.2-fp4-mi355x-atom-agentic-mtp, toconfigs/amd-master.yaml, plus the corresponding new benchmark scriptbenchmarks/single_node/agentic/glm5.2_fp4_mi355x_atom_mtp.sh. Both changes squarely fall under "recipe addition" per the invariant. Yet the PR diff touches only these two files —perf-changelog.yamlis not present in the diff at all.Why this isn't caught elsewhere: There's no CI lint step visible in this repo that mechanically enforces the invariant (it's documented policy, not a script-enforced check), so a reviewer/author has to remember to append the entry manually. The immediately preceding entry in
configs/amd-master.yaml,glm5.2-fp4-mi355x-sglang-agentic-mtp(the direct SGLang sibling of this ATOM recipe, same model/precision/SKU/scenario), did add itsperf-changelog.yamlentry in its own commit — confirming this is the expected, followed convention for this exact recipe family, not a check nobody actually does in practice.Impact: Without the changelog entry, there is no recorded baseline/description for this new recipe's expected performance characteristics, which is the entire purpose of the changelog per the AGENTS.md invariant — it's how the team tracks what performance-affecting changes were intentional versus regressions.
Proof / how to verify:
grep -n "atom-agentic-mtp" AGENTS.md configs/amd-master.yaml perf-changelog.yaml— the recipe key appears inconfigs/amd-master.yaml(this PR's diff) but not inperf-changelog.yaml.grep -n "sglang-agentic-mtp" perf-changelog.yamlshows the SGLang sibling's entry exists.benchmarks/single_node/agentic/glm5.2_fp4_mi355x_atom_mtp.sh,configs/amd-master.yaml) confirms only 2 files changed —perf-changelog.yamlis absent.Fix: Append a new entry to the tail of
perf-changelog.yaml(config-keys / description / pr-link, matching the format of theglm5.2-fp4-mi355x-sglang-agentic-mtpentry) forglm5.2-fp4-mi355x-atom-agentic-mtp, being careful to preserve existing bytes/whitespace and append-only per the invariant's own byte-sensitivity warning.