Puzzletron v2 sync 20260730

.agents/scripts/nel-next.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# nel-next.sh — run NeMo Evaluator "next" (nel 0.3.x) via uvx, isolated from 0.2.6.
+#
+# A few AA benchmarks (Terminal-Bench 2.x, SWE-bench) run on `nemo-evaluator`
+# 0.3.x + `harbor` extra — a different package/CLI/config-schema from the eval
+# skill's default `nemo-evaluator-launcher` 0.2.6 (CLI `nel eval run X.yaml`,
+# overrides `-O a.b.c=v`, schema services/benchmarks/cluster/output). Installing
+# it into the 0.2.6 env would clobber `nel`, so this runs 0.3.x in a throwaway
+# `uvx` environment (uv resolves + caches + reuses it) and forwards to its `nel`.
+#
+# Usage (source .env FIRST so the config's ${VAR}s resolve; this never reads secrets):
+#   .agents/scripts/nel-next.sh --setup-only|--which|--version
+#   .agents/scripts/nel-next.sh eval run <config.yaml> [--dry-run|--submit] [-O k=v ...]
+#   .agents/scripts/nel-next.sh eval {status|logs|report|merge|resume|stop} -r <run_id>
+#   .agents/scripts/nel-next.sh mlflow-push -r <run_id> -c <config.yaml> [-- -o k=v ...]
+#     Post-run: SLURM doesn't auto-export. Pulls the merged bundle(s) + pushes to MLflow
+#     using the config's export_config.mlflow (resolves ${MLFLOW_TRACKING_URI}, forces
+#     emit_traces=false to avoid the per-sample hang). Run after `source .env`.
+#
+# Install source (env overrides): NEL_NEXT_SPEC (PyPI default, "nemo-evaluator[harbor,export]==0.3.*"
+# — [export] pulls mlflow for mlflow-push; pin an exact 0.3.x here for reproducibility), or
+# NEL_NEXT_ORIGIN [+ NEL_NEXT_REF] for the internal git build. uv caches the resolved env and
+# refreshes it when the spec changes.
+set -euo pipefail
+
+# [harbor] = agentic/sandbox deps; [export] pulls mlflow for `mlflow-push`.
+NEL_NEXT_SPEC="${NEL_NEXT_SPEC:-nemo-evaluator[harbor,export]==0.3.*}"
+NEL_NEXT_ORIGIN="${NEL_NEXT_ORIGIN:-}"
+NEL_NEXT_REF="${NEL_NEXT_REF:-}"
+
+if [[ -n "$NEL_NEXT_ORIGIN" ]]; then
+  INSTALL_SPEC="nemo-evaluator[harbor,export] @ ${NEL_NEXT_ORIGIN}${NEL_NEXT_REF:+@${NEL_NEXT_REF}}"
+else
+  INSTALL_SPEC="$NEL_NEXT_SPEC"
+fi
+
+_log() { printf '\033[2m  %s\033[0m\n' "$*" >&2; }
+# Run a command (nel, python, …) from the uvx-managed 0.3.x environment.
+_uvx() { uvx --python 3.12 --from "$INSTALL_SPEC" "$@"; }
+
+# Post-run MLflow push. SLURM runs don't auto-export; this stages each merged
+# benchmark bundle's eval-*.json from the cluster (the dev box doesn't mount the
+# run dir) and exports it to MLflow with traces off (avoids the per-sample hang).
+_mlflow_push() {
+  local rid="" cfg=""; local -a extra=()
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      -r|--run-id) rid="$2"; shift 2 ;;
+      -c|--config) cfg="$2"; shift 2 ;;
+      --)          shift; extra+=("$@"); break ;;
+      *)           extra+=("$1"); shift ;;
+    esac
+  done
+  [[ -n "$rid" && -n "$cfg" ]] || { echo "usage: nel-next.sh mlflow-push -r <run_id> -c <config.yaml> [-- -o k=v ...]" >&2; return 2; }
+  [[ -f "$cfg" ]] || { echo "ERROR: config not found: $cfg" >&2; return 2; }
+
+  # Derive cluster + mlflow settings from the config (literals; no env interpolation needed).
+  local cfgvars
+  cfgvars="$(_uvx python - "$cfg" <<'PY'
+import os, sys, yaml, json, shlex
+c = yaml.safe_load(open(sys.argv[1])) or {}
+cl = c.get("cluster", {}) or {}
+out = c.get("output", {}) or {}
+ml = ((out.get("export_config", {}) or {}).get("mlflow", {})) or {}
+# Resolve ${MLFLOW_TRACKING_URI} (set by modelopttools:eval-config, sourced from .env).
+turi = os.path.expandvars(ml.get("tracking_uri") or "")
+# Fall back to the canonical host if unset/unresolved or the broken mlflow-nemo-evaluator
+# alias (its 308 redirect strips /api/... -> REST 405).
+if (not turi) or turi.startswith("${") or ("mlflow-nemo-evaluator" in turi):
+    turi = os.getenv("MLFLOW_TRACKING_URI") or "https://mlflow.frontier-evals.nvidia.com/"
+def emit(k, v): print(f"{k}={shlex.quote(str(v))}")
+emit("MLP_HOST", cl.get("hostname", "")); emit("MLP_USER", cl.get("username", ""))
+emit("MLP_OUTDIR", out.get("dir", "")); emit("MLP_TURI", turi)
+emit("MLP_EXP", ml.get("experiment_name", "") or ""); emit("MLP_DESC", ml.get("description", "") or "")
+emit("MLP_TAGS", json.dumps(ml.get("tags") or {}))
+PY
+)" || { echo "ERROR: failed to parse $cfg" >&2; return 1; }
+  local MLP_HOST MLP_USER MLP_OUTDIR MLP_TURI MLP_EXP MLP_DESC MLP_TAGS
+  eval "$cfgvars"
+  [[ -n "$MLP_HOST" && -n "$MLP_OUTDIR" ]] || { echo "ERROR: cluster.hostname / output.dir missing in $cfg" >&2; return 1; }
+
+  local sshdest="${MLP_USER:+$MLP_USER@}$MLP_HOST" run="$MLP_OUTDIR/$rid"
+  _log "Locating merged bundles under $run on $sshdest …"
+  local -a benchdirs
+  mapfile -t benchdirs < <(ssh -o BatchMode=yes "$sshdest" \
+    "find '$run' -mindepth 2 -maxdepth 2 -name 'eval-*.json' -not -path '*/shard_*' -printf '%h\n' 2>/dev/null | sort -u")
+  [[ ${#benchdirs[@]} -gt 0 ]] || { echo "ERROR: no merged eval-*.json under $run — run 'nel-next.sh eval merge -r $rid' first" >&2; return 1; }
+
+  local staged; staged="$(mktemp -d "${TMPDIR:-/tmp}/nel-mlflow-push.XXXXXX")"
+  trap 'rm -rf "$staged"' RETURN
+  local -a localbundles=(); local b name
+  for b in "${benchdirs[@]}"; do
+    name="$(basename "$b")"; mkdir -p "$staged/$name"
+    rsync -rt --no-perms --no-group -e "ssh -o BatchMode=yes" "$sshdest:$b/eval-*.json" "$staged/$name/" >&2
+    localbundles+=("$staged/$name"); _log "staged bundle: $name"
+  done
+
+  local -a oargs=(-o "tracking_uri=$MLP_TURI" -o emit_traces=false)
+  [[ -n "$MLP_EXP"  ]] && oargs+=(-o "experiment_name=$MLP_EXP")
+  [[ -n "$MLP_DESC" ]] && oargs+=(-o "description=$MLP_DESC")
+  [[ "$MLP_TAGS" != "{}" ]] && oargs+=(-o "tags=$MLP_TAGS")
+  _log "Exporting ${#localbundles[@]} bundle(s) to MLflow: $MLP_TURI"
+  MLFLOW_TRACKING_URI="$MLP_TURI" _uvx nel export "${localbundles[@]}" --dest mlflow "${oargs[@]}" "${extra[@]}"
+}
+
+# --help needs no environment; handle it before requiring uvx.
+case "${1:-}" in
+  -h|--help) awk '/^# nel-next\.sh/{p=1} /^set /{p=0} p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+esac
+
+command -v uvx >/dev/null 2>&1 || { echo "ERROR: 'uvx' not found (curl -LsSf https://astral.sh/uv/install.sh | sh)" >&2; exit 1; }
+
+case "${1:-}" in
+  --setup-only) _uvx nel --version >/dev/null 2>&1 && _log "nel-next ready — ${INSTALL_SPEC}"; exit 0 ;;
+  --which)      echo "uvx --python 3.12 --from \"${INSTALL_SPEC}\" nel"; exit 0 ;;
+  --version)    _uvx python -c 'import nemo_evaluator; print(nemo_evaluator.__version__)'; exit 0 ;;
+  mlflow-push)  _mlflow_push "${@:2}"; exit $? ;;
+  "")           echo "ERROR: no args. Try: nel-next.sh eval run <config.yaml> [--dry-run]  (or --help)" >&2; exit 2 ;;
+esac
+
+exec uvx --python 3.12 --from "$INSTALL_SPEC" nel "$@"

.agents/skills/benchmark-model-kernels/SKILL.md

@@ -0,0 +1,145 @@
+---
+name: benchmark-model-kernels
+description: >-
+  Inspect Hugging Face decoder layers on meta tensors and plan or run per-rank
+  BF16, FP8, and NVFP4 GEMM or fused-MoE microbenchmarks with the bundled
+  scripts and a local FlashInfer checkout. Use when choosing a model, GPU, TP,
+  EP, or M/token-concurrency sweep; deriving common fused QKV and gate/up
+  shapes without loading checkpoint weights; or using FlashInfer benchmark
+  utilities. Do not use for end-to-end server throughput or request latency.
+---
+
+# Benchmark Model Kernels
+
+Plan a single-GPU microbenchmark for the per-rank shapes of an intended
+deployment. The scripts never load checkpoint weights, launch distributed
+workers, or measure collectives, serving throughput, or request latency.
+
+## Interview, one decision per message
+
+Ask exactly one unresolved decision per message with two or three concrete
+options, state the default, and wait for the answer. Recommend an option only
+when it is substantively better. Skip decisions already answered. Follow this
+order:
+
+1. **Model** — a local directory, a `config.json`, or a Hub ID (equivalent; no
+   default). The script builds the model on meta tensors from configuration
+   only. Do not enable `--trust-remote-code` without explicit approval; pin
+   `--revision <commit>` when it is needed.
+2. **TP and EP** — accept both directly, or derive them from the intended
+   deployment's GPU model and count and confirm. Defaults: `TP=1`, `EP=1`.
+   Derive parallelism from the intended deployment, never from the GPU that
+   runs the microbenchmark. The script validates every sharding rule
+   (divisibility, GQA replication, expert partitioning) and errors loudly.
+3. **M sweep** — balanced default (recommended): `1 8 64 512`; decode-focused:
+   `1 4 16 32`; throughput-focused: `64 256 1024 4096`. M is roughly the
+   tokens scheduled per step (decode: active sequences), not endpoint
+   concurrency. With EP, an MoE row models one rank's share of the global
+   batch: a global batch of B tokens corresponds to the column M = B/EP, so
+   do not compare different EP values at the same M.
+4. **Shape preview** — always run this before anything else; no FlashInfer or
+   GPU needed:
+
+   ```bash
+   python .agents/skills/benchmark-model-kernels/scripts/benchmark_model.py <model> \
+     --tp <tp> --ep <ep> --ms <m1> <m2> ... --print_only
+   ```
+
+   Review the printed shapes and the MoE tuple with the user.
+   `# unsupported:` lines mean the list is partial and the script exits
+   nonzero — handle those via **Manual supplements** below.
+5. **FlashInfer checkout and GPU** — the full benchmark needs a FlashInfer
+   *source checkout* containing `benchmarks/flashinfer_benchmark.py`; the
+   installed wheel alone is not enough. Prefer a clean checkout matching the
+   installed `flashinfer` version; ask before cloning or installing anything.
+   On the benchmark machine, check `nvidia-smi` and package versions, verify
+   the target GPU is idle (concurrent work on the same GPU skews timings),
+   and verify CUPTI timing with a tiny `bench_gpu_time(..., enable_cupti=True)`
+   probe — a warning that falls back to CUDA events is a failure (the
+   `cupti-python`/`nvidia-cuda-cupti` packages must match PyTorch's CUDA
+   major). Ask for a GPU index only when several are visible. Pick a fresh
+   workdir and state it.
+6. **Full benchmark**:
+
+   ```bash
+   CUDA_VISIBLE_DEVICES=<gpu-index> \
+   python .agents/skills/benchmark-model-kernels/scripts/benchmark_model.py <model> \
+     --tp <tp> --ep <ep> --ms <m1> <m2> ... \
+     --flashinfer_repo <flashinfer-repo> --workdir <workdir>
+   ```
+
+   Run a short plumbing check first (append
+   `--dry_run_iters 1 --num_iters 3 --no_autotune`, throwaway workdir),
+   especially after changing FlashInfer versions or shape logic; never present
+   it as a performance result. Then run with defaults. The first fused-MoE
+   build or autotune can take several minutes; its cache is reused.
+
+## Rules the scripts own
+
+Do not restate, re-derive, or override these — the code enforces them:
+
+- `benchmark_model.py` derives `--nks` (as `N,K,module-name` triples) and
+  all `--moe_*` arguments; overrides are rejected.
+- Row labels are logical shapes; the runner applies vLLM's physical padding.
+  Report both shapes whenever they differ.
+- A failed case writes FlashInfer's error message (or a pointer to
+  `driver.log`) into its `combined_results.csv` cell, and the command exits
+  nonzero after the table is written. Never present a partial table as a
+  successful benchmark; read `driver.log` before rerunning anything.
+
+## Known backend and driver limits
+
+- `mm_fp4` TensorRT-LLM needs `N % 128 == 0` (shuffled weight layout): its
+  cell reports no result row, or on stock 0.6.x drivers an empty assertion
+  that fails every `mm_fp4` backend for that shape.
+- `mm_fp8` (trtllm_low_latency) needs `K % 128 == 0`.
+- MoE rows cover FlashInfer's CUTLASS fused MoE, the trtllm-gen NVFP4 and
+  per-tensor FP8 MoE, and the CuteDSL NVFP4 MoE. The CuteDSL MoE row appears
+  only for Swiglu models (the kernel supports nothing else); the `cutedsl` and
+  `trtllm` backends require recent GPUs and report per-case errors elsewhere.
+- Gated NVFP4 CUTLASS MoE with `2F % 128 != 0` per rank fails — vLLM raises
+  instead of padding — often as a `CUDA error: misaligned address`. Prefer EP
+  over TP for the experts to keep the per-rank width legal.
+- Do not benchmark a padded shape to dodge these limits and call it
+  vLLM-equivalent; report the limit instead.
+
+## Manual supplements
+
+For each `# unsupported:` layout from the preview: inspect that module's
+forward path and the intended runtime's TP/EP sharding, derive the per-rank
+shape (never guess sharding from a weight shape alone), and benchmark only the
+missing shape:
+
+```bash
+CUDA_VISIBLE_DEVICES=<gpu-index> \
+python .agents/skills/benchmark-model-kernels/scripts/benchmark_via_builtin.py \
+  --flashinfer_repo <flashinfer-repo> --ms <m1> <m2> ... \
+  --nks <n>,<k>,<name> --workdir <workdir>
+```
+
+Label these rows as manual supplements; this is not a substitute for running
+`benchmark_model.py` first. Shell-quote user-supplied paths.
+
+## Report
+
+Report the command, GPU, versions, TP/EP, M values, shapes (logical and
+physical where they differ), warnings, and the artifacts: `testlist.txt`,
+`driver.log` (full driver output), `builtin_results.csv` (milliseconds,
+success-only), and `combined_results.csv` (long form, microseconds: columns `module_name,
+M, N, K, backend, with_quant, runtime` in `GEMM` and `MoE` sections; modules
+fused into one GEMM are joined with `|` in `module_name`, while distinct
+same-shape modules appear as duplicated rows sharing one measurement; MoE
+rows keep the `H= F= E= top_k=` parameter line and leave N/K empty). In quantization-recipe terms:
+`bf16` rows are the unquantized W16A16 baseline, `fp8` rows are per-tensor
+W8A8, and `nvfp4` rows are W4A4. Plain quantized rows time the kernel with
+pre-quantized activations; `*_with_quant` rows add a separately measured
+activation-quantization time in the scale-factor layout that backend
+consumes, except the NVFP4 CUTLASS MoE row, which is a single fused
+measurement. MoE routing is synthetic: uniform expert
+distribution everywhere (real skewed routing is slower), and the trtllm-gen
+rows, which route in-kernel, use a fixed `renormalize` method regardless of
+the model's routing scheme. CUTLASS and CuteDSL MoE rows receive precomputed
+expert indices and exclude routing-selection cost entirely. No row includes
+the router GEMM itself. These are kernel times
+only — never describe them as end-to-end latency or throughput; they omit
+weights, layer frequency, communication, KV cache, and scheduling.