From 26ddf7c9e2903273b6581af0a31b6c253d0de112 Mon Sep 17 00:00:00 2001 From: Chong Jia Zhen <110578271+chongjiazhen@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:07:10 +0800 Subject: [PATCH] The build and the fetch tooling reach for python3 by name too Follow-up to the suite fix in #51, same finding one layer out: on Windows `python3` on PATH is usually the Microsoft Store App Execution Alias, a zero-byte reparse point that exists, exits 49, and prints an advert instead of running anything. run.sh can shim PATH for its own call sites, but that does not reach a recipe make runs itself, nor either tools script when run directly. Eleven call sites: three recipes, four in fetch_weights.sh, four in pipeline.sh. Makefile: serve-check, fuzz and fuzz-asan get $(PY). fetch_weights.sh and pipeline.sh get the same answer as $PY; pipeline.sh exports it so the fetch_weights.sh it drives inherits rather than resolving per stage. $(PY) is recursive rather than `:=` on purpose. Immediate assignment runs the probe on every make invocation, `make clean` and a no-op build included, and where python3 is the alias the first spawn is the alias itself: measured 196 ms per invocation on this box, which nearly triples a no-op make. Lazy, the probe runs only when a Python recipe expands it, and a no-op make is back to 0.664 s against 0.678 s on main. The fallback sits inside the shell for the same reason, since an ifeq on $(PY) would force the expansion at parse time. pipeline.sh is the one that misdiagnoses rather than merely failing: its first two probes end `2>/dev/null || echo 0`, so an interpreter that is not one yields 0 and the script dies "download (no index at $SRC)", naming a missing index that is present and readable. Nothing skips here. If no candidate answers, PY stays python3 and the recipe or script fails loudly at first use, which is right for a build target and for a 1.4 TB download: unlike the suite there is nothing to skip, and a run that cannot read its own index must stop. Verified on Windows 10, MinGW-w64 GCC 15.2. PY resolves to `python`; `make serve-check` reports OK (skipped=3) and `make fuzz FUZZ_RUNS=20` reports 20 cases, 0 crashed, 0 hung, both exit 0; with all three names shimmed to exit 49, serve-check fails loudly with Error 49 rather than skipping. The two tools scripts are not run end to end here, since that needs the full source weights: they carry bash -n plus a three-state check of the resolver. Assisted by AI. --- Makefile | 31 ++++++++++++++++++++++++++++--- tools/fetch_weights.sh | 28 ++++++++++++++++++++++++---- tools/pipeline.sh | 30 ++++++++++++++++++++++++++---- 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index f9d5504..9cbffb3 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,31 @@ CC ?= cc CFLAGS ?= -O2 -std=gnu11 -Wall -Wextra LDLIBS := -lm -lpthread +# The interpreter for the three Python recipes below, resolved by RUNNING a +# candidate rather than by looking one up. On Windows the name `python3` on +# PATH is usually the Microsoft Store App Execution Alias: a zero-byte +# reparse point that exists, exits 49, and prints an advert for the Store +# instead of running anything, so a recipe reports its own tool broken. Same +# finding as tests/run.sh, which shims PATH because it has 49 call sites and +# subprocesses to cover; three recipes want a name instead. PATH set inside +# run.sh does not reach a recipe make runs itself, which is why this is here +# and not there. +# +# Windows installs the versionless `python`, and `py` besides, so a working +# interpreter is normally there under another name. If none answers this +# stays `python3` and the recipe fails loudly at first use, which is right +# for a build target: unlike the suite, there is nothing here to skip. +# +# Recursive rather than `:=` so the probe runs only when a recipe actually +# expands $(PY). Immediate assignment costs every `make` invocation two +# process spawns, `make clean` and a no-op build included, and on the box +# this was found the first spawn is the Store alias itself: measured 196 ms +# added per invocation, which nearly triples a no-op make. The fallback +# lives inside the shell for the same reason, since an `ifeq` on $(PY) +# would force the expansion at parse time and undo it. +PY = $(shell for p in python3 python py; do \ + "$$p" -c '' >/dev/null 2>&1 && { echo "$$p"; exit 0; }; done; echo python3) + # The target triple rather than `uname -m`, because those differ the moment # anyone cross-compiles: building for Windows from this ARM laptop, uname # says arm64 and would put the NEON translation unit into an x86 binary @@ -299,7 +324,7 @@ check: test # libwaste as a shared object rather than the archive the CLI links, and # because it is the one part of this repo that is not C. serve-check: libwaste.$(SOEXT) - @python3 -m unittest discover -s tests/serve -t . -p "test_*.py" + @$(PY) -m unittest discover -s tests/serve -t . -p "test_*.py" # Sanitizers. Separate targets rather than a flag on `make`, because they # need a full rebuild: mixing instrumented and uninstrumented objects @@ -320,7 +345,7 @@ asan: $(MAKE) --no-print-directory clean; exit $$rc fuzz: test - @python3 tools/fuzz_container.py --runs $(FUZZ_RUNS) + @$(PY) tools/fuzz_container.py --runs $(FUZZ_RUNS) FUZZ_RUNS ?= 200 @@ -332,7 +357,7 @@ fuzz-asan: CFLAGS="-std=gnu11 -Wall -Wextra -DVQ_SUPER=$(VQ_SUPER) -MMD -MP $(SAN_FLAGS)" \ LDLIBS="-lm -lpthread $(SAN_FLAGS)" @rc=0; ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=print_stacktrace=1 \ - python3 tools/fuzz_container.py --runs $(FUZZ_RUNS) || rc=$$?; \ + $(PY) tools/fuzz_container.py --runs $(FUZZ_RUNS) || rc=$$?; \ $(MAKE) --no-print-directory clean; exit $$rc .PHONY: all test check serve-check clean asan fuzz fuzz-asan diff --git a/tools/fetch_weights.sh b/tools/fetch_weights.sh index 798b583..1df54e9 100755 --- a/tools/fetch_weights.sh +++ b/tools/fetch_weights.sh @@ -30,6 +30,26 @@ set -uo pipefail +# python3, resolved by RUNNING a candidate rather than by looking one up. +# On Windows the name on PATH is usually the Microsoft Store App Execution +# Alias: a zero-byte reparse point that exists, exits 49, and prints an +# advert for the Store instead of running anything. Same finding as +# tests/run.sh, which shims PATH because it has 49 call sites and +# subprocesses to cover; the handful here get a name and pass it down. +# +# Windows installs the versionless `python`, and `py` besides, so a working +# interpreter is normally there under another name. If none answers, this +# stays `python3` and the script fails loudly at first use, which is right +# here: unlike the suite there is nothing to skip, and a run that cannot +# read its own index must stop rather than carry on. +if [ -z "${PY:-}" ]; then + for _cand in python3 python py; do + "$_cand" -c '' >/dev/null 2>&1 && { PY="$_cand"; break; } + done + : "${PY:=python3}" +fi +export PY + REPO="${REPO:-moonshotai/Kimi-K3}" DEST="${DEST:-/Volumes/WasteDisk/k3}" JOBS="${JOBS:-3}" @@ -185,7 +205,7 @@ if [ "$DRY" = 0 ] && [ "$CHECK_ONLY" = 0 ]; then # missed preprocessor_config.json, so the image normalization was the # CLIP convention for a day when the release states mean = std = 0.5. # A whitelist cannot report what it never knew to ask for. - SMALL=$(hcurl -sfL --max-time 60 "$API" 2>/dev/null | python3 -c ' + SMALL=$(hcurl -sfL --max-time 60 "$API" 2>/dev/null | "$PY" -c ' import json, sys try: d = json.load(sys.stdin) @@ -247,14 +267,14 @@ fi # build that cannot be tested from here, and deleting a byte that must never # appear in a safetensors filename is checkable by reading. It is a no-op # everywhere else. -python3 - "$DEST/model.safetensors.index.json" <<'PY' | tr -d '\r' > "$DEST/.shards" +"$PY" - "$DEST/model.safetensors.index.json" <<'PY' | tr -d '\r' > "$DEST/.shards" import json, sys idx = json.load(open(sys.argv[1])) for s in sorted(set(idx["weight_map"].values())): print(s) PY TOTAL=$(wc -l < "$DEST/.shards" | tr -d ' ') -TOTAL_BYTES=$(python3 - "$DEST/model.safetensors.index.json" <<'PY' | tr -d '\r' +TOTAL_BYTES=$("$PY" - "$DEST/model.safetensors.index.json" <<'PY' | tr -d '\r' import json, sys print(json.load(open(sys.argv[1])).get("metadata", {}).get("total_size", 0)) PY @@ -272,7 +292,7 @@ while read -r f; do done < "$DEST/.shards" log "repo $REPO -> $DEST (stat: $STAT_MODE)" -python3 - "$TOTAL_BYTES" "$have_bytes" "$(free_kb)" "$TOTAL" "$have" <<'PY' +"$PY" - "$TOTAL_BYTES" "$have_bytes" "$(free_kb)" "$TOTAL" "$have" <<'PY' import sys tot, got, availkb, n, nhave = (int(x) for x in sys.argv[1:6]) g = 1 << 30 diff --git a/tools/pipeline.sh b/tools/pipeline.sh index c58889a..310766c 100755 --- a/tools/pipeline.sh +++ b/tools/pipeline.sh @@ -39,6 +39,28 @@ # be downloaded again — so it stays something the caller asks for. set -uo pipefail + +# python3, resolved by RUNNING a candidate rather than by looking one up. +# On Windows the name on PATH is usually the Microsoft Store App Execution +# Alias: a zero-byte reparse point that exists, exits 49, and prints an +# advert for the Store instead of running anything. Same finding as +# tests/run.sh, which shims PATH because it has 49 call sites and +# subprocesses to cover; the handful here get a name and pass it down. +# +# Windows installs the versionless `python`, and `py` besides, so a working +# interpreter is normally there under another name. If none answers, this +# stays `python3` and the script fails loudly at first use, which is right +# here: unlike the suite there is nothing to skip, and a run that cannot +# read its own index must stop rather than carry on. +# Exported rather than local, so the fetch_weights.sh this drives inherits +# the same answer instead of resolving it again per stage. +if [ -z "${PY:-}" ]; then + for _cand in python3 python py; do + "$_cand" -c '' >/dev/null 2>&1 && { PY="$_cand"; break; } + done + : "${PY:=python3}" +fi +export PY cd "$(dirname "$0")/.." SRC="${SRC:-/Volumes/WasteDisk/k3}" @@ -61,7 +83,7 @@ say "=== pipeline start ===" say "src $SRC -> out $OUT, $JOBS conversion processes" # ---- 1. download --------------------------------------------------------- -NEED=$(python3 -c " +NEED=$("$PY" -c " import json;print(len(set(json.load(open('$SRC/model.safetensors.index.json'))['weight_map'].values())))" 2>/dev/null || echo 0) [ "$NEED" -gt 0 ] || die "download (no index at $SRC)" @@ -83,7 +105,7 @@ say "stage 2: probe — ${FREE} GB free on the target volume" # The first MoE layer. K3 nests the text model, and the dense prefix has no # experts to round-trip, so neither is a detail this can guess at. -PROBE=$(python3 -c " +PROBE=$("$PY" -c " import json c = json.load(open('$SRC/config.json')) c = c.get('text_config', c) @@ -91,7 +113,7 @@ print(c.get('first_k_dense_replace', 0))" 2>/dev/null || echo "") [ -n "$PROBE" ] || die "probe (no config.json at $SRC)" probe_done() { - python3 -c " + "$PY" -c " import json, sys try: m = json.load(open('$OUT/manifest.json')) @@ -156,7 +178,7 @@ say " prompt ids: $IDS" $UV tools/kimi_ref.py --container "$OUT" --tokens 0 --prompt-ids "$IDS" \ --dump "$RUN/logits_ref.bin" >>"$LOG" 2>&1 || die "oracle" -python3 - "$RUN/logits_c.bin" "$RUN/logits_ref.bin" > "$RUN/diff.txt" <<'PY' +"$PY" - "$RUN/logits_c.bin" "$RUN/logits_ref.bin" > "$RUN/diff.txt" <<'PY' import struct, sys def L(p): b = open(p, "rb").read()