diff --git a/.github/build.sh b/.github/build.sh index 13b4f179b..a28b83fc8 100755 --- a/.github/build.sh +++ b/.github/build.sh @@ -105,18 +105,29 @@ if [ "${USE_CACHE:-true}" = "true" ] && command -v sccache >/dev/null 2>&1 \ && [ -n "${SCCACHE_WEBDAV_TOKEN:-}${SCCACHE_GHA_ENABLED:-}" ] \ && sccache_can_wrap_compiler; then LAUNCH="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" - # CUDA builds: also wrap nvcc so the per-arch .cu device passes are cached too — not just - # the gcc host TUs. Those per-architecture device-pass objects are the dominant cost of the - # full-arch CUDA job, and sccache does support nvcc as a compiler. Scoped to CUDA builds - # (GGML_CUDA in the cmake args): CMAKE_CUDA_COMPILER_LAUNCHER is inert when CUDA is not an - # enabled language, but keeping it scoped leaves the CPU/Android jobs' configure output clean. - # If sccache cannot wrap nvcc it runs it directly (uncached); and the mid-build retry below - # also catches an sccache "Compiler not supported" failure and rebuilds without the launcher, - # so an nvcc-hostile sccache can never red the build. + # CUDA builds: nvcc through sccache is OPT-IN (SCCACHE_WRAP_NVCC=true), off by default. + # Wrapping nvcc cached the per-arch .cu device passes too, which are the dominant cost of the + # full-arch CUDA job (measured: ~51 min cold -> ~15 min warm with CUDA 13.2). Since the CUDA + # 13.4 toolkit (2026-09-27), sccache 0.18.0 fails on the very first .cu TUs, ten seconds into + # the build, in every job log examined (four of four: runs 37643964068, 37680715063, + # 37831180985, 37925157175): `Missing "cubin" file output` / `Compiler killed by signal 126` + # -- sccache's nvcc command-line parsing, not a compile error. The retry below then rebuilt + # EVERYTHING without a launcher, gcc TUs included (47-80 min depending on the runner). Not + # wrapping nvcc keeps the ~280 gcc objects cached and runs nvcc directly from the start, with + # no wasted first attempt. Re-try the opt-in on a PR run after bumping SCCACHE_DL_VERSION (or + # a CUDA bump); flip the default back once a warm run shows CUDA hits in the stats table. + # Scoped to CUDA builds (GGML_CUDA in the cmake args): CMAKE_CUDA_COMPILER_LAUNCHER is inert + # when CUDA is not an enabled language, but keeping it scoped leaves the CPU/Android jobs' + # configure output clean. ggml does not wrap nvcc on its own here: its GGML_CCACHE block only + # fires when neither CMAKE_C_COMPILER_LAUNCHER nor CMAKE_CXX_COMPILER_LAUNCHER is set. case " $* " in *" -DGGML_CUDA=1 "* | *" -DGGML_CUDA=ON "* | *" -DGGML_CUDA=on "*) - LAUNCH="$LAUNCH -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache" - echo "build.sh: sccache will also wrap nvcc (CUDA build detected)" ;; + if [ "${SCCACHE_WRAP_NVCC:-false}" = "true" ]; then + LAUNCH="$LAUNCH -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache" + echo "build.sh: sccache will also wrap nvcc (CUDA build detected, SCCACHE_WRAP_NVCC=true)" + else + echo "build.sh: nvcc runs uncached (CUDA build detected, SCCACHE_WRAP_NVCC is not 'true'); sccache wraps the C/C++ TUs only" + fi ;; esac echo "build.sh: sccache ON (endpoint=${SCCACHE_WEBDAV_ENDPOINT:-default}), building with -j${JOBS}" else diff --git a/CHANGELOG.md b/CHANGELOG.md index d9d3ccde4..69147ddc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,24 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by core's; `check-natives.py` fails when they differ. ### Changed +- **`NativeServer` binds port 9931 when no `--port` is given** (llama.cpp b11521, + [#30159](https://github.com/ggml-org/llama.cpp/pull/30159) moved the server default off 8080), in classic and + attach mode alike; `NativeServer.getPort()` reports 9931 accordingly. `OpenAiCompatServer` keeps its own default + of 8080 (`OpenAiServerConfig.DEFAULT_PORT`, a public constant of this project with no upstream coupling). Pass + `--port` explicitly where a fixed port matters; every README example already does. +- **CI: the Linux CUDA build no longer routes nvcc through sccache by default** (`SCCACHE_WRAP_NVCC=true` + in `build.sh` opts back in). With CUDA 13.4 and sccache 0.18.0 the nvcc launcher failed on the first `.cu` + files in two confirmed runs (`Missing "cubin" file output`, `Compiler killed by signal 126`), and the + uncached retry that followed made those jobs slower than a cold build; the C/C++ objects stay cached. +- **Upgraded the pinned llama.cpp from b11512 to b11529**, in 5 reviewed steps, each ending at a tag. No + carried patch needed a refresh, and every drop-check still finds its defect. In the range: the server's + default port moves from 8080 to 9931 (#30159, the `NativeServer` entry above); CUDA top-k selection is + reworked (#28713) and the MMQ config helpers take the `src1` precision (#30168); SYCL gains a Q5_K + reorder-layout MMVQ and a fused GLU (#29375); Metal gets flash-attention kernels for the 128/96 head-size + pair (#30209); Vulkan's `rms_norm` no longer overflows its workgroup count (#30145); `ggml_acc` no longer + writes out of bounds for a large offset (#30135); the meta backend handles host views under + `--split-mode tensor` (#30217); the WebUI gains a models manager (#29583, auto-followed by `build-webui`). + Per-step record: `docs/history/llama-cpp-breaking-changes.md`. - **Windows natives: no Visual C++ redistributable, and `ucrtbase.dll` stays serviceable — but Windows 10 or newer is now required.** The Windows CPU jars (`cpu-windows-*`, `msvc-windows-*`) and the CUDA, Vulkan, OpenCL and ROCm jars use the **hybrid CRT**: the C++ diff --git a/CLAUDE.md b/CLAUDE.md index cca403231..1a814b312 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b11512** +Current llama.cpp pinned version: **b11529** ## Natives jars: one directory per backend (`.github/natives.csv`) @@ -368,13 +368,16 @@ and the Windows redist components at `https://developer.download.nvidia.com/comp The CUDA artifact must ship kernels for **every supported GPU generation**, so the default build — and every CI build — compiles the **full `CMAKE_CUDA_ARCHITECTURES` set** that ggml/llama.cpp selects. nvcc recompiles each `.cu` kernel once per architecture, which is the -dominant cost of the ~70 min CUDA job. **`sccache` now wraps nvcc too:** `build.sh` adds -`-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache` for CUDA builds (it detects `GGML_CUDA` in the cmake -args), so the per-arch `.cu` device passes are cached over Depot alongside the gcc C/C++ TUs. -Because the kernels are content-addressed and llama.cpp is pinned, a **warm** cache recompiles -only what changed — so CI keeps the **full arch set on every run** (release-safe everywhere) -and relies on the cache, not a reduced arch set, for speed. The first (cold-cache) run still -pays the full nvcc cost; the win shows on subsequent warm runs. +dominant cost of the CUDA job. **nvcc through `sccache` is opt-in since CUDA 13.4** +(`SCCACHE_WRAP_NVCC=true`: `build.sh` then adds `-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache` when it +detects `GGML_CUDA` in the cmake args) and **off by default** -- the rollout item 2 below has the +measurements. With CUDA 13.2 it cached the per-arch `.cu` device passes over Depot alongside the +gcc C/C++ TUs (~51 min cold, ~15 min warm); with CUDA 13.4 and sccache 0.18.0 it fails on the first +`.cu` TUs of every run examined (`Missing "cubin" file output`, `Compiler killed by signal 126`), and +the retry that follows rebuilds everything without any launcher, gcc TUs included (47-80 min +depending on the runner). Today the gcc TUs are cached and nvcc runs directly: CI keeps the **full +arch set on every run** (release-safe everywhere) at roughly the cold nvcc cost, never a reduced +arch set. `CUDA_FAST_BUILD` remains as a **local-dev** single-arch knob (CI no longer sets it). `build_cuda_linux.sh` honors it — default **off** (full arch set, release-safe): @@ -395,15 +398,14 @@ runs on only the single GPU generation it was compiled for, so the **distributed the full arch set**. The script default is **off** (full) so any *local/manual* build is release-safe, and **CI no longer sets `CUDA_FAST_BUILD` at all** — the `crosscompile-linux-x86_64-cuda` job always builds the full set on PR / push / dispatch / publish, so every artifact (not just the ones -that reach Central) runs on every GPU generation. The full-arch CI cost is absorbed by the -sccache-over-Depot cache, which now wraps nvcc (`-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache`, added by -`build.sh` for CUDA builds, gated behind the same probe). The launcher is safe to enable -unconditionally: if sccache cannot wrap nvcc it runs it directly (uncached), and `build.sh`'s -mid-build retry treats an sccache `Compiler not supported` failure like any other cache error and -rebuilds the job without the launcher rather than redding it. **Verified:** a warm run in the -manylinux_2_28 container hit **100%** on CUDA / CUBIN / device-code (139 CUDA hits, 99.86% overall, -3 misses) and cut the job from **~51 min cold to ~15 min warm** — nvcc caching works here. `build.sh` -prints `sccache --show-stats` at the end of every run so the hit table stays visible. +that reach Central) runs on every GPU generation. The full-arch CI cost is paid by the job +itself since CUDA 13.4 (nvcc uncached, see above); the sccache-over-Depot cache still covers the gcc +TUs, and `SCCACHE_WRAP_NVCC=true` re-enables the nvcc launcher (gated behind the same probe; +`build.sh`'s mid-build retry treats an sccache `Compiler not supported` / `Compiler killed by signal` +failure like any other cache error and rebuilds the job without any launcher rather than redding it). +**Measured with CUDA 13.2:** a warm run in the manylinux_2_28 container hit **100%** on CUDA / CUBIN / +device-code (139 CUDA hits, 99.86% overall, 3 misses) and cut the job from **~51 min cold to ~15 min +warm**. `build.sh` prints `sccache --show-stats` at the end of every run so the hit table stays visible. ## Android minimum API level @@ -895,7 +897,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network for the asset build; the embed step is plain cmake -P -git clone --depth 1 --branch b11512 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b11529 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build ) mkdir -p webui-generated /tmp/ui-gen cmake -DUI_SOURCE_DIR=/tmp/lc/tools/ui -DUI_BINARY_DIR=/tmp/ui-gen \ @@ -935,7 +937,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b11512`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b11529`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -985,9 +987,10 @@ v0.16.0 + the probe this is no longer a risk.) Job-by-job status: warm cache 277/278 hits (99.64%), 1m46s build time. Since the CPU variants it runs in the manylinux_2_28 image the CUDA job already used (gcc-toolset-14), with the same env. 2. `crosscompile-linux-x86_64-cuda` (via `build_cuda_linux.sh`, which execs `build.sh`) — - ✅ **verified green with nvcc caching, full-arch always.** `build.sh` also wraps nvcc + ✅ **green, full-arch always; the gcc TUs cache, nvcc has been uncached since CUDA 13.4** (the + paragraph at the end of this item). With CUDA 13.2 `build.sh` also wrapped nvcc (`-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache`, scoped to CUDA builds), so both the gcc C/C++ TUs - (134 model files + ggml + httplib) **and** the per-arch `.cu` device passes cache over Depot. + (134 model files + ggml + httplib) **and** the per-arch `.cu` device passes cached over Depot. CI dropped the single-arch validation shortcut (`CUDA_FAST_BUILD`/`CUDA_ARCH` removed from the job) — every run builds the full arch set and leans on the warm cache for speed. A warm run hit **100%** on CUDA / CUBIN / device-code (139 CUDA hits, 99.86% overall, 3 misses), cutting the job @@ -1011,7 +1014,17 @@ v0.16.0 + the probe this is no longer a risk.) Job-by-job status: retry silently re-enabled the very launcher it was trying to avoid. Fix: the retry's `cmake -Bbuild` now also passes `-DGGML_CCACHE=OFF`, so a genuinely uncached build is guaranteed regardless of what's left on `PATH`. This fallback now falls back to a real, green `-O3` build - like every other sccache/nvcc incompatibility instead of redding the job. + like every other sccache/nvcc incompatibility instead of redding the job. **CUDA 13.4 with sccache 0.18.0: a third + failure shape, and the reason nvcc wrapping is opt-in now (`SCCACHE_WRAP_NVCC=true`).** + `sccache: caused by: Missing "cubin" file output` followed by `sccache: Compiler killed by signal + 126`, on the first `.cu` TUs ten seconds into the build, in **every** CUDA job log examined since + the 13.4 bump of 2026-09-27 -- runs 37643964068 and 37831180985 (2026-10-07/08), 37680715063 (the + b11476 bump) and 37925157175 (#492, 2026-10-09). Each fell back to the uncached retry, which took + 47 to 80 minutes depending on the runner (job totals of 54 to 93 min; none came near the warm + ~15 min), so the launcher never paid anything back and the retry threw the gcc cache away with it. + `build.sh` therefore wraps nvcc only when `SCCACHE_WRAP_NVCC=true` (the CUDA job does not set it) + and keeps the gcc TUs cached. To re-test: bump `SCCACHE_DL_VERSION`, set the variable on the CUDA + job for one PR run, and flip the default back once a warm run shows CUDA hits in the stats table. 3. `crosscompile-linux-aarch64` — ✅ **enabled**, a **native `ubuntu-24.04-arm` build** inside the pypa manylinux_2_28 aarch64 image (`MANYLINUX_ARGS` forwards the env like `DOCKCROSS_ARGS`): `build.sh` self-fetches the aarch64 static-musl sccache in the container (the fetch block in @@ -1156,7 +1169,7 @@ Current patches: | `0012-model-guard-zero-split-sum-and-name-the-device-index.patch` | **A GPU that reports zero free memory makes every model load fail with the unactionable `error loading model: vector`.** `llama_model_base::load_tensors` (`src/llama-model.cpp`) weights the per-device layer split by `ggml_backend_dev_memory()`'s `free`, then normalises: `splits[i] /= split_sum`. With a single device reporting `free == 0` that is `0/0` → **NaN** in every split point; NaN compares false against everything, so the `std::upper_bound` below returns the end iterator, `layer_gpu == n_devices()`, and `devices.at(layer_gpu)` throws `std::out_of_range` — whose libc++ `what()` is the bare string `"vector"`, which `llama.cpp`'s `catch (const std::exception &)` prints verbatim. Upstream's `free == 0 && total == 0` host-memory fallback does **not** fire, because `total` is `recommendedMaxWorkingSetSize` and is non-zero. **Reachable since b10618..b10797**: upstream `8c0b9cd04` ("metal : fix memory query under low-memory conditions", [#27701](https://github.com/ggml-org/llama.cpp/pull/27701)) changed `ggml-metal-device.m` to `*free = *total > cur ? *total - cur : 0`; before that clamp an over-committed device (`currentAllocatedSize > recommendedMaxWorkingSetSize`) *underflowed* to a huge `size_t`, which normalised fine, so the same precondition was harmless. That is why the `Java Tests macOS …` jobs went red at the b10792→b10797 step while every Linux/Windows job stayed green — **and why only a GPU build can fail this way at all**: `act_gpu_layers` is `devices.empty() ? 0 : …`, so with no GPU backend `devices` is empty, every layer returns early on `cpu_dev`, and the `.at()` line is unreachable. **Shape:** the two blocks are lifted out of `load_tensors` into free functions declared in `src/llama-model.h`, purely so they can be driven by a test — the failing state needs a real over-committed GPU and cannot be arranged through any public API. `llama_model_splits_normalize()` carries **the fix**: on `split_sum == 0` it `LLAMA_LOG_WARN`s and falls back to an even split (`splits[i] = float(i+1)/splits.size()`), the only neutral choice when no device can be preferred and exactly right for a single device. `llama_model_splits_select_device()` carries **the diagnostic**: it bounds-checks the index and throws a `std::runtime_error` naming the function, the offloaded layer, the device index, the split-point count **and the split points themselves** — with NaN splits that message prints `nan` and names the cause outright, which is precisely what was missing when this had to be diagnosed by reading source. **A second, backend-independent trigger reaches the same line**, found while writing this up and verified against the unfixed library: `--tensor-split` values are parsed with `std::stof` and never range-checked (`common/arg.cpp`), so `-ts 1,-1` cancels out, `split_sum` is 0 again, the split points become `[inf, -nan]`, and every layer maps one past the last device — on CUDA, Vulkan or ROCm just as much as on Metal, with no memory pressure involved. That is what makes this an ordinary upstream defect rather than a Metal edge case, and the warning names both causes rather than only the memory one. Also adds upstream `tests/test-model-split.cpp` (5 cases in upstream's `testing.h` style) + its `llama_build_and_test` registration. Touches `src/llama-model.{cpp,h}`, `tests/test-model-split.cpp` and `tests/CMakeLists.txt` — **none** of which any other patch touches, so it is independent of all of them. Upstream-submittable ("model: fall back to an even split when no device reports free memory"); **not yet filed upstream**. **Runnable guard: `src/test/cpp/test_model_split.cpp`** — a FetchContent subproject builds with `LLAMA_BUILD_TESTS=OFF`, so the upstream test above is applied-but-never-compiled here (same as `0001`'s test). That file drives the same two functions from `jllama_test`, which runs on **every** platform in `C++ Tests`, so a bump that drops this patch fails the build at link time everywhere instead of surfacing as one red macOS Java job. **Verification limit — read before assuming this can be dropped:** the *failing path* still cannot be reached without a GPU backend, so the guard pins the arithmetic (what actually broke), not the end-to-end load; the end-to-end proof is the macOS CI job. On a bump, re-check whether upstream added its own `split_sum == 0` guard (grep `split_sum` in `src/llama-model.cpp`) and **drop this patch rather than refreshing it** if they did — the fail-loud applier detects "does not apply", never "upstream already fixed this". | | `0014-common-log-callback-sink.patch` | **Gives `common_log` a callback sink: `common_log_set_callback(log, cb, user_data)` (`common/log.{h,cpp}`).** This is what `LlamaModel.setLogger` hooks. Before it, the Java logger was a `llama_log_set()` callback, which has two holes, both found while chasing `slot print_timing` lines interleaving with the Atmosphere agent's streamed answer: **(a)** every model load runs `common_init()`, which re-points `llama_log_set()` at `common_log_default_callback` (`common.cpp:394`), so `setLogger(…)` *before* `new LlamaModel(…)` silently lost the callback; **(b)** the server's own `SRV_*`/`SLT_*` macros are `LOG_INF` and write straight into `common_log`, which `llama_log_set()` never carried, so the per-request `slot …`/`srv …` lines could not be routed to Java at all (the reason `LlamaModelTest#testLogText/JSON` sat `@Disabled` for years). `common_log` upstream offers file, colors, prefix, timestamps, verbosity and JSONL but no hook. The patch adds one: while a callback is set, the worker thread hands every entry to it **instead of** printing to stdout/stderr (a `--log-file` still receives them); the callback gets the bare formatted message (no prefix/timestamp/colors) with the `ggml_log_callback` signature; swapping the callback pauses the worker first, so queued entries reach the *previous* sink (which is what makes `setLogger(format, null)` a synchronous drain). With the sink, `common_init()`'s `llama_log_set()` reset is harmless — it points at the default callback that feeds `common_log`, i.e. exactly the path into the sink — so the ordering problem (a) disappears without any re-install logic, and the `srv`/`slot` lines (b) arrive because they are `common_log` entries. `jllama.cpp`'s `setLogger` therefore sets `common_log_set_callback(common_log_main(), trampoline)` **plus** `llama_log_set(common_log_default_callback)` (so llama/ggml lines feed `common_log` even before the first load). Two consequences to know: the callback runs on `common_log`'s **worker thread**, a plain `std::thread` llama.cpp re-creates on every pause/resume and never attaches to the JVM — the trampoline attaches per call and detaches again (`get_jni_env_attaching`; a thread that exits while attached leaks a `JavaThread`, and this thread is not ours — the leak-free choice, not the cheapest: each attach creates a `java.lang.Thread` object, so a `thread_local` guard that detaches once at thread exit is the optimisation on file in `TODO.md`), `setLogger` must call `common_log_set_callback` **outside** `g_log_mutex`, because the pause joins the worker, which needs that mutex to read the callback, and `setLogger` callers are serialized by a **separate** `g_set_logger_mutex`: two unserialized swaps race on the worker's `std::thread` (one joins it while the other assigns a fresh thread over the still-joinable object = `std::terminate`, the whole JVM), which `LlamaLoggerTest#concurrentSetLoggerCallsDoNotRaceOnTheLogWorker` reproduced before the mutex existed. Two caveats the Javadoc carries: a caller must not hold a lock the *previous* callback needs (the drain runs it on the worker while the caller waits), and the verbosity threshold is process-wide and reset by **every** load (`common_params_parse` ends with `common_log_set_verbosity_thold(params.verbosity)`, default 3), so a load without `-lv` puts it back to 3 — a review assumed the opposite, and `LlamaLoggerTest#verbosityThresholdIsProcessWideAndEveryLoadSetsIt` now pins the measured behaviour. And the verbosity threshold applies *before* the sink: at the default (`3`) the Java logger sees errors, warnings and the server's INFO lines, while llama/ggml INFO lines (`common_log_get_verbosity` maps them to TRACE = 4) arrive only from `setLogVerbosity(4)` on — the same filtering the console gets, and a behaviour change for consumers who captured the unfiltered `llama_log_set()` stream before. **Runnable guards:** `src/test/cpp/test_common_log_callback.cpp` (6 tests over a private `common_log_init()` instance: delivery, bare text under prefix+timestamps, clear, swap-drains-to-old-sink, file kept, levels pass through) links the function on every platform, so a bump that drops the patch reds `C++ Tests` at link time; `LlamaLoggerTest` (model-free, needs only `libjllama`: a logger set before a deliberately failing load on a non-GGUF file sees the `srv … loading model` INFO line and llama's ERROR line, in TEXT and JSON) and the re-enabled `LlamaModelTest#testLogText/testLogJSON` plus `#testLoggerSetBeforeLoadSurvivesTheLoad` (vocab-only load) cover the Java side. Upstream-submittable ("common : add a callback sink to common_log for embedding hosts"); **not yet filed upstream**. Touches only `common/log.{h,cpp}`, which no other patch touches. **Refreshed at the b11401 bump** (#29895 adds a `colors` member next to the sink's members and a `get_colors()` next to `set_callback()`; only context moved, the `+`/`-` lines are unchanged, and upstream still has no hook -- it added `common_log_get_colors` only). **On a bump, check whether upstream added a hook of its own (grep `callback` in `common/log.h`) and, if so, DROP this patch and port `setLogger` to theirs rather than refreshing it.** | | `0015-rpc-embeddable-client-and-stoppable-server.patch` | **Makes ggml-rpc usable inside a JVM** (see "RPC backend" below). Upstream treats every client-side RPC problem as `GGML_ABORT` — a malformed endpoint, a server that is not running, a failed handshake — which in a JVM kills the application over a typo in `--rpc`. **(1)** The *registration* path reports failure instead: `rpc_dispatcher::start()` returns `false`, `try_get_dispatcher()` `nullptr`, `ggml_backend_rpc_get_device_count()` `0`, `ggml_backend_rpc_add_server()` `nullptr`, and `add_rpc_devices()` (`common/arg.cpp`) throws `std::invalid_argument` naming the server where it used to register `nullptr` and silently run without it. Every path *after* registration keeps the original contract through `get_dispatcher()`, which still aborts — a server that vanishes mid-inference stays fatal (TODO). **(2)** `add_server()`'s cache hit is re-checked (a stopped server used to pass registration and abort on the first tensor upload), and `ggml_backend_rpc_get_device_memory()` of a gone server returns 0/0 instead of aborting — RPC devices stay in ggml's process-wide registry forever, and `common_init()` queries the memory of **every** registered device at `-lv 4`, so a stopped server would otherwise take down an unrelated later load. **(3)** The server becomes stoppable: `ggml_backend_rpc_stop_server()` (disconnects the client being served, wakes `accept()` with a loopback connection — the one portable way) and `ggml_backend_rpc_server_listening()`; the loop now reaches its cleanup and frees its backends on every early return, but deliberately **not** `rpc_transport_shutdown()`, whose `WSACleanup()` would invalidate every other RPC socket in the process. **(4)** transport: `MSG_NOSIGNAL`/`SO_NOSIGPIPE`, `socket_t::shutdown()`, and the fds `connect()`/`create_server()`/`accept()` leaked on their error paths are closed. Touches `ggml/include/ggml-rpc.h`, `ggml/src/ggml-rpc/{ggml-rpc.cpp,transport.cpp,transport.h}` and one function in `common/arg.cpp` (`add_rpc_devices`, away from `0001`'s `common_params_parse` hunks). **Runnable guard: `src/test/cpp/test_rpc.cpp`** — links both new functions (a bump that drops the patch fails `C++ Tests` at link time on every platform) and drives registration, stop and the memory query over loopback. Upstream-submittable; **not yet filed upstream**. On a bump, check whether upstream grew its own stop function or non-aborting registration (grep `stop_server` / `GGML_ABORT("Failed to connect` in `ggml-rpc.cpp`). **Refreshed at the b11450 bump** (#26610, `-sm tensor` over RPC, ~800 lines in `ggml-rpc.cpp`, protocol 7 → 8): three hunks lost their context -- new dispatcher members beside `start()`, and new `get_proc_address` names before `return NULL` (the two new names now sit after `ggml_backend_comm_allreduce_tensor`); the `+`/`-` lines are unchanged. The server-to-server comm #26610 adds (a listener on `0.0.0.0`, an `accept()` a stop does not wake) is on file in `TODO.md`. | -| `0017-ggml-cpu-x86-prefetch-byte-offset-and-msvc-type.patch` | **Makes `GGML_CPU_ALL_VARIANTS=ON` buildable with plain clang on Windows, and fixes a prefetch address that is wrong everywhere.** The four `_mm_prefetch` calls in `ggml/src/ggml-cpu/arch/x86/quants.c` (the SSSE3-without-AVX branch of `ggml_vec_dot_q4_0_q8_0`, and the only `_mm_prefetch` calls in the whole ggml tree) carry two independent defects; both are fixed by routing the address through one new `static inline` helper in the new header `ggml/src/ggml-cpu/arch/x86/prefetch.h`, which is included relatively, so **no CMake change is needed**. **(1) Type -- fatal only on a current clang:** MSVC declares the intrinsic as `void _mm_prefetch(char const *, int)`; clang's own `xmmintrin.h` defines a casting macro but only `#ifndef _MSC_VER`, so on a `*-windows-msvc` target clang leaves the declaration to Microsoft and a typed pointer is `-Wincompatible-pointer-types`. **clang 22 promoted that to an error by default** ([llvm-project #157364](https://github.com/llvm/llvm-project/pull/157364); the [22.1.0 release notes](https://releases.llvm.org/22.1.0/tools/clang/docs/ReleaseNotes.html) add that `-Wno-error=incompatible-pointer-types` downgrades it) -- *not* clang 16, which promoted only `-Wincompatible-FUNCTION-pointer-types`; an earlier version of this row said otherwise. Measured on the real file at `-msse4.2`, unpatched → patched: clang **20.1.8** 4 warnings → 0/0, clang **22.1.3** 4 errors → 0/0, clang **23.1.3** 4 errors → 0/0. Plain clang is exactly what `cmake/x64-windows-llvm.cmake` selects -- and plain clang is the only way to get all **14** CPU variants, because `clang-cl` makes CMake's `MSVC` true and the `if (NOT MSVC)` block in `ggml/src/CMakeLists.txt` then drops `zen4`, `cooperlake`, `sapphirerapids`, `ivybridge` and `piledriver`. **(2) Arithmetic:** `&x[ib] + sizeof(block_q4_0)` adds `sizeof()` *elements*, not bytes -- +324 and +1156 instead of +18 and +34 (the sizes `ggml-common.h` pins by `static_assert`), and +648 instead of +36 for the `2 *` pair. That half is wrong on every platform, including where the call compiles. **Measured:** no throughput difference (see the test-table row), so this is a correctness and portability fix, not a performance one. **Upstream-submittable and meant to be submitted.** **Why upstream's own build is green is measured, not an open question:** their `windows-cpu` release job builds exactly this with plain clang, `GGML_CPU_ALL_VARIANTS=ON` and no suppressing flag (`CMAKE_ARGS` carries only build-target toggles), and ships `ggml-cpu-sse42.dll` -- because the `windows-2025-vs2026` image has **LLVM 20.1.8** on `PATH`, where these are warnings. That is narrower than it looks: the same image also ships **clang 22.1.3** inside Visual Studio 2026 (`VC/Tools/Llvm/x64`), which *fails* on the unpatched file -- upstream reaches the working compiler only because `C:\Program Files/LLVM/bin` comes first on `PATH`. It is **not** a header difference: all three MSVC toolsets on that image (14.29.30133, 14.44.35207, 14.51.36231) declare it identically as `extern void _mm_prefetch(char const*_A, int _Sel);` -- that was the first hypothesis and `.github/workflows/probe-msvc-prefetch.yml` disproved it. So this patch is **ahead of** upstream rather than diverging: their build breaks the moment the image ships a clang that makes the diagnostic fatal, and until then they compile four warnings and a prefetch address 18x/34x too far. **Runnable guard: `src/test/cpp/test_prefetch.cpp`.** On a bump, run `git grep -n _mm_prefetch ggml/src/ggml-cpu/arch/x86/quants.c` at the new tag: if the call sites already cast **before** the addition, **drop this patch rather than refreshing it**. | +| `0017-ggml-cpu-x86-prefetch-byte-offset-and-msvc-type.patch` | **Makes `GGML_CPU_ALL_VARIANTS=ON` buildable with plain clang on Windows, and fixes a prefetch address that is wrong everywhere.** The four `_mm_prefetch` calls in `ggml/src/ggml-cpu/arch/x86/quants.c` (the SSSE3-without-AVX branch of `ggml_vec_dot_q4_0_q8_0`, and the only `_mm_prefetch` calls in the whole ggml tree) carry two independent defects; both are fixed by routing the address through one new `static inline` helper in the new header `ggml/src/ggml-cpu/arch/x86/prefetch.h`, which is included relatively, so **no CMake change is needed**. **(1) Type -- fatal only on a current clang:** MSVC declares the intrinsic as `void _mm_prefetch(char const *, int)`; clang's own `xmmintrin.h` defines a casting macro but only `#ifndef _MSC_VER`, so on a `*-windows-msvc` target clang leaves the declaration to Microsoft and a typed pointer is `-Wincompatible-pointer-types`. **clang 22 promoted that to an error by default** ([llvm-project #157364](https://github.com/llvm/llvm-project/pull/157364); the [22.1.0 release notes](https://releases.llvm.org/22.1.0/tools/clang/docs/ReleaseNotes.html) add that `-Wno-error=incompatible-pointer-types` downgrades it) -- *not* clang 16, which promoted only `-Wincompatible-FUNCTION-pointer-types`; an earlier version of this row said otherwise. Measured on the real file at `-msse4.2`, unpatched → patched: clang **20.1.8** 4 warnings → 0/0, clang **22.1.3** 4 errors → 0/0, clang **23.1.3** 4 errors → 0/0. Plain clang is exactly what `cmake/x64-windows-llvm.cmake` selects -- and plain clang is the only way to get all **14** CPU variants, because `clang-cl` makes CMake's `MSVC` true and the `if (NOT MSVC)` block in `ggml/src/CMakeLists.txt` then drops `zen4`, `cooperlake`, `sapphirerapids`, `ivybridge` and `piledriver`. **(2) Arithmetic:** `&x[ib] + sizeof(block_q4_0)` adds `sizeof()` *elements*, not bytes -- +324 and +1156 instead of +18 and +34 (the sizes `ggml-common.h` pins by `static_assert`), and +648 instead of +36 for the `2 *` pair. That half is wrong on every platform, including where the call compiles. **Measured:** no throughput difference (see the test-table row), so this is a correctness and portability fix, not a performance one. **Upstream-submittable and meant to be submitted.** **Why upstream's own build is green is measured, not an open question:** their `windows-cpu` release job builds exactly this with plain clang, `GGML_CPU_ALL_VARIANTS=ON` and no suppressing flag (`CMAKE_ARGS` carries only build-target toggles), and ships `ggml-cpu-sse42.dll` -- because the `windows-2025-vs2026` image has **LLVM 20.1.8** on `PATH`, where these are warnings. That is narrower than it looks: the same image also ships **clang 22.1.3** inside Visual Studio 2026 (`VC/Tools/Llvm/x64`), which *fails* on the unpatched file -- upstream reaches the working compiler only because `C:\Program Files/LLVM/bin` comes first on `PATH`. It is **not** a header difference: all three MSVC toolsets on that image (14.29.30133, 14.44.35207, 14.51.36231) declare it identically as `extern void _mm_prefetch(char const*_A, int _Sel);` -- that was the first hypothesis, and a one-off probe workflow (deleted again once it had answered) disproved it by printing the declaration from each toolset on the image. So this patch is **ahead of** upstream rather than diverging: their build breaks the moment the image ships a clang that makes the diagnostic fatal, and until then they compile four warnings and a prefetch address 18x/34x too far. **Runnable guard: `src/test/cpp/test_prefetch.cpp`.** On a bump, run `git grep -n _mm_prefetch ggml/src/ggml-cpu/arch/x86/quants.c` at the new tag: if the call sites already cast **before** the addition, **drop this patch rather than refreshing it**. | | `0016-model-kolibri1.patch` | **Adds Aleph Alpha's Kolibri-1 (architecture `kolibri1`) ahead of upstream** -- a 78B German/English reasoning MoE (3.46B active) that upstream does not support yet ([ggml-org/llama.cpp#29922](https://github.com/ggml-org/llama.cpp/issues/29922), no PR at the time). A **temporary carry, not a fix**: drop it the moment upstream registers the architecture (`git grep -n kolibri src/llama-arch.cpp` at the new tag), and keep `test_kolibri1.cpp` -- its numerical comparisons must then pass against upstream's implementation, while its GGUF-format rows follow whatever format upstream's converter fixes (see the test-table row). **Architecture** (Aleph Alpha's vLLM reference, `aleph_alpha_inference/kolibri1.py` at `049a6a7`): GQA with per-head q/k RMSNorm; `layer_types` interleaves sliding-window layers (NEOX RoPE) with full-attention layers that have **no** positional encoding; sandwich norms around attention and MoE; every layer routed MoE + one ungated shared expert; the router selects top-k on **`logits + expert_bias`** and weights by the **unbiased `sigmoid(logits)`** -- not DeepSeek-V3's sigmoid router (selection on `sigmoid(logits) + bias`), which picks other experts as soon as the bias is non-zero. The graph therefore builds the selection itself and passes it to `build_moe_ffn` (`selected_experts_in`), leaving upstream's shared MoE code untouched. **Derived from two community ports, which are incompatible with each other's GGUFs** -- Eliasfpv28's AFMoE-based one (base b11378, `expert_gating_func` 2, pre-tokenizer `qwen2`) and the Qwen3-MoE-based one behind Hob-forge's GGUFs (base b11381, as carried in [mjwsolo/localcode#101](https://github.com/mjwsolo/localcode/pull/101): gating 5, pre-tokenizer `kolibri1`, rejects gating 2). This patch **loads both dialects**: gating 2, 5 or absent (all mean the one router), `kolibri1` mapped to the Qwen2 pre-tokenizer in `llama-vocab.cpp`, `expert_weights_norm` / `expert_shared_count` / `output.weight` optional. Every source, with license, is listed in the patch header; `REUSE.toml` annotates the file `MIT AND Apache-2.0` (Eliasfpv28's additions are Apache-2.0). Touches `src/llama-arch.{h,cpp}`, `src/llama-model.cpp` (mapping + NEOX rope list), `src/llama-vocab.cpp` (one name), `src/models/models.h` and the new `src/models/kolibri1.cpp`; no converter. **Runnable guard: `src/test/cpp/test_kolibri1.cpp`** (writes tiny random GGUFs in both dialects and compares every logit with an independent double-precision reference). **Not verified here:** the real 78B model (HuggingFace is unreachable from the sessions this was written in, and the smallest GGUF is 28.6 GB) and GPU backends -- the community ports report real-model checks (top-1 67/67 against a float64 reference for Q8_0, tool calls, German text). | **Dropped patches** (`0004`, `0005`, `0009`, `0010`, `0011`, `0013` -- upstream fixed each defect @@ -2085,7 +2098,7 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b11512`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b11529`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.18.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the diff --git a/README.md b/README.md index 2f4b08f6a..f49bf3039 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b11512](https://img.shields.io/badge/llama.cpp-%23b11512-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b11512) +[![llama.cpp b11529](https://img.shields.io/badge/llama.cpp-%23b11529-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b11529) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) diff --git a/TODO.md b/TODO.md index a196f3494..83c553bbf 100644 --- a/TODO.md +++ b/TODO.md @@ -166,14 +166,22 @@ binaries; 2026-10-08) established: 7. **Unsigned binaries.** Upstream ships its DLLs unsigned, and so do we; on a WDAC/AppLocker-managed client, loading unsigned DLLs from `%TEMP%` is blocked. Out of scope here; worth a README note. -### CUDA job: nvcc through sccache failed once and fell back - -- Run 37680715063 (b11476 bump): the Linux CUDA build logged `Missing cubin file output` and - `sccache: Compiler killed by signal 126` on `.cu` TUs, and `build.sh`'s retry rebuilt without the - launcher (green, ~50 min instead of ~15). Not the CUDA 13.3 `-virtual` failure already handled - (that one was `fatbinary ... acc.compute_75.ptx`). Check whether sccache 0.18.0 and CUDA 13.4 disagree - on the device-compile pipeline, or whether it was a transient cache-storage error; a second - occurrence makes the CUDA cache effectively off. +### CUDA job: nvcc through sccache (`SCCACHE_WRAP_NVCC`, off since the second failure) + +- Run 37680715063 (b11476 bump) and run 37925157175 (#492, 2026-10-09): the Linux CUDA build logged + `sccache: caused by: Missing "cubin" file output` and `sccache: Compiler killed by signal 126` on + the first three `.cu` TUs, one minute into the build, and `build.sh`'s retry rebuilt everything + without any launcher (green; 86 min in total the second time, longer than a cold build). Not the + CUDA 13.3 `-virtual` `.ptx` failure, and not a storage error (the gcc probe passed and the gcc TUs + were being served): sccache 0.18.0 and CUDA 13.4's nvcc disagree on the device-compile command + line, in every job log examined since the 13.4 bump of 2026-09-27 (also runs 37643964068 and + 37831180985; the retry alone takes 47-80 min depending on the runner, and no job came near the warm + ~15 min). Since the second confirmed occurrence `build.sh` + wraps nvcc only with `SCCACHE_WRAP_NVCC=true` (CLAUDE.md "Fast local CUDA builds"), so the gcc TUs + stay cached and nvcc runs directly from the start. **Open:** re-test the opt-in with a newer + sccache (bump `SCCACHE_DL_VERSION` on a branch, set the variable on the CUDA job for one PR run, + look for CUDA hits in the stats table) and flip the default back when a warm run shows them -- the + win was ~51 -> ~15 min with CUDA 13.2. ### macOS dylib links Homebrew OpenSSL (found by `verify-native-deps.py`) diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 8f9646489..b40596f3f 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -850,3 +850,13 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b11495–b11505 | patches + upstream verification | **All ten patches apply unchanged**, verified in order against pristine b11505 with `git apply`. No patch-target file in the range; nothing under `src/`, `common/`, `tools/server/` or `include/` changed, so every drop-check of the b11480 row holds unchanged. | | b11505–b11512 | Seven commits, 25 files, ~716/202 lines (79 KiB, the final step). **#26004 keeps the context checkpoints across a slot save and restore**: the slot file gains an appendix after the llama state payload (magic `SCKP`, version 1, a count, then per checkpoint `n_tokens`/`pos_min`/`pos_max` and the target, draft and speculative blobs), counted in the `n_written` / `n_read` of the save and restore results; on restore it is read bounds-checked -- a missing, truncated or foreign appendix is ignored with a warning, never fatal, so a slot file written before this restores as before -- and one draft checkpoint is test-loaded, the draft data dropped when it does not fit the current draft context, where `common_prompt_checkpoint::load_dft()` used to `GGML_ABORT` on a size mismatch (`load_tgt`/`load_dft` return `bool` now, `common/common.h`). `LlamaModel.handleSlotAction(SAVE/RESTORE)` carries the same payload; the response keys are unchanged. **#30112 splits the MoE expert cache across several GPUs** like the layers (`--tensor-split`; the `--moe-cache-mib` help text, `common_params`, `llama-moe-cache.cpp`) -- `setMoeCacheMib`'s Javadoc already says so. **CUDA:** MMQ out-of-bounds reads (#29953), a looped PAD kernel beyond 65535 rows or slices (#30147), and the CCCL version guard in `argsort.cu` compares the packed `CCCL_VERSION` (#29453: the per-component compare turned `STRIDED_ITERATOR_AVAILABLE` off on a major rollover -- with the pinned CCCL v3.4.3 it is true before and after, so no change for the CUDA jars until CCCL 4). #30111 reads DFlash's tied output weights from the GGUF metadata; #29668 WebUI settings on the first visit in router mode (auto-followed). **No project source change.** | | b11505–b11512 | patches + upstream verification | **All ten patches apply unchanged**, verified in order against pristine b11512 with `git apply` and by a fresh configure through the real `FetchContent` path. Patch-target files in the range: `common/arg.cpp` (`0001`, `0015`: a help-text change only) and `tools/server/server-context.cpp` (`0002`, `0003`: the checkpoint appendix helpers are private members added away from `load_model` and the similarity getter/setter). Drop-checks against pristine b11512 -- the final state of the bump: `common_params_parse` still carries the `#ifdef _WIN32` `argv = utf8.ptrs.data()` override and `common_params_parse_main` is absent (`0001`); the load-progress assignment is unconditional (`0002`); no `slot_prompt_similarity` getter (`0003`); no embedded-mode, attach or worker-command counterpart (`0006`-`0008`); no `split_sum == 0` guard (`0012`); `common/log.h` names only `common_log_default_callback`, no sink (`0014`); no `stop_server` and the connect failure still aborts in `ggml-rpc.cpp` (`0015`); no `kolibri` under `src/` (`0016`). No `.github/` change upstream in the whole range. | +| b11512–b11516 | Four commits, 11 files, ~295/167 lines (40 KiB, one step), backends only. **CUDA: top-k algorithm selection reworked** (#28713): a grid-over-rows radix select replaces CUB's per-row `DeviceTopKKernel` once a batch has `GGML_CUDA_TOPK_RADIX_MIN_ROWS` rows, bitonic stays for short rows that fit one wave of blocks, DeviceTopK or the CUB argsort serves a single long row, and radix select works in chunks so its scratch memory stays bounded -- upstream measured top-k on a 34,816-token MoE prompt from 5,762 ms to 942 ms; the thresholds stay build-time overridable, HIP and MUSA keep theirs. **SYCL: Q5_K reorder-layout MMVQ and a fused GLU** (#29375; the two Linux SYCL jars and the Windows one). A MUSA FWHT fix (#30167) and a Hexagon IM2COL DMA-ring overflow (#30189), neither built here. **No project source change.** | +| b11512–b11516 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11516 with `git apply`. No patch-target file in the range; nothing under `src/`, `common/`, `tools/server/`, `include/` or `ggml/src/ggml-cpu/` changed, so every drop-check of the b11505–b11512 row holds unchanged -- and `patches/0017`'s own check, the four uncast `_mm_prefetch` calls in `ggml/src/ggml-cpu/arch/x86/quants.c` (lines 782/783 and 800/801), still finds them uncast. | +| b11516–b11517 | One commit, 4 files, ~223/225 lines (81 KiB, one step). **CUDA MMQ: the host-side config helpers take the `src1` precision** (#30168; `mmq.cu`, `mmq.cuh`, `mmq-load-tiles.cuh`, `mmq-vec-dot.cuh` -- a signature change threaded through the tile loaders and vector dot products, which is why a one-line idea weighs 81 KiB). Inside the CUDA backend, which `jllama` builds as a whole from upstream's sources; nothing of it is a public API. **No project source change.** | +| b11516–b11517 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11517 with `git apply`. No patch-target file in the range; nothing outside `ggml/src/ggml-cuda/` changed, so every drop-check of the b11505–b11512 row holds unchanged. | +| b11517–b11518 | One commit, 17 files, ~39/4 lines (32 KiB, one step). **Metal: flash-attention kernels for the 128/96 head-size pair, and every compiled (K, V) pair is accepted** (#30209; `ggml-metal-device.m` decides `supports_op` from the compiled kernel set instead of a fixed list, `ggml-metal-ops.cpp`, `-tuning.cpp`, and one instantiation line in each of the 14 `fa_*.metal` / `fa_vec_*.metal` kernel files). The shipped `metal-macos-aarch64` jar embeds the Metal library (`LLAMA_METAL_EMBED_LIBRARY=ON`), so a model with that head-size pair now runs flash attention on the GPU where it fell back before. **No project source change.** | +| b11517–b11518 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11518 with `git apply`. No patch-target file in the range; nothing outside `ggml/src/ggml-metal/` changed, so every drop-check of the b11505–b11512 row holds unchanged. | +| b11518–b11521 | Three commits, 214 files, ~8430/2907 lines (530 KiB, one step -- no intermediate tag between b11518 and b11521). **#30159 changes the server's default port from 8080 to 9931** (`common_params::port` in `common/common.h`, `server_http_context::port` in `tools/server/server-http.h`, the `--server-base` help text in `common/arg.cpp`; the "default port will be changed to :9931" notice `llama_server()` printed since #26508 is removed from `tools/server/server.cpp`; the READMEs, bench scripts and Dockerfiles follow). It reaches this project through `NativeServer`: an argv without `--port` now binds 9931, in classic mode and in attach mode alike (`llama_server_attach` parses its HTTP argv through `common_params_parse`), so **`NativeServer.DEFAULT_PORT` -- what `getPort()` reports when no port flag is given -- moves to 9931** with its Javadoc and `NativeServerSmokeTest`. `OpenAiCompatServer` keeps its own `DEFAULT_PORT = 8080`: it is the Java transport with no upstream coupling, and the constant is a public API of this project. Every README example passes `--port` explicitly and is unaffected. **#29583 adds the models manager to the WebUI** (~200 files under `tools/ui/`: drawer and sheet overlays, a models store with a download-status feed, Hugging Face and migration services, the selector regrouped by provider, the manager table with repo/quant/status rows, `vaul-svelte` and a newer `@lucide/svelte` in `package.json`). `build-webui` auto-follows the tag (`npm ci && npm run build`, `scripts/ui-assets.cmake` unchanged), and the routes the manager drives are the router's model-management endpoints the embedded server already carries. #30212 is upstream's own HIP CI quality check (`scripts/hip/gcn-cdna-vgpr-check.py`). **Project source change: `NativeServer.DEFAULT_PORT` and its test; CHANGELOG.** | +| b11518–b11521 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11521 with `git apply`. Patch-target files in the range: `common/arg.cpp` (`0001`, `0015`: one help string of `--server-base`, away from `common_params_parse` and `add_rpc_devices`) and `tools/server/server.cpp` (`0006`, `0007`: the eight removed lines sat after the `listening on` loop of `llama_server()`, between `0006`'s embedded-mode parse hunk and the router branch, and outside the route table `0007` factors out -- both hunks apply with their context intact). Drop-checks unchanged: `common_params_parse` still carries the Windows `argv` override, the load-progress assignment is still unconditional, no embedded-mode or attach counterpart upstream. | +| b11521–b11529 | Eight commits, 22 files, ~311/187 lines (49 KiB, the final step). **ggml: `ggml_acc` out-of-bounds write with a large offset** (#30135, `ggml/src/ggml.c`): the `size_t` offset was narrowed to `int32_t` unchecked, a value past `INT32_MAX` became negative, the forward sign-extended it and the bounds assertion wrapped, so the write landed below the destination buffer; the offset is now checked before the narrowing, as `ggml_set_impl` already did -- core code every natives jar builds. **Vulkan: `rms_norm` workgroup-count overflow** (#30145; `ggml-vulkan.cpp`, `rms_norm.comp`) for the three Vulkan jars. **Meta backend handles host views** (#30217): `--split-mode tensor` with partial offload asserted on KV-cache views left inside a meta split; such nodes are accepted when they are views of host tensors, and `LLM_ARCH_K2_HORIZON` leaves the list of architectures `llm_arch_supports_sm_tensor()` refuses (`src/llama-arch.cpp`, one line) -- relevant to `GpuSplitMode.TENSOR`. MUSA drops `mp_21` from its default architectures (#30203), WebGPU gets 2D workgroup dispatch and `GGML_OP_DUP` (#30219, #30216), the converter reads compressed-tensor mixed-precision NVFP4 checkpoints (#28636, Python), upstream's publish workflow gets a run name (#30214) -- none of it built or run here. **No project source change.** | +| b11521–b11529 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11529 with `git apply` and by a fresh configure through the real `FetchContent` path. Patch-target file in the range: `src/llama-arch.cpp` (`0016`: the Kolibri-1 entries sit in the arch and tensor tables, the removed line in `llm_arch_supports_sm_tensor()`). Drop-checks against pristine b11529 -- the final state of the bump: `common_params_parse` still carries the `#ifdef _WIN32` `argv = utf8.ptrs.data()` override and `common_params_parse_main` is absent (`0001`); the load-progress assignment is unconditional (`0002`); no `slot_prompt_similarity` getter (`0003`); no embedded-mode, attach or worker-command counterpart (`0006`-`0008`); no `split_sum == 0` guard (`0012`); `common/log.h` names only `common_log_default_callback`, no sink (`0014`); no `stop_server` and the connect failure still aborts in `ggml-rpc.cpp` (`0015`); no `kolibri` under `src/` (`0016`); the four `_mm_prefetch` calls in `quants.c` are still uncast element arithmetic (`0017`). No `.github/` change upstream that concerns this project (only its own `release-publish.yml`). | diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 38e4da798..ea36b9d42 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -261,7 +261,7 @@ endif() FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b11512 + GIT_TAG b11529 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= diff --git a/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java b/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java index d11a65ae8..b91527267 100644 --- a/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java +++ b/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java @@ -80,8 +80,11 @@ public final class NativeServer implements AutoCloseable { /** Default bind host reported by {@link #getHost()} when {@code --host} is not passed. */ private static final String DEFAULT_HOST = "127.0.0.1"; - /** Default port reported by {@link #getPort()} when no port flag is passed. */ - private static final int DEFAULT_PORT = 8080; + /** + * Default port reported by {@link #getPort()} when no port flag is passed: llama.cpp's own + * default, 9931 since b11521 (upstream #30159; 8080 before), in classic and attach mode alike. + */ + private static final int DEFAULT_PORT = 9931; /** The llama-server argument vector, forwarded verbatim to the native entry point. */ private final String[] args; @@ -230,8 +233,8 @@ public List getHosts() { } /** - * Returns the port parsed from the arguments ({@code --port} / {@code -p}), or {@code 8080} when - * absent or unparseable. Best-effort convenience for logging. + * Returns the port parsed from the arguments ({@code --port} / {@code -p}), or {@code 9931} (the + * upstream default since llama.cpp b11521) when absent or unparseable. Best-effort convenience for logging. * * @return the configured port */ diff --git a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java index 54f6aea8e..550fb48b2 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java +++ b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java @@ -9,13 +9,13 @@ * library was compiled against, exposed as a compile-time constant so callers can render a badge or * emit a startup log line without loading the native library. * - *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b11512"}) that mirrors the + *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b11529"}) that mirrors the * {@code GIT_TAG} in {@code llama/CMakeLists.txt}. It is available even when {@code libjllama} is * absent (pure-Java checkout, before {@code System.load}), which is what makes it suitable for a * lightweight version badge in Android or other UIs.

* *

For the authoritative value that is baked into the native binary — the build number - * plus the resolved upstream commit, e.g. {@code "b11512-"} — call + * plus the resolved upstream commit, e.g. {@code "b11529-"} — call * {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} instead; that reads llama.cpp's own * {@code build-info} through JNI and therefore cannot drift from the compiled library (but requires * the native library to be loaded).

@@ -23,14 +23,14 @@ public final class LlamaCppVersion { /** - * The pinned llama.cpp release tag this library was built against, e.g. {@code "b11512"}. + * The pinned llama.cpp release tag this library was built against, e.g. {@code "b11529"}. * *

Kept in lockstep with {@code GIT_TAG} in {@code llama/CMakeLists.txt} — see the * "Upgrading/Downgrading llama.cpp Version" checklist in {@code CLAUDE.md}. This is the * compile-time pin; use {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} for the * value actually linked into the native binary.

*/ - public static final String LLAMA_CPP_VERSION = "b11512"; + public static final String LLAMA_CPP_VERSION = "b11529"; // Constants holder — not instantiable. private LlamaCppVersion() {} diff --git a/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java b/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java index c994867eb..c0267c3c1 100644 --- a/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java +++ b/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java @@ -54,13 +54,13 @@ public void defaultsWhenFlagsAbsent() { NativeServer server = new NativeServer("-m", "m.gguf"); assertThat(server.getHost(), is("127.0.0.1")); assertThat(server.getHosts(), contains("127.0.0.1")); - assertThat(server.getPort(), is(8080)); + assertThat(server.getPort(), is(9931)); } @Test public void nonIntegerPortFallsBackToDefault() { NativeServer server = new NativeServer("-m", "m.gguf", "--port", "abc"); - assertThat(server.getPort(), is(8080)); + assertThat(server.getPort(), is(9931)); } @Test