From 1eef547b6a1f6500d23e4b0fc8eb37f651367b5c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:10:23 +0000 Subject: [PATCH 1/4] Upgrade llama.cpp from b11534 to b11538 All eleven patches apply unchanged (verified in order against the pristine tag); see docs/history/llama-cpp-breaking-changes.md for the range. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- CLAUDE.md | 8 ++++---- README.md | 2 +- docs/history/llama-cpp-breaking-changes.md | 2 ++ llama/CMakeLists.txt | 2 +- .../java/net/ladenthin/llama/value/LlamaCppVersion.java | 8 ++++---- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e07a3cfc7..4f01daf2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b11534** +Current llama.cpp pinned version: **b11538** ## Natives jars: one directory per backend (`.github/natives.csv`) @@ -897,7 +897,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network for the asset build; the embed step is plain cmake -P -git clone --depth 1 --branch b11534 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b11538 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build ) mkdir -p webui-generated /tmp/ui-gen cmake -DUI_SOURCE_DIR=/tmp/lc/tools/ui -DUI_BINARY_DIR=/tmp/ui-gen \ @@ -937,7 +937,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b11534`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b11538`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -2167,7 +2167,7 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b11534`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b11538`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.18.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the diff --git a/README.md b/README.md index 551c3e6ac..525e13a54 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b11534](https://img.shields.io/badge/llama.cpp-%23b11534-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b11534) +[![llama.cpp b11538](https://img.shields.io/badge/llama.cpp-%23b11538-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b11538) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 8b67ec159..e3ed6bc5e 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -866,3 +866,5 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b11530–b11531 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11531 with `git apply` and by a fresh configure, build and `ctest` (603/603) through the real `FetchContent` path, since this step changes project code. Patch-target files in the range: `tools/server/server-context.cpp` and `server-context.h` (`0002`, `0003`: the session plumbing sits in `handle_completions_impl` and the route lambdas, away from `load_model` and the similarity getter/setter). Drop-checks unchanged. | | b11531–b11534 | Three commits, 12 files, ~297/33 lines (32 KiB, the final step). **#30176 fixes OpenCL kernel compilation on Adreno A6x** (`ggml-opencl.cpp`, three kernels): the A6x shader compiler crashes on `kernel_cpy_f32_f32_pack`'s argument count and mis-folds `get_local_size` in the GEMV kernels, so the pack kernel is skipped and the GEMV kernels get a constant-fold workaround on that family -- this is the `opencl-android-aarch64` jar and the `llama-android-opencl` AAR on A6x devices. **#30108 uses exact GELU for ModernBERT encoders** (`src/models/modern-bert.cpp`, `llama-graph`, `llama-model.cpp`; tanh-GELU aliases kept on `ggml_geglu`): a numerics change for ModernBERT embedding and reranking models, not for the CI reranker (JinaBERT). **#29807 removes redundant CUDA copies after `SSM_SCAN`** (the state snapshot is written into the recurrent cache by the kernel; Mamba-style models on the CUDA jars). **No project source change.** | | b11531–b11534 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11534 with `git apply` and by a fresh configure through the real `FetchContent` path. Patch-target file in the range: `src/llama-model.cpp` (`0012`, `0016`: the GELU change is in the ModernBERT graph wiring, away from `load_tensors`' split block and the Kolibri-1 entries). Drop-checks against pristine b11534 -- the final state of the bump: `common_params_parse` still carries the `#ifdef _WIN32` `argv = utf8.ptrs.data()` override and `common_params_parse_main` is absent (`0001`); the load-progress assignment is unconditional (`0002`); no `slot_prompt_similarity` getter (`0003`); no embedded-mode, attach or worker-command counterpart (`0006`-`0008`); no `split_sum == 0` guard (`0012`); `common/log.h` names only `common_log_default_callback`, no sink (`0014`); no `stop_server` and the connect failure still aborts in `ggml-rpc.cpp` (`0015`); no `kolibri` under `src/` (`0016`); the four `_mm_prefetch` calls in `quants.c` are still uncast element arithmetic (`0017`). No `.github/` change upstream in the range. | +| b11534–b11538 | Four commits, 23 files, ~457/152 lines (45 KiB, one step; 21 KiB without the auto-followed WebUI). **#30228 "ui: Models Manager follow-up improvements"** is the bulk (12 of the files, `tools/ui`, auto-followed by `build-webui`) and the one commit that reaches a project surface: the router's `GET /models` now carries **`context_length`** per model, read offline from the GGUF (`.context_length`) by the new `common_get_gguf_info()` (`common/common.{h,cpp}`), which **replaces** `common_get_decision_type(const std::string &)` and returns a `common_gguf_info {decision_type, n_ctx_train}`; `server_model_meta` gains `n_ctx_train` (`server-models.{h,cpp}`, `update_caps`). No project C++ called the replaced overload. **#30160 "graph : reorder get_rows for embeddings"** (`src/llama-graph.cpp`, `gemma4.cpp`, `qwen4exp.cpp`): the embedding `get_rows` is built after the input tensors, upstream-only. **#30229** fixes a CUDA rounding mismatch between CPU and GPU under MSVC (`ggml-cuda/unary.cu`, one line); **#28331** makes `llama-bench` respect `-fitc` (tool only). **Project source change:** `RouterModel.getContextLength()` + the parser read `context_length` (0 when the server omits it, i.e. before b11538 or for a file without the key). | +| b11534–b11538 | patches + upstream verification | **All eleven patches apply unchanged**, verified in order against pristine b11538 with `git apply` and by a fresh configure, build and `ctest` through the real `FetchContent` path. Patch-target files in the range: `tools/server/server-models.{cpp,h}` (`0008`: `update_caps()` gains the `n_ctx_train` read, two hunks after `update_args`, which is the patch's region -- the context lines the patch carries are unchanged). Drop-checks against pristine b11538: `common_params_parse` still carries the `#ifdef _WIN32` `argv = utf8.ptrs.data()` override and `common_params_parse_main` is absent (`0001`); the load-progress assignment is unconditional (`0002`); no `slot_prompt_similarity` getter (`0003`); no embedded-mode, attach or worker-command counterpart (`0006`-`0008`); no `split_sum == 0` guard (`0012`); `common/log.h` names only `common_log_default_callback`, no sink (`0014`); no `stop_server` and the connect failure still aborts in `ggml-rpc.cpp` (`0015`); no `kolibri` under `src/` (`0016`); the four `_mm_prefetch` calls in `quants.c` are still uncast element arithmetic (`0017`). No `.github/` change upstream in the range. | diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index d276247fa..74b759dd6 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -278,7 +278,7 @@ endif() FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b11534 + GIT_TAG b11538 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= diff --git a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java index b8f6c0255..f338ef8cd 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java +++ b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java @@ -9,13 +9,13 @@ * library was compiled against, exposed as a compile-time constant so callers can render a badge or * emit a startup log line without loading the native library. * - *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b11534"}) that mirrors the + *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b11538"}) that mirrors the * {@code GIT_TAG} in {@code llama/CMakeLists.txt}. It is available even when {@code libjllama} is * absent (pure-Java checkout, before {@code System.load}), which is what makes it suitable for a * lightweight version badge in Android or other UIs.

* *

For the authoritative value that is baked into the native binary — the build number - * plus the resolved upstream commit, e.g. {@code "b11534-"} — call + * plus the resolved upstream commit, e.g. {@code "b11538-"} — call * {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} instead; that reads llama.cpp's own * {@code build-info} through JNI and therefore cannot drift from the compiled library (but requires * the native library to be loaded).

@@ -23,14 +23,14 @@ public final class LlamaCppVersion { /** - * The pinned llama.cpp release tag this library was built against, e.g. {@code "b11534"}. + * The pinned llama.cpp release tag this library was built against, e.g. {@code "b11538"}. * *

Kept in lockstep with {@code GIT_TAG} in {@code llama/CMakeLists.txt} — see the * "Upgrading/Downgrading llama.cpp Version" checklist in {@code CLAUDE.md}. This is the * compile-time pin; use {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} for the * value actually linked into the native binary.

*/ - public static final String LLAMA_CPP_VERSION = "b11534"; + public static final String LLAMA_CPP_VERSION = "b11538"; // Constants holder — not instantiable. private LlamaCppVersion() {} From 70a673ba9273155ac071e432cf5d299ae1561b0a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:13:25 +0000 Subject: [PATCH 2/4] RouterModel.getContextLength(): the router's context_length (llama.cpp b11538) Upstream #30228 makes GET /models carry the trained context length per model, read offline from the GGUF. The parser reads it, 0 when absent (a server before b11538, or a file without the key); value tests cover the getter, the constructor delegation and equals/hashCode. CHANGELOG entries for the bump and the getter. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- CHANGELOG.md | 10 ++++ .../json/RouterModelsResponseParser.java | 6 ++- .../ladenthin/llama/value/RouterModel.java | 43 ++++++++++++++++- .../json/RouterModelsResponseParserTest.java | 13 +++++ .../llama/value/RouterModelTest.java | 47 +++++++++++++++++++ 5 files changed, 116 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf5b0d80..a445c9dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by ## [Unreleased] ### Added +- **`RouterModel.getContextLength()`** (llama.cpp b11538, #30228): the context length a model was + trained with, which the router now reads offline from the GGUF for `GET /models` (`context_length`, + `.context_length`) -- known before the first load, like the modalities; `0` from a server before + b11538 or for a file without the key. - **CI proves HTTPS in both directions on the release assets.** The fat-jar smokes (Linux x86-64 / aarch64, Windows x86-64 / arm64) and the macOS smoke each start the server behind a self-signed certificate (`--ssl-key-file` / `--ssl-cert-file`; a plain-HTTP request to the port must be refused) @@ -69,6 +73,12 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by in `build.sh` opts back in). With CUDA 13.4 and sccache 0.18.0 the nvcc launcher failed on the first `.cu` files in two confirmed runs (`Missing "cubin" file output`, `Compiler killed by signal 126`), and the uncached retry that followed made those jobs slower than a cold build; the C/C++ objects stay cached. +- **Upgraded the pinned llama.cpp from b11534 to b11538**, one reviewed step. All eleven patches apply + unchanged and every drop-check still finds its defect. In the range: the router's `GET /models` carries + `context_length` per model (#30228, the `RouterModel` getter above; `common_get_decision_type(fname)` + became `common_get_gguf_info()`, which no project code called), the embedding `get_rows` is built after + the inputs (#30160), a CUDA rounding mismatch between CPU and GPU under MSVC is fixed (#30229), and + `llama-bench` respects `-fitc` (#28331). Per-step record: `docs/history/llama-cpp-breaking-changes.md`. - **CI: the three dockcross images are pinned by digest** (`android-arm64`, `android-x86_64`, `manylinux_2_28-x64`, all tag `20260712-79e54f9`), as the manylinux aarch64 image already was. - **Upgraded the pinned llama.cpp from b11529 to b11534**, in 3 reviewed steps, each ending at a tag. No diff --git a/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java b/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java index 97b1b9222..fd64a6998 100644 --- a/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java +++ b/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java @@ -66,7 +66,8 @@ public List parse(String json) { * {@link RouterModel.Status#UNKNOWN} with an empty raw value. The failure marker is read * from {@code status.failed} / {@code status.exit_code}, the modalities from * {@code architecture.input_modalities} / {@code architecture.output_modalities} (empty when - * absent, as from a server before llama.cpp b11429). + * absent, as from a server before llama.cpp b11429), and the trained context length from + * {@code context_length} ({@code 0} when absent, as from a server before b11538). * * @param root pre-parsed router {@code GET /models} response * @return list of models; empty list when no entry array is present @@ -92,7 +93,8 @@ public List parse(JsonNode root) { status.path("failed").asBoolean(false), status.path("exit_code").asInt(0), strings(architecture.path("input_modalities")), - strings(architecture.path("output_modalities")))); + strings(architecture.path("output_modalities")), + entry.path("context_length").asInt(0))); } return models; } diff --git a/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java b/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java index 1cfed7673..878a72210 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java +++ b/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java @@ -17,7 +17,8 @@ * status string as emitted by the server, the failure marker the router attaches when a * worker exited abnormally, and the model's input and output modalities (llama.cpp b11429+), which * the router computes without loading the model -- so {@link #isDecisionModel()} finds a decision - * model for {@code /v1/systemone} before its first load. + * model for {@code /v1/systemone} before its first load -- and, since llama.cpp b11538, the context + * length the model was trained with ({@code context_length}), read from the GGUF the same way. * *

{@code equals}/{@code hashCode} are generated by Lombok over all fields. * {@code toString} is intentionally handwritten (not Lombok-generated) so that router traces @@ -86,6 +87,7 @@ public static Status fromValue(String value) { private final int exitCode; private final List inputModalities; private final List outputModalities; + private final int contextLength; /** * Construct a router model entry. @@ -130,6 +132,31 @@ public RouterModel( int exitCode, Collection inputModalities, Collection outputModalities) { + this(id, status, statusValue, failed, exitCode, inputModalities, outputModalities, 0); + } + + /** + * Construct a router model entry with the model's modalities and trained context length. + * + * @param id the model identifier + * @param status the parsed lifecycle status + * @param statusValue the raw {@code status.value} string as emitted by the server + * @param failed whether the router flagged the model's worker as failed + * @param exitCode the failed worker's exit code; {@code 0} when not failed + * @param inputModalities {@code architecture.input_modalities}; empty when the server omits it + * @param outputModalities {@code architecture.output_modalities}; empty when the server omits it + * @param contextLength {@code context_length}, the trained context in tokens; {@code 0} when + * the server omits it + */ + public RouterModel( + String id, + Status status, + String statusValue, + boolean failed, + int exitCode, + Collection inputModalities, + Collection outputModalities, + int contextLength) { this.id = id; this.status = status; this.statusValue = statusValue; @@ -137,6 +164,7 @@ public RouterModel( this.exitCode = exitCode; this.inputModalities = Collections.unmodifiableList(new ArrayList<>(inputModalities)); this.outputModalities = Collections.unmodifiableList(new ArrayList<>(outputModalities)); + this.contextLength = contextLength; } /** @@ -212,6 +240,19 @@ public boolean isDecisionModel() { return outputModalities.contains(ModelMeta.OUTPUT_MODALITY_DECISIONS); } + /** + * The context length the model was trained with ({@code context_length}, the GGUF's + * {@code .context_length}), which the router reads offline since llama.cpp b11538 (#30228). + * The size a worker actually serves with is its own {@code -c}; this is the ceiling worth + * requesting. + * + * @return the trained context length in tokens; {@code 0} when the server did not report it + * (a server before b11538, or a GGUF without the key) + */ + public int getContextLength() { + return contextLength; + } + @Override public String toString() { if (failed) { diff --git a/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java b/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java index ff37dd990..cf9e57d93 100644 --- a/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java +++ b/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java @@ -115,6 +115,19 @@ public void parsesTheArchitectureModalities() { assertThat(model.isDecisionModel(), is(true)); } + @Test + public void parsesTheContextLength() { + // GET /models since llama.cpp b11538 (#30228): .context_length, read offline from the GGUF. + String json = "{\"data\":[{\"id\":\"qwen\",\"status\":{\"value\":\"unloaded\"},\"context_length\":40960}]}"; + assertThat(parser.parse(json).get(0).getContextLength(), is(40960)); + } + + @Test + public void missingContextLengthYieldsZero() { + // A server before b11538, or a GGUF without the key: the router omits the field. + assertThat(parser.parse("{\"data\":[{\"id\":\"old\"}]}").get(0).getContextLength(), is(0)); + } + @Test public void missingArchitectureYieldsNoModalities() { // A server before b11429 omits the object; upstream asks clients to fall back, not to guess. diff --git a/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java b/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java index 2a1abd767..21fb69b66 100644 --- a/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java +++ b/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java @@ -136,6 +136,53 @@ public void modalitiesAreACopyAndUnmodifiable() { () -> model.getOutputModalities().add("x")); } + @Test + public void legacyAndModalityConstructorsReportNoContextLength() { + assertThat(new RouterModel("m", RouterModel.Status.LOADED, "loaded", false, 0).getContextLength(), is(0)); + assertThat( + withModalities(Collections.singletonList("text"), Collections.singletonList("text")) + .getContextLength(), + is(0)); + } + + @Test + public void contextLengthRoundTrips() { + RouterModel model = new RouterModel( + "m", + RouterModel.Status.UNLOADED, + "unloaded", + false, + 0, + Collections.singletonList("text"), + Collections.singletonList("text"), + 40960); + assertThat(model.getContextLength(), is(40960)); + } + + @Test + public void equals_differsPerContextLength() { + RouterModel a = new RouterModel( + "m", + RouterModel.Status.UNLOADED, + "unloaded", + false, + 0, + Collections.singletonList("text"), + Collections.singletonList("text"), + 4096); + RouterModel b = new RouterModel( + "m", + RouterModel.Status.UNLOADED, + "unloaded", + false, + 0, + Collections.singletonList("text"), + Collections.singletonList("text"), + 8192); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); + } + @Test public void equals_differsPerModality() { RouterModel base = withModalities(Collections.singletonList("text"), Collections.singletonList("text")); From 72ddd58e446fdd867c3a7ef795443bd2150d2f33 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:33:08 +0000 Subject: [PATCH 3/4] Agent follow-ups: the approval gate and the tool-round carry exist; TODO and handover updated Two entries of the Atmosphere agent's follow-up list in TODO.md were stale: - "Approval for destructive tools": the gate has shipped (ApprovalMode, ConsoleApprovalStrategy, AgentRunner.approval with Atmosphere's ToolApprovalPolicy, --auto for unattended runs; ApprovalWireTest and ConsoleApprovalStrategyTest pin it). Removed. - "Tool rounds are not carried across REPL turns": rewritten with the measured reason the asked-for form is impossible on atmosphere-ai 4.0.72 (AbstractAgentRuntime.assembleMessages rebuilds every history entry as new ChatMessage(role, content), checked in the bytecode), what ships instead (AgentSession.toolNote in front of the next user message, pinned by LocalAgentTest), and when to reopen it. docs/handover/local-agent-pr-a.md said it is deleted once PR A is merged; PR A and the agent's corrections are, but deliverable 6 carries the agent's measurements and the four OPEN items the Windows CPU-variants job (B2) still needs, so the file stays with a status note and TODO.md's "CPU variants" entry points at it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- TODO.md | 24 +++++++++++++++++------- docs/handover/local-agent-pr-a.md | 9 +++++++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/TODO.md b/TODO.md index ffa82a1c3..05f3a935c 100644 --- a/TODO.md +++ b/TODO.md @@ -21,7 +21,12 @@ so everything below is genuinely still open. Linux x86-64 and aarch64 ship the variant build since 5.2.0 (CLAUDE.md "CPU variants"). The rest, with what a measurement on a Windows 11 machine (Ryzen 7 5800H, RTX 3070, JDK 21, upstream b11476 -binaries; 2026-10-08) established: +binaries; 2026-10-08) established. The local agent's follow-up measurements at b11534 on the same +machine, through this project's own CMake (plain clang 23.1.3 builds all 14 variants, 18 DLLs, +`verify-native-deps.py` 0 violations) and the four items still `OPEN` for the CI job -- the +`build.bat` command line, the Java suite against the variant build, which variant ggml picks on +Zen 3, pp512/tg128 against the static build -- are in `docs/handover/local-agent-pr-a.md`, +deliverable 6: 1. **Windows x86-64 build.** Only plain `clang`/`clang++` with the GNU driver -- upstream's `cmake/x64-windows-llvm.cmake`, four lines -- produces all 14 x86 variants: CMake sets `MSVC` for @@ -250,12 +255,17 @@ binaries; 2026-10-08) established: The headless loop is verified, including the model-backed CI job (run 35600558852: tool call answered, read→write→read loop changed the file). Still open: -- **Tool rounds are not carried across REPL turns** — only `user`/`assistant` text is replayed, so a - second question cannot refer to a tool result of the first. Keep the full Atmosphere - `ChatMessage` list (incl. `tool_calls`/`tool` messages) per turn instead. -- **Approval for destructive tools.** `write_file`/`delete`/`run_command` run unasked. Atmosphere's - `ToolDefinition.requiresApproval` + an `ApprovalStrategy` on the context would give a Claude-Code - style "allow this?" prompt on the console. +- **Tool rounds across REPL turns ride as a note, not as `tool_calls` messages -- and that cannot + change on this Atmosphere.** The earlier entry here asked to keep the full `ChatMessage` list + (incl. `tool_calls`/`tool`) per turn; measured against `atmosphere-ai` 4.0.72, that is impossible: + `AbstractAgentRuntime.assembleMessages` rebuilds every history entry as + `new ChatMessage(h.role(), h.content())` (checked in the bytecode), so the tool-call array and the + tool-call id never leave the framework. What ships instead: `AgentSession.toolNote` carries each + turn's calls and results (cut at 400 chars) in front of the next user message, pinned by + `LocalAgentTest.aToolCallStaysInTheHistorySoTheNextTurnSeesItHappened`; the full placement + argument is in `llama-atmosphere-agent/CLAUDE.md`, point 8. Reopen only with an Atmosphere + release whose `assembleMessages` preserves tool calls (then a protocol-faithful replay is the + better form, and the note goes). - **In-stream engine errors are swallowed by Atmosphere** (pinned in `AtmosphereWireContractTest.midStreamEngineFailureCompletesSilentlyRatherThanErroring`): an SSE `data: {"error":…}` after HTTP 200 is ignored by `OpenAiCompatibleClient.processSSELine` (it reads diff --git a/docs/handover/local-agent-pr-a.md b/docs/handover/local-agent-pr-a.md index 15113422b..ea57dc15e 100644 --- a/docs/handover/local-agent-pr-a.md +++ b/docs/handover/local-agent-pr-a.md @@ -2,8 +2,13 @@ Paste the text below the line into the local agent (Windows workstation: Ryzen 7 5800H, RTX 3070, AMD iGPU, MSVC + LLVM + Docker + JDK 21 + Maven). It describes what this branch already contains, -what the remote session cannot verify from its sandbox, and the exact deliverables. The file is -deleted again once PR A is merged. +what the remote session cannot verify from its sandbox, and the exact deliverables. + +**Status (2026-10-09).** PR A (#496) and the agent's own corrections to this file (#498) are merged; +the branch has since moved on (b11538, `InvalidRequestException`). The file stays for one reason: +deliverable 6 (B2) carries the agent's measurements and the four `OPEN` items that the Windows +CPU-variants CI job still needs, and `TODO.md` ("CPU variants") points here for them. Deliverables +1-5 were not reported and are not asked for again; delete the file when B2 lands. **Deliverable 6 (B2) carries measurements taken on this same workstation on 2026-10-09**, marked `MEASURED` or `OPEN` per item: four are done and only need confirming, three are genuinely open, and From 8e0283e1b97c5ddbe313964fc089f057566a3f9b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:33:25 +0000 Subject: [PATCH 4/4] InvalidRequestException: the JNI layer types a rejected request, OpenAiCompatServer answers 400 llama.cpp's own HTTP server has answered a malformed or empty request with 400 (invalid_request_error) since b11337 (#29060); the Java transport answered 500, because the JNI layer threw the same LlamaException for a rejected request and a failed inference and the server could not tell them apart without matching on message text (TODO.md, "400 vs. 500"). Native side: a second exception class, exception.InvalidRequestException (a LlamaException), cached as c_llama_invalid_request next to c_llama_error. Three helpers decide which one a failure gets, each with upstream's rule: throw_invalid_request for request parsing (an invalid request whatever the parser threw; now also behind parse_json_params and require_json_field), throw_by_exception_type for upstream code whose exception type carries the verdict (std::invalid_argument and common_json_error are 400, as in upstream's ex_wrapper; tokenization, format_prompt_infill, the /v1/systemone handler), and error_class_for for a failed task result, from the new json_helpers get_result_error_code (400 for ERROR_TYPE_INVALID_REQUEST and ERROR_TYPE_EXCEED_CONTEXT_SIZE). The empty-input and pooling-type checks of handleEmbeddings and a 400 from the systemone route are typed the same way. The message of a rejected request is the bare reason now, no longer the JSON object the old throw_invalid_request wrapped it in; the type is the class. Java side: OpenAiCompatServer catches InvalidRequestException together with the IllegalArgumentException of its own validation at the ten request sites (400, or the invalid_request_error SSE event once a stream has started), on every surface. A plain LlamaException stays 500. The Server layer may now see the Exception layer (LlamaArchitectureTest). Tests: four C++ tests for get_result_error_code (607 total); six model-free HTTP tests (a backend throwing the typed exception is 400 for chat, embeddings, completions, Ollama and Anthropic, an SSE error event when streaming, a plain LlamaException 500); ErrorHandlingTest tightened to the typed exception plus an empty input array and a malformed body; model-backed 400 cases in the embeddings and completion server integration tests. Javadoc on every entry point that can throw it; CHANGELOG, CLAUDE.md (the JNI exception boundary, the helper and test tables) and TODO.md updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AytmJF9faEiQEVt6eetQS2 --- CHANGELOG.md | 19 +++ CLAUDE.md | 27 ++++- TODO.md | 6 - llama/src/main/cpp/jllama.cpp | 88 ++++++++++---- llama/src/main/cpp/json_helpers.hpp | 19 +++ .../java/net/ladenthin/llama/LlamaModel.java | 32 +++++- .../exception/InvalidRequestException.java | 41 +++++++ .../llama/server/LlamaModelBackend.java | 6 + .../ladenthin/llama/server/OpenAiBackend.java | 8 ++ .../llama/server/OpenAiCompatServer.java | 26 +++-- llama/src/test/cpp/test_json_helpers.cpp | 35 ++++++ .../ladenthin/llama/ErrorHandlingTest.java | 51 +++++++-- .../llama/LlamaArchitectureTest.java | 6 +- .../server/OpenAiCompatServerHttpTest.java | 108 ++++++++++++++++++ ...OpenAiServerCompletionIntegrationTest.java | 9 ++ ...OpenAiServerEmbeddingsIntegrationTest.java | 21 ++++ 16 files changed, 443 insertions(+), 59 deletions(-) create mode 100644 llama/src/main/java/net/ladenthin/llama/exception/InvalidRequestException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a445c9dbc..dbb762a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by ## [Unreleased] ### Added +- **`InvalidRequestException`** (`net.ladenthin.llama.exception`, a `LlamaException`): the JNI layer now + throws it wherever llama.cpp's own HTTP server answers `400` (`invalid_request_error`) -- a body that is + not valid JSON, a missing `"prompt"` / `"messages"` / `"input_prefix"`, an empty embedding input, an + unknown `encoding_format`, a parameter outside its hard limits, a prompt over the context size, and a + `std::invalid_argument` or `common_json_error` out of tokenization, prompt formatting or the + `/v1/systemone` handler (the same rule as upstream's `ex_wrapper`). Everything else stays a plain + `LlamaException`. A `catch (LlamaException)` keeps catching both. - **`RouterModel.getContextLength()`** (llama.cpp b11538, #30228): the context length a model was trained with, which the router now reads offline from the GGUF for `GET /models` (`context_length`, `.context_length`) -- known before the first load, like the modalities; `0` from a server before @@ -64,6 +71,10 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by core's; `check-natives.py` fails when they differ. ### Changed +- **The message of a rejected request is the bare reason** (`"prompt" must not be empty`), no longer the + JSON object `{"code":400,"message":…,"type":"invalid_request_error"}` the old `throw_invalid_request` + wrapped it in: the type is the exception class now (`InvalidRequestException`), as it is for every + other `LlamaException`. - **`NativeServer` binds port 9931 when no `--port` is given** (llama.cpp b11521, [#30159](https://github.com/ggml-org/llama.cpp/pull/30159) moved the server default off 8080), in classic and attach mode alike; `NativeServer.getPort()` reports 9931 accordingly. `OpenAiCompatServer` keeps its own default @@ -293,6 +304,14 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by 4.4.7 (none of the files the agent's carried JLine fixes touch changed between 4.4.6 and 4.4.7). ### Fixed +- **`OpenAiCompatServer` answers `400`, not `500`, for a request the native layer rejects** -- an empty + `input` array for `/v1/embeddings`, a `/v1/completions` body without a `prompt`, a malformed body, a + message the chat template cannot take, a prompt over the context size -- on every surface (OpenAI, + Ollama, Anthropic, Responses; as an `invalid_request_error` SSE event once a stream has started). + Upstream's server has answered these with `400` since b11337 (#29060); the Java transport reported them + as `server_error`, because the JNI layer threw the same `LlamaException` for a rejected request and a + failed inference. The server now maps `InvalidRequestException` like the `IllegalArgumentException` of + its own validation, without matching on message text; a plain `LlamaException` stays `500`. - **HTTPS works out of every desktop natives jar, and the macOS dylib no longer needs Homebrew's OpenSSL.** `jllama` compiled cpp-httplib itself, without SSL, next to the SSL-enabled copy upstream's `llama-common` (the download code) links -- two incompatible copies of the same classes in one link, and which one the diff --git a/CLAUDE.md b/CLAUDE.md index 4f01daf2e..8e56483b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1710,7 +1710,7 @@ root from the loader's own package (which moved to `…loader`) and (b) `jllama.cpp` still `FindClass`-ing the old flat paths — and neither was visible to a local `mvn test` (model tests skipped) or to the pure-Java unit tests. **When you move a Java class the JNI layer references by name** (`LlamaModel` -[root], `exception.LlamaException`, `value.LogLevel`, `args.LogFormat`, +[root], `exception.LlamaException`, `exception.InvalidRequestException`, `value.LogLevel`, `args.LogFormat`, `callback.LoadProgressCallback`), update the matching `FindClass` / `"L…;"` signature string in `src/main/cpp/jllama.cpp` and keep the native-resource root anchored at `net/ladenthin/llama` in `LlamaLoader.NATIVE_RESOURCE_BASE` (it must @@ -1866,7 +1866,7 @@ The project C++ helpers follow a strict semantic split: Functions: `get_result_error_message`, `results_to_json`, `rerank_results_to_json`, `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`, -`server_metrics_to_json`, `route_error_message`. +`server_metrics_to_json`, `route_error_message`, `get_result_error_code`. **`log_helpers.hpp`** — Pure log-formatting transforms. - Input: `ggml_log_level`, message text (`const char*`), an explicit `std::time_t` timestamp. @@ -1955,6 +1955,25 @@ rather than a Java exception, and `train_engine.cpp` deliberately keeps its own and never includes `jni_helpers.hpp`, so its backstop returns an error string to preserve that contract. +**Two Java classes, chosen by what upstream would answer.** `c_llama_error` (`LlamaException`) is the +backstop and the class for everything that is not the request's fault; `c_llama_invalid_request` +(`InvalidRequestException`, a subclass) is thrown wherever llama.cpp's own HTTP server answers `400`, +so that `OpenAiCompatServer` can answer `400` too without matching on message text (it maps the type +like the `IllegalArgumentException` of its own validation; everything else is `500`). Three helpers +in `jllama.cpp` decide it: `throw_invalid_request` for request *parsing* (an invalid request whatever +the parser threw — upstream's parsers mix `std::invalid_argument` and `std::runtime_error` for the +same kind of defect; also behind `parse_json_params` and `require_json_field`), +`throw_by_exception_type` for upstream code whose exception type carries the verdict +(`std::invalid_argument` and `common_json_error` → 400, else 500 — the rule of upstream's `ex_wrapper` +in `tools/server/server.cpp`; tokenization, `format_prompt_infill`, the `/v1/systemone` handler), and +`error_class_for` for a failed task result (`get_result_error_code` reads the code +`format_error_response` put into it: 400 for `ERROR_TYPE_INVALID_REQUEST` and +`ERROR_TYPE_EXCEED_CONTEXT_SIZE`). A `ThrowNew(c_llama_error, …)` is still right for a missing +capability (`--embedding`, `--reranking`, FIM tokens: upstream answers those `501`) and for a broken +result. Pinned model-free by `OpenAiCompatServerHttpTest` (a backend throwing the typed exception is +`400` on every surface, a plain `LlamaException` `500`) and model-backed by `ErrorHandlingTest` and +the two server integration tests (an empty `input` array, a `/v1/completions` body without a prompt). + **When you add a native method, wrap it.** The guard is not enforced by a test — a new unguarded entry point is invisible until something throws through it in production. @@ -2148,7 +2167,7 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" |------|-------|-------| | `src/test/cpp/test_utils.cpp` | 168 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse`, `parse_lora_request`, `common_chat_parse` over malformed UTF-8 (the `ContentOnlyParseUtf8` guard, which pins upstream #29161's one-U+FFFD-per-invalid-run contract — formerly the guard for the dropped `patches/0011`) | | `src/test/cpp/test_server.cpp` | 206 | Upstream result types: `server_slot_stats` (the `timings` JSON payload; replaced `result_timings` in b10408), `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics` (`to_metrics()` = the `/metrics` Prometheus exposition text; its `to_json()` has been unused since b10519 and returns `json{}` = JSON null), `server_task_result_slots` (`to_json()` = the `/slots` array, fed by the b10519 `SERVER_TASK_TYPE_SLOT_GET` task), `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | -| `src/test/cpp/test_json_helpers.cpp` | 67 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`, `server_metrics_to_json`, `route_error_message` | +| `src/test/cpp/test_json_helpers.cpp` | 71 | All functions in `json_helpers.hpp`: `get_result_error_message`, `get_result_error_code` (the HTTP code of a failed task result, from which `jllama.cpp` picks `InvalidRequestException` or `LlamaException`), `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk`, `server_metrics_to_json`, `route_error_message` | | `src/test/cpp/test_log_helpers.cpp` | 13 | All functions in `log_helpers.hpp`: `log_level_name`, `format_log_as_json` | | `src/test/cpp/test_common_log_callback.cpp` | 6 | **The runnable guard for `patches/0014`**: `common_log_set_callback()` on a private `common_log_init()` instance (never `common_log_main()`, so the process-wide logger the other tests print through is untouched) — delivery of level + bare text, no prefix/timestamp even when both are on (what `common_init()` does), clearing stops delivery, a swap drains queued entries to the *previous* sink (the property behind `LlamaModel.setLogger(format, null)` being a synchronous flush), a `--log-file` keeps being written alongside the sink, and every `ggml_log_level` passes through unchanged. The Java half (`LlamaLoggerTest`, model-free) proves the JNI trampoline on top of it. | | `src/test/cpp/test_jni_helpers.cpp` | 70 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw). Seven of them pin `jni_guard_impl` — the JNI exception boundary every `Java_*` entry point runs inside — including the `catch (...)` arm that is the only backstop for a non-`std::exception` type, and its two refusals (never `ThrowNew` over a pending Java exception, never with a null class). | @@ -2163,7 +2182,7 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | `src/test/cpp/test_native_server_shutdown.cpp` | 3 | **The runnable guard for the shutdown-handler guard in `patches/0006`/`0007`.** Runs the real `llama_server()` in router mode over an empty `--models-dir` on an ephemeral loopback port (no model, no worker) and stops it the way `native_server.cpp` does. Pins a clean stop (exit code 0), that `llama_server_request_shutdown()` **after** the server returned is a no-op (the deterministic form of the CI `SIGSEGV` — it crashed every run before the fix), and that requests hammered from another thread while the server tears down are safe (5 rounds). Compiled on non-Android only, like `server.cpp` itself. | -**Current total: 603 tests (all passing).** +**Current total: 607 tests (all passing).** #### Upstream source location (in CMake build tree) diff --git a/TODO.md b/TODO.md index 05f3a935c..7aba9ff38 100644 --- a/TODO.md +++ b/TODO.md @@ -324,12 +324,6 @@ round-trips — see CLAUDE.md "Two server modes"). **Owner priority: the native- `/infill` applies the model's FIM tokens server-side, so low value. - **Multi-model registry (Java transport).** The native surface has this via router mode + `RouterClient`; the Java `OpenAiCompatServer` still advertises/serves a single model id. -- **400 vs. 500 for an invalid request body.** Since llama.cpp b11337 (#29060) upstream's server - answers a malformed or empty embedding `"prompt"` (and any `common_json_error`) with 400. The JNI - layer throws a plain `LlamaException` for both, and `LlamaModelBackend` does not translate it into - the `IllegalArgumentException` that `completeNonStreaming` maps to 400, so `OpenAiCompatServer` - answers 500. A fix needs a typed signal from native (an invalid-request exception subclass, or the - `throw_invalid_request` JSON shape parsed on the Java side) rather than message matching. - **Manual real-client validation.** Server-side round-trips exist for every surface; what remains is pointing the actual editor clients (Copilot Ollama provider / Custom Endpoint, Claude Code, a Responses client) at a running server, since round-trips confirm wire shapes but not each client's diff --git a/llama/src/main/cpp/jllama.cpp b/llama/src/main/cpp/jllama.cpp index 1dd69fd8f..8f6d0f3d4 100644 --- a/llama/src/main/cpp/jllama.cpp +++ b/llama/src/main/cpp/jllama.cpp @@ -91,6 +91,7 @@ jclass c_integer = nullptr; jclass c_float = nullptr; jclass c_biconsumer = nullptr; jclass c_llama_error = nullptr; +jclass c_llama_invalid_request = nullptr; jclass c_log_level = nullptr; jclass c_log_format = nullptr; jclass c_error_oom = nullptr; @@ -145,8 +146,9 @@ jobject o_log_callback = nullptr; // within each table only matters for the human reader. // --------------------------------------------------------------------------- static jclass *const g_global_class_refs[] = { - &c_llama_model, &c_string, &c_hash_map, &c_map, &c_set, &c_entry, &c_iterator, - &c_integer, &c_float, &c_biconsumer, &c_llama_error, &c_log_level, &c_log_format, &c_error_oom, + &c_llama_model, &c_string, &c_hash_map, &c_map, &c_set, &c_entry, + &c_iterator, &c_integer, &c_float, &c_biconsumer, &c_llama_error, &c_llama_invalid_request, + &c_log_level, &c_log_format, &c_error_oom, }; static jobject *const g_global_object_refs[] = { @@ -203,11 +205,35 @@ static const static_object_binding g_static_object_bindings[] = { } /** - * Formats e as a JSON invalid-request error and throws it via JNI. + * Throws e as an InvalidRequestException: the request was rejected while it was parsed, which + * upstream's HTTP server answers with 400 (ERROR_TYPE_INVALID_REQUEST). The message is the bare + * reason, as upstream puts it into `error.message`; the type is the Java class. Request parsing is + * an invalid request whatever it throws (upstream's parsers mix std::invalid_argument and + * std::runtime_error for the same kind of defect); for the paths that run upstream code whose + * exception type carries the verdict, see throw_by_exception_type. */ static void throw_invalid_request(JNIEnv *env, const std::exception &e) { - const auto &err = format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST); - env->ThrowNew(c_llama_error, err.dump().c_str()); + env->ThrowNew(c_llama_invalid_request, e.what()); +} + +/** + * Throws e as the class upstream's `ex_wrapper` (tools/server/server.cpp) would answer with: + * std::invalid_argument and common_json_error are invalid requests (400), anything else is a server + * error (500). For tokenization, prompt formatting and the route handlers called directly. + */ +static void throw_by_exception_type(JNIEnv *env, const std::exception &e) { + const bool invalid = dynamic_cast(&e) != nullptr || + dynamic_cast(&e) != nullptr; + env->ThrowNew(invalid ? c_llama_invalid_request : c_llama_error, e.what()); +} + +/** + * The Java class for a failed task result: an error the server reports with code 400 + * (ERROR_TYPE_INVALID_REQUEST, ERROR_TYPE_EXCEED_CONTEXT_SIZE) is an InvalidRequestException, every + * other one a LlamaException. + */ +[[nodiscard]] static jclass error_class_for(const server_task_result_ptr &result) { + return get_result_error_code(result) == 400 ? c_llama_invalid_request : c_llama_error; } /** @@ -215,8 +241,12 @@ static void throw_invalid_request(JNIEnv *env, const std::exception &e) { * On failure throws via JNI and returns false. Callers must return immediately. */ [[nodiscard]] static bool result_ok_or_throw(JNIEnv *env, const server_task_result_ptr &result) { - if (!result || result->is_error()) { - env->ThrowNew(c_llama_error, result ? get_result_error_message(result).c_str() : "No result"); + if (!result) { + env->ThrowNew(c_llama_error, "No result"); + return false; + } + if (result->is_error()) { + env->ThrowNew(error_class_for(result), get_result_error_message(result).c_str()); return false; } return true; @@ -228,7 +258,7 @@ static void throw_invalid_request(JNIEnv *env, const std::exception &e) { */ [[nodiscard]] static bool batch_ok_or_throw(JNIEnv *env, const server_response_reader::batch_response &br) { if (br.error) { - env->ThrowNew(c_llama_error, get_result_error_message(br.error).c_str()); + env->ThrowNew(error_class_for(br.error), get_result_error_message(br.error).c_str()); return false; } if (br.is_terminated) { @@ -432,24 +462,26 @@ static void wake_and_post(server_response_reader &rd, std::vector & * function needs before it can read its arguments. */ // Parse the JSON request body. On malformed input, converts the C++ parse error into a Java -// LlamaException (so it never crosses the JNI boundary — which is undefined behavior) and returns -// false; callers must return their sentinel when this returns false. +// InvalidRequestException (upstream answers a common_json_error with 400) so it never crosses the +// JNI boundary — which is undefined behavior — and returns false; callers must return their sentinel +// when this returns false. [[nodiscard]] static bool parse_json_params(JNIEnv *env, jstring jparams, json &out) { try { out = json::parse(parse_jstring(env, jparams)); return true; } catch (const std::exception &e) { - env->ThrowNew(c_llama_error, e.what()); + throw_invalid_request(env, e); return false; } } /** * Convenience wrapper around require_json_field_impl (jni_helpers.hpp). - * Returns false and throws if `field` is absent from `data`. + * Returns false and throws an InvalidRequestException if `field` is absent from `data` (upstream: + * `"input_prefix" is required`, ERROR_TYPE_INVALID_REQUEST). */ [[nodiscard]] static bool require_json_field(JNIEnv *env, const json &data, const char *field) { - return require_json_field_impl(env, data, field, c_llama_error); + return require_json_field_impl(env, data, field, c_llama_invalid_request); } // Build a single indexed token task for batch submission (rerank and embedding). @@ -861,12 +893,14 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) try { c_float = env->FindClass("java/lang/Float"); c_biconsumer = env->FindClass("java/util/function/BiConsumer"); c_llama_error = env->FindClass("net/ladenthin/llama/exception/LlamaException"); + c_llama_invalid_request = env->FindClass("net/ladenthin/llama/exception/InvalidRequestException"); c_log_level = env->FindClass("net/ladenthin/llama/value/LogLevel"); c_log_format = env->FindClass("net/ladenthin/llama/args/LogFormat"); c_error_oom = env->FindClass("java/lang/OutOfMemoryError"); if (!(c_llama_model && c_standard_charsets && c_string && c_hash_map && c_map && c_set && c_entry && c_iterator && - c_integer && c_float && c_biconsumer && c_llama_error && c_log_level && c_log_format && c_error_oom)) { + c_integer && c_float && c_biconsumer && c_llama_error && c_llama_invalid_request && c_log_level && + c_log_format && c_error_oom)) { goto error; } @@ -1404,7 +1438,7 @@ JNIEXPORT jfloatArray JNICALL Java_net_ladenthin_llama_LlamaModel_embed(JNIEnv * try { tokens = tokenize_mixed(jctx->vocab, prompt, true, true); } catch (const std::exception &e) { - env->ThrowNew(c_llama_error, e.what()); + throw_by_exception_type(env, e); return nullptr; } auto rd = ctx_server->get_response_reader(); @@ -1504,9 +1538,18 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleSystemOne(JN const std::function should_stop = [jctx] { return jctx->closing.load(); }; const server_http_req req{{}, {}, "/v1/systemone", "", parse_jstring(env, jrequest), {}, should_stop}; - const server_http_res_ptr res = jctx->routes->post_systemone(req); + server_http_res_ptr res; + try { + res = jctx->routes->post_systemone(req); + } catch (const std::exception &e) { + // upstream's ex_wrapper: a std::invalid_argument / common_json_error out of a handler is a 400 + throw_by_exception_type(env, e); + return nullptr; + } if (res->status != 200) { - env->ThrowNew(c_llama_error, route_error_message(res->data).c_str()); + // the handler's own verdict: server_res_generator::error() sets the status from the error code + env->ThrowNew(res->status == 400 ? c_llama_invalid_request : c_llama_error, + route_error_message(res->data).c_str()); return nullptr; } return utf8_to_jstring(env, res->data); @@ -1876,7 +1919,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleInfill(JNIEn jctx->params.n_batch, jctx->params.n_predict, meta.slot_n_ctx, jctx->params.spm_infill, tokenized_prompts.empty() ? llama_tokens() : tokenized_prompts[0].get_tokens()); } catch (const std::exception &e) { - env->ThrowNew(c_llama_error, e.what()); + throw_by_exception_type(env, e); return nullptr; } @@ -1898,7 +1941,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleEmbeddings(J { auto meta = wake_and_get_meta(jctx); if (res_type != TASK_RESPONSE_TYPE_NONE && meta.pooling_type == LLAMA_POOLING_TYPE_NONE) { - env->ThrowNew(c_llama_error, + env->ThrowNew(c_llama_invalid_request, "Pooling type 'none' is not OAI compatible. Please use a different pooling type"); return nullptr; } @@ -1916,7 +1959,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleEmbeddings(J prompt = extract_embedding_prompt(body, force_no_oaicompat); use_base64 = parse_encoding_format(body); } catch (const std::exception &e) { - env->ThrowNew(c_llama_error, e.what()); + throw_invalid_request(env, e); return nullptr; } if (force_no_oaicompat) @@ -1927,13 +1970,14 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleEmbeddings(J tokenized_prompts = tokenize_input_prompts(jctx->vocab, nullptr, prompt, true, true, mtmd_helper_init_opt_default()); } catch (const std::exception &e) { - env->ThrowNew(c_llama_error, e.what()); + // `"prompt" must not be empty` is a std::invalid_argument, as upstream throws it + throw_by_exception_type(env, e); return nullptr; } for (const auto &toks : tokenized_prompts) { if (toks.get_tokens().empty()) { - env->ThrowNew(c_llama_error, "Input content cannot be empty"); + env->ThrowNew(c_llama_invalid_request, "Input content cannot be empty"); return nullptr; } } diff --git a/llama/src/main/cpp/json_helpers.hpp b/llama/src/main/cpp/json_helpers.hpp index 0c7d31a7c..edca72ef0 100644 --- a/llama/src/main/cpp/json_helpers.hpp +++ b/llama/src/main/cpp/json_helpers.hpp @@ -35,6 +35,7 @@ // 9. wrap_stream_chunk — used by nothing above it // 10. server_metrics_to_json — used by nothing above it // 11. route_error_message — used by nothing above it +// 12. get_result_error_code — used by nothing above it #include #include @@ -327,3 +328,21 @@ } return body; } + +// --------------------------------------------------------------------------- +// get_result_error_code +// +// The HTTP status code a failed task result carries: format_error_response() +// derives it from the result's error_type (400 for ERROR_TYPE_INVALID_REQUEST +// and ERROR_TYPE_EXCEED_CONTEXT_SIZE, 501 for ERROR_TYPE_NOT_SUPPORTED, 503 for +// ERROR_TYPE_UNAVAILABLE, 500 for ERROR_TYPE_SERVER). 0 when the result's JSON +// carries no integral "code" -- a result that is not an error. +// +// Used by result_ok_or_throw / batch_ok_or_throw in jllama.cpp to choose +// between InvalidRequestException (400) and LlamaException, so a task-level +// rejection (a prompt over the context size, a parameter the slot refuses) +// reaches the OpenAI-compatible server as the 400 upstream answers. +// --------------------------------------------------------------------------- +[[nodiscard]] inline int get_result_error_code(const server_task_result_ptr &result) { + return json_value(result->to_json(), "code", 0); +} diff --git a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java index 82756724a..937e6c8d9 100644 --- a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java +++ b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java @@ -399,9 +399,10 @@ public LlamaIterable generate(InferenceParameters parameters) { * * @param prompts the strings to embed; may be empty (returns an empty list) * @return one embedding vector per prompt, in the same order as {@code prompts} + * @throws net.ladenthin.llama.exception.InvalidRequestException if a prompt is rejected, e.g. because it tokenizes to nothing * @throws net.ladenthin.llama.exception.LlamaException if the model was not loaded for embeddings (see - * {@link net.ladenthin.llama.parameters.ModelParameters#enableEmbedding()}), a prompt is empty, or the - * native response cannot be parsed + * {@link net.ladenthin.llama.parameters.ModelParameters#enableEmbedding()}) or the native response cannot + * be parsed */ public List embed(java.util.Collection prompts) { if (prompts.isEmpty()) { @@ -613,8 +614,9 @@ public LlamaOutput rerank(String query, String... documents) { * * @param requestJson the request body as JSON * @return the response body as JSON - * @throws LlamaException when the model is not a decision model, the request is malformed, or it - * carries images the loaded model cannot take + * @throws net.ladenthin.llama.exception.InvalidRequestException when the request is malformed + * @throws LlamaException when the model is not a decision model or the request carries images the + * loaded model cannot take */ public native String handleSystemOne(String requestJson); @@ -653,6 +655,8 @@ public String applyTemplate(InferenceParameters parameters) { * * @param parameters the inference parameters including messages * @return the model's response as a JSON string containing the completion result + * @throws net.ladenthin.llama.exception.InvalidRequestException if the request is rejected (see + * {@link #handleChatCompletions(String)}) * @throws net.ladenthin.llama.exception.LlamaException if the model was loaded in embedding mode or if inference fails */ public String chatComplete(InferenceParameters parameters) { @@ -765,6 +769,9 @@ public LlamaIterable generateChat(InferenceParameters parameters) { * * @param parameters the inference parameters including messages (and optional tools) * @param chunkSink receiver for each {@code chat.completion.chunk} JSON string, in order + * @throws net.ladenthin.llama.exception.InvalidRequestException if the request is rejected (see + * {@link #handleChatCompletions(String)}); a prompt over the context size is reported this way too, on + * the first chunk * @throws net.ladenthin.llama.exception.LlamaException if inference fails */ public void streamChatCompletion(InferenceParameters parameters, Consumer chunkSink) { @@ -793,6 +800,9 @@ public void streamChatCompletion(InferenceParameters parameters, Consumer +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.exception; + +/** + * Thrown by the JNI layer when a request is rejected before or while it is validated — the + * cases llama.cpp's own HTTP server answers with status {@code 400} ({@code invalid_request_error}): + * a body that is not valid JSON, a missing or malformed field ({@code "prompt"}, {@code "messages"}, + * {@code "input_prefix"}, …), an empty embedding input, a parameter outside its hard limits, + * or a prompt that exceeds the context size. The message is the bare reason, as upstream puts it + * into {@code error.message}. + * + *

Everything else the native layer reports — a failed load, an operation the model was + * not loaded for, an inference failure — stays a plain {@link LlamaException}, which the + * OpenAI-compatible server ({@code net.ladenthin.llama.server.OpenAiCompatServer}) answers with + * {@code 500}. This subclass is what lets it answer {@code 400} instead, the way upstream does, + * without matching on message text.

+ */ +public class InvalidRequestException extends LlamaException { + + /** + * Creates a new {@link InvalidRequestException} with the given message. + * + * @param message the detail message; may be {@code null} + */ + public InvalidRequestException(String message) { + super(message); + } + + /** + * Creates a new {@link InvalidRequestException} with the given message and cause. + * + * @param message the detail message; may be {@code null} + * @param cause the underlying cause; may be {@code null} + */ + public InvalidRequestException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/server/LlamaModelBackend.java b/llama/src/main/java/net/ladenthin/llama/server/LlamaModelBackend.java index 2075f61fe..3d47cacd5 100644 --- a/llama/src/main/java/net/ladenthin/llama/server/LlamaModelBackend.java +++ b/llama/src/main/java/net/ladenthin/llama/server/LlamaModelBackend.java @@ -27,6 +27,12 @@ * model API takes a {@link java.util.function.Consumer} (no checked exceptions), that failure is * relayed across the boundary via {@link java.io.UncheckedIOException} and unwrapped here so the * in-flight native task is cancelled. + * + *

A request the native layer rejects -- a body that is not valid JSON, a missing field, an empty + * embedding input, a prompt over the context size -- arrives as + * {@link net.ladenthin.llama.exception.InvalidRequestException} and is left untouched: the server maps + * that type to {@code 400}, as upstream's HTTP server does for the same failures. Every other + * {@link net.ladenthin.llama.exception.LlamaException} stays a {@code 500}. */ final class LlamaModelBackend implements OpenAiBackend { diff --git a/llama/src/main/java/net/ladenthin/llama/server/OpenAiBackend.java b/llama/src/main/java/net/ladenthin/llama/server/OpenAiBackend.java index cb5486234..335353647 100644 --- a/llama/src/main/java/net/ladenthin/llama/server/OpenAiBackend.java +++ b/llama/src/main/java/net/ladenthin/llama/server/OpenAiBackend.java @@ -20,6 +20,14 @@ * incrementally. The {@code GET /v1/models} response is built from configuration alone and so is not * part of this seam. * + *

Errors. An {@link IllegalArgumentException} or a + * {@link net.ladenthin.llama.exception.InvalidRequestException} thrown by any method is answered as the + * client's fault: {@code 400} with {@code "type":"invalid_request_error"}, or the same error object as + * an SSE {@code data:} event once a stream has started. Every other exception is {@code 500} + * ({@code "type":"server_error"}). The production backend lets the native layer's typed exception + * through untouched; it is thrown exactly where llama.cpp's own server would answer {@code 400}, so + * the two transports agree without matching on message text. + * *

Public so that sibling modules can drive the real HTTP surface with a scripted backend — the * {@code llama-atmosphere-agent} wire-contract tests replay llama.cpp-shaped * {@code chat.completion.chunk} sequences through {@link OpenAiCompatServer} to prove an OpenAI client's diff --git a/llama/src/main/java/net/ladenthin/llama/server/OpenAiCompatServer.java b/llama/src/main/java/net/ladenthin/llama/server/OpenAiCompatServer.java index efa75be7e..e74951500 100644 --- a/llama/src/main/java/net/ladenthin/llama/server/OpenAiCompatServer.java +++ b/llama/src/main/java/net/ladenthin/llama/server/OpenAiCompatServer.java @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import net.ladenthin.llama.LlamaModel; +import net.ladenthin.llama.exception.InvalidRequestException; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -329,7 +330,7 @@ private void streamCompletions(HttpExchange exchange, JsonNode request) throws I TimeUnit.MILLISECONDS); backend.streamCompletions(request, chunkJson -> out.writeStrict(OpenAiSseFormatter.sseData(chunkJson))); out.writeStrict(OpenAiSseFormatter.sseDone()); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { out.writeQuietly( OpenAiSseFormatter.sseData(OpenAiSseFormatter.errorJson(message(e), ERROR_TYPE_REQUEST, null))); } catch (IOException e) { @@ -381,14 +382,17 @@ private void handleRerank(HttpExchange exchange) throws IOException { /** * Run a non-streaming request through {@code producer} and write its JSON body, translating an - * {@link IllegalArgumentException} to {@code 400} and any other failure to {@code 500}. + * {@link IllegalArgumentException} (a request this layer rejected) or an + * {@link InvalidRequestException} (one the native layer rejected, where upstream's server answers + * {@code 400}) to {@code 400} and any other failure to {@code 500}. The streaming routes make the + * same distinction for the error event they write once the headers are out. */ private void completeNonStreaming(HttpExchange exchange, JsonNode request, BodyProducer producer) throws IOException { final String body; try { body = producer.produce(request); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { sendError(exchange, HTTP_BAD_REQUEST, ERROR_TYPE_REQUEST, message(e)); return; } catch (IOException | RuntimeException e) { @@ -416,7 +420,7 @@ private void streamChat(HttpExchange exchange, JsonNode request) throws IOExcept chunkJson -> out.writeStrict( OpenAiSseFormatter.sseData(OpenAiSseFormatter.ensureUsageCachedTokens(chunkJson)))); out.writeStrict(OpenAiSseFormatter.sseDone()); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { out.writeQuietly( OpenAiSseFormatter.sseData(OpenAiSseFormatter.errorJson(message(e), ERROR_TYPE_REQUEST, null))); } catch (IOException e) { @@ -492,7 +496,7 @@ private void handleOllamaChat(HttpExchange exchange) throws IOException { final String body; try { body = backend.complete(openAiRequest); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { sendJson(exchange, HTTP_BAD_REQUEST, ollamaError(message(e))); return; } catch (IOException | RuntimeException e) { @@ -523,7 +527,7 @@ private void streamOllamaChat(HttpExchange exchange, JsonNode openAiRequest, Str } }); out.writeStrict(OllamaApiSupport.toOllamaDoneLine(model, accumulator)); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { out.writeQuietly(ollamaError(message(e)) + "\n"); } catch (IOException e) { LOG.debug("ollama client disconnected during stream", e); @@ -552,7 +556,7 @@ private void handleOllamaGenerate(HttpExchange exchange) throws IOException { text = OllamaApiSupport.extractCompletionText( backend.completions(OllamaApiSupport.toOpenAiCompletionRequest(request))); } - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { sendJson(exchange, HTTP_BAD_REQUEST, ollamaError(message(e))); return; } catch (IOException | RuntimeException e) { @@ -596,7 +600,7 @@ private void handleAnthropicMessages(HttpExchange exchange) throws IOException { final String body; try { body = backend.complete(openAiRequest); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { sendJson(exchange, HTTP_BAD_REQUEST, anthropicError(message(e))); return; } catch (IOException | RuntimeException e) { @@ -634,7 +638,7 @@ private void streamAnthropic(HttpExchange exchange, JsonNode openAiRequest, Stri } }); out.writeStrict(translator.end()); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { out.writeQuietly(AnthropicApiSupport.sseEvent("error", anthropicError(message(e)))); } catch (IOException e) { LOG.debug("anthropic client disconnected during stream", e); @@ -684,7 +688,7 @@ private void handleResponses(HttpExchange exchange) throws IOException { final String body; try { body = backend.complete(openAiRequest); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { sendError(exchange, HTTP_BAD_REQUEST, ERROR_TYPE_REQUEST, message(e)); return; } catch (IOException | RuntimeException e) { @@ -722,7 +726,7 @@ private void streamResponses(HttpExchange exchange, JsonNode openAiRequest, Stri } }); out.writeStrict(translator.end()); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidRequestException e) { out.writeQuietly("event: error\ndata: " + OpenAiSseFormatter.errorJson(message(e), ERROR_TYPE_REQUEST, null) + "\n\n"); } catch (IOException e) { diff --git a/llama/src/test/cpp/test_json_helpers.cpp b/llama/src/test/cpp/test_json_helpers.cpp index cfbf11a71..33148a0d8 100644 --- a/llama/src/test/cpp/test_json_helpers.cpp +++ b/llama/src/test/cpp/test_json_helpers.cpp @@ -681,3 +681,38 @@ TEST(RouteErrorMessage, ReturnsJsonWithoutAnErrorMessageUnchanged) { EXPECT_EQ(route_error_message(R"({"error":"plain"})"), R"({"error":"plain"})"); EXPECT_EQ(route_error_message(R"({"error":{"message":42}})"), R"({"error":{"message":42}})"); } + +// ============================================================ +// get_result_error_code +// ============================================================ + +namespace { + +// An error result of a given type: to_json() → format_error_response() maps the type to the code. +static server_task_result_ptr make_error_of_type(int id_, const std::string &msg, error_type type) { + auto r = std::make_unique(); + r->id = id_; + r->err_msg = msg; + r->err_type = type; + return r; +} + +} // namespace + +TEST(GetResultErrorCode, AServerErrorIs500) { EXPECT_EQ(get_result_error_code(make_error(1, "boom")), 500); } + +TEST(GetResultErrorCode, AnInvalidRequestAndAnExceededContextAre400) { + EXPECT_EQ(get_result_error_code(make_error_of_type(2, "bad", ERROR_TYPE_INVALID_REQUEST)), 400); + EXPECT_EQ(get_result_error_code(make_error_of_type(3, "too long", ERROR_TYPE_EXCEED_CONTEXT_SIZE)), 400); +} + +TEST(GetResultErrorCode, TheOtherTypesKeepUpstreamsCodes) { + EXPECT_EQ(get_result_error_code(make_error_of_type(4, "no", ERROR_TYPE_NOT_SUPPORTED)), 501); + EXPECT_EQ(get_result_error_code(make_error_of_type(5, "gone", ERROR_TYPE_UNAVAILABLE)), 503); + EXPECT_EQ(get_result_error_code(make_error_of_type(6, "who", ERROR_TYPE_AUTHENTICATION)), 401); +} + +TEST(GetResultErrorCode, AResultWithoutACodeIsZero) { + EXPECT_EQ(get_result_error_code(make_ok(7, "fine")), 0); + EXPECT_EQ(get_result_error_code(make_embedding(8)), 0); +} diff --git a/llama/src/test/java/net/ladenthin/llama/ErrorHandlingTest.java b/llama/src/test/java/net/ladenthin/llama/ErrorHandlingTest.java index b7a3e77a4..701f40dfa 100644 --- a/llama/src/test/java/net/ladenthin/llama/ErrorHandlingTest.java +++ b/llama/src/test/java/net/ladenthin/llama/ErrorHandlingTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.*; import java.io.File; +import net.ladenthin.llama.exception.InvalidRequestException; import net.ladenthin.llama.exception.LlamaException; import net.ladenthin.llama.parameters.ModelParameters; import org.junit.jupiter.api.AfterAll; @@ -120,8 +121,9 @@ public void testHandleEmbeddingsInvalidEncodingFormat() { String json = "{\"input\":\"hello world\",\"encoding_format\":\"invalid\"}"; try { String result = model.handleEmbeddings(json, false); - fail("Expected LlamaException for invalid encoding_format"); - } catch (LlamaException e) { + fail("Expected InvalidRequestException for invalid encoding_format"); + } catch (InvalidRequestException e) { + // the typed rejection is what lets OpenAiCompatServer answer 400 instead of 500 assertTrue(e.getMessage().contains("encoding_format"), "Error should mention encoding_format"); } } @@ -137,8 +139,36 @@ public void testHandleEmbeddingsEmptyInput() { String result = model.handleEmbeddings(json, false); // Native code may handle empty input gracefully — that's acceptable assertNotNull(result, "Result should not be null"); - } catch (LlamaException e) { - // Also acceptable if the native code rejects empty input + } catch (InvalidRequestException e) { + // Also acceptable if the native code rejects empty input -- but then as the typed rejection + assertNotNull(e.getMessage(), "Exception message should not be null"); + } + } + + @Test + public void testHandleEmbeddingsEmptyInputArrayIsAnInvalidRequest() { + // upstream tokenize_input_prompts: `"prompt" must not be empty` (std::invalid_argument), a 400 + try { + model.handleEmbeddings("{\"input\":[]}", false); + fail("Expected InvalidRequestException for an empty input array"); + } catch (InvalidRequestException e) { + assertTrue(e.getMessage().contains("must not be empty"), "Error should name the reason: " + e.getMessage()); + } + } + + @Test + public void testMalformedJsonBodyIsAnInvalidRequest() { + // upstream answers a common_json_error with 400; the JNI layer parses the body the same way + try { + model.handleEmbeddings("{not json", false); + fail("Expected InvalidRequestException for a malformed body"); + } catch (InvalidRequestException e) { + assertNotNull(e.getMessage(), "Exception message should not be null"); + } + try { + model.handleCompletions("[1, 2"); + fail("Expected InvalidRequestException for a malformed body"); + } catch (InvalidRequestException e) { assertNotNull(e.getMessage(), "Exception message should not be null"); } } @@ -165,7 +195,8 @@ public void testHandleInfillMissingInputSuffix() { try { model.handleInfill(json); // May succeed with empty suffix or throw - } catch (LlamaException e) { + } catch (InvalidRequestException e) { + // a missing required field is upstream's `"input_suffix" is required`, a 400 assertTrue(e.getMessage().contains("input_suffix"), "Error should mention input_suffix"); } } @@ -217,11 +248,11 @@ public void testConfigureParallelInferenceZeroNThreads() { @Test public void testHandleCompletionsMissingPromptThrows() { // No "prompt" key → data.at("prompt") throws json::out_of_range → - // caught by the std::exception catch → throw_invalid_request → LlamaException + // caught by the std::exception catch → throw_invalid_request → InvalidRequestException try { model.handleCompletions("{\"n_predict\":1}"); - fail("Expected LlamaException for missing 'prompt' key"); - } catch (LlamaException e) { + fail("Expected InvalidRequestException for missing 'prompt' key"); + } catch (InvalidRequestException e) { assertNotNull(e.getMessage(), "Exception message must not be null"); } } @@ -230,8 +261,8 @@ public void testHandleCompletionsMissingPromptThrows() { public void testHandleCompletionsOaiMissingPromptThrows() { try { model.handleCompletionsOai("{\"n_predict\":1}"); - fail("Expected LlamaException for missing 'prompt' key"); - } catch (LlamaException e) { + fail("Expected InvalidRequestException for missing 'prompt' key"); + } catch (InvalidRequestException e) { assertNotNull(e.getMessage(), "Exception message must not be null"); } } diff --git a/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java b/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java index 68f8e8f65..ae337602f 100644 --- a/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java +++ b/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java @@ -101,7 +101,9 @@ public class LlamaArchitectureTest { * optional OpenAI-compatible HTTP / native-server entry point; it is the only layer permitted to * access the {@code Api} root, and it also reaches the {@code Loader} ({@code NativeServer} * triggers {@code LlamaLoader.initialize()} before starting the embedded native server) and the - * {@code Args} enums ({@code OpenAiServerCli} maps {@code -ctk}/{@code -ctv} to {@code CacheType}). + * {@code Args} enums ({@code OpenAiServerCli} maps {@code -ctk}/{@code -ctv} to {@code CacheType}), and + * the {@code Exception} layer ({@code OpenAiCompatServer} answers an {@code InvalidRequestException} + * with {@code 400}). */ @ArchTest static final ArchRule layeredArchitecture = layeredArchitecture() @@ -139,7 +141,7 @@ public class LlamaArchitectureTest { .whereLayer("Callback") .mayOnlyBeAccessedByLayers("Api") .whereLayer("Exception") - .mayOnlyBeAccessedByLayers("Api", "Loader") + .mayOnlyBeAccessedByLayers("Api", "Loader", "Server") .whereLayer("Args") .mayOnlyBeAccessedByLayers("Api", "Loader", "Parameters", "Server") .whereLayer("Server") diff --git a/llama/src/test/java/net/ladenthin/llama/server/OpenAiCompatServerHttpTest.java b/llama/src/test/java/net/ladenthin/llama/server/OpenAiCompatServerHttpTest.java index e3d121499..2ce0f5220 100644 --- a/llama/src/test/java/net/ladenthin/llama/server/OpenAiCompatServerHttpTest.java +++ b/llama/src/test/java/net/ladenthin/llama/server/OpenAiCompatServerHttpTest.java @@ -10,6 +10,8 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; +import net.ladenthin.llama.exception.InvalidRequestException; +import net.ladenthin.llama.exception.LlamaException; import org.junit.jupiter.api.Test; /** @@ -362,6 +364,74 @@ public void malformedJsonReturns400() throws IOException { } } + // ------------------------------------------------------------------------- + // A request the NATIVE layer rejects (InvalidRequestException, thrown by the JNI layer where + // llama.cpp's own server answers 400) is the client's fault on every surface; a plain + // LlamaException stays a server error. Pins the fix for the "400 vs. 500" entry of TODO.md. + // ------------------------------------------------------------------------- + + @Test + public void nativeRejectionOfAChatCompletionReturns400() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new RejectingBackend(), config()).start()) { + Response response = post(server.getPort(), "/v1/chat/completions", CHAT_BODY, ""); + assertThat(response.code, is(400)); + assertThat(response.body, containsString("\"type\":\"invalid_request_error\"")); + assertThat(response.body, containsString(RejectingBackend.REASON_FRAGMENT)); + } + } + + @Test + public void nativeRejectionOfAnEmbeddingReturns400() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new RejectingBackend(), config()).start()) { + Response response = post(server.getPort(), "/v1/embeddings", "{\"input\":[]}", ""); + assertThat(response.code, is(400)); + assertThat(response.body, containsString("\"type\":\"invalid_request_error\"")); + } + } + + @Test + public void nativeRejectionOfACompletionReturns400() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new RejectingBackend(), config()).start()) { + Response response = post(server.getPort(), "/v1/completions", "{\"prompt\":\"\"}", ""); + assertThat(response.code, is(400)); + assertThat(response.body, containsString("\"type\":\"invalid_request_error\"")); + } + } + + @Test + public void nativeRejectionOfAStreamedChatCompletionIsAnInvalidRequestEvent() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new RejectingBackend(), config()).start()) { + String body = "{\"stream\":true,\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}"; + Response response = post(server.getPort(), "/v1/chat/completions", body, ""); + // the headers are out before the backend runs, so the verdict travels as an SSE data event + assertThat(response.code, is(200)); + assertThat(response.body, containsString("data: ")); + assertThat(response.body, containsString("\"type\":\"invalid_request_error\"")); + assertThat(response.body, containsString(RejectingBackend.REASON_FRAGMENT)); + } + } + + @Test + public void nativeRejectionOnTheOllamaAndAnthropicSurfacesReturns400() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new RejectingBackend(), config()).start()) { + String ollama = "{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]," + + "\"stream\":false}"; + assertThat(post(server.getPort(), "/api/chat", ollama, "").code, is(400)); + String anthropic = "{\"model\":\"test-model\",\"max_tokens\":8," + + "\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}"; + assertThat(post(server.getPort(), "/v1/messages", anthropic, "").code, is(400)); + } + } + + @Test + public void aPlainLlamaExceptionStaysAServerError() throws IOException { + try (OpenAiCompatServer server = new OpenAiCompatServer(new FailingBackend(), config()).start()) { + Response response = post(server.getPort(), "/v1/chat/completions", CHAT_BODY, ""); + assertThat(response.code, is(500)); + assertThat(response.body, containsString("\"type\":\"server_error\"")); + } + } + @Test public void getOnChatCompletionsReturns405() throws IOException { try (OpenAiCompatServer server = new OpenAiCompatServer(new FakeBackend(), config()).start()) { @@ -511,6 +581,44 @@ public String rerank(JsonNode request) { } } + /** + * Backend whose every operation is rejected the way the JNI layer rejects a request llama.cpp's + * validation refuses: an {@link InvalidRequestException} with the bare upstream reason. + */ + static final class RejectingBackend extends FakeBackend { + static final String REASON = "\"prompt\" must not be empty"; + /** The part of {@link #REASON} without the quotes, which the error JSON escapes. */ + static final String REASON_FRAGMENT = "must not be empty"; + + @Override + public String complete(JsonNode request) { + throw new InvalidRequestException(REASON); + } + + @Override + public void stream(JsonNode request, ChunkSink sink) { + throw new InvalidRequestException(REASON); + } + + @Override + public String completions(JsonNode request) { + throw new InvalidRequestException(REASON); + } + + @Override + public String embeddings(JsonNode request) { + throw new InvalidRequestException(REASON); + } + } + + /** Backend whose chat completion fails like an inference failure: a plain {@link LlamaException}. */ + static final class FailingBackend extends FakeBackend { + @Override + public String complete(JsonNode request) { + throw new LlamaException("decoding failed"); + } + } + /** Backend that stalls before emitting, so the server's heartbeat fires during the gap. */ static final class SlowFakeBackend implements OpenAiBackend { @Override diff --git a/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerCompletionIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerCompletionIntegrationTest.java index c5601acfb..de607bd1a 100644 --- a/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerCompletionIntegrationTest.java +++ b/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerCompletionIntegrationTest.java @@ -73,6 +73,15 @@ public void completionsReturnsTextChoice() throws IOException { assertThat(json.path("choices").get(0).path("text").isTextual(), is(true)); } + @Test + public void aCompletionWithoutAPromptIsAnswered400LikeUpstream() throws IOException { + // oaicompat_completion_params_parse refuses the body natively; the typed rejection reaches the + // server as InvalidRequestException and is a 400, not the 500 it used to be + Response response = post(port, "/v1/completions", "{\"model\":\"" + MODEL_ID + "\",\"max_tokens\":1}", ""); + assertThat(response.code, is(400)); + assertThat(MAPPER.readTree(response.body).path("error").path("type").asText(), is("invalid_request_error")); + } + @Test public void infillReturnsContent() throws IOException { String body = "{\"input_prefix\":\"def add(a, b):\\n return \",\"input_suffix\":\"\\n\",\"n_predict\":16}"; diff --git a/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerEmbeddingsIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerEmbeddingsIntegrationTest.java index 9bebae78f..55babab27 100644 --- a/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerEmbeddingsIntegrationTest.java +++ b/llama/src/test/java/net/ladenthin/llama/server/OpenAiServerEmbeddingsIntegrationTest.java @@ -5,6 +5,7 @@ package net.ladenthin.llama.server; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -85,6 +86,26 @@ public void embeddingsReturnsAVector() throws IOException { assertThat(first.path("embedding").size(), greaterThan(0)); } + @Test + public void anEmptyInputArrayIsAnswered400LikeUpstream() throws IOException { + // llama.cpp's own server answers this with 400 since b11337 (#29060); the JNI layer throws the + // typed InvalidRequestException for it, which the server maps to 400 rather than 500 + String body = "{\"model\":\"" + MODEL_ID + "\",\"input\":[]}"; + Response response = post(port, "/v1/embeddings", body, ""); + assertThat(response.code, is(400)); + JsonNode error = MAPPER.readTree(response.body).path("error"); + assertThat(error.path("type").asText(), is("invalid_request_error")); + assertThat(error.path("message").asText(), containsString("must not be empty")); + } + + @Test + public void anUnknownEncodingFormatIsAnswered400() throws IOException { + String body = "{\"model\":\"" + MODEL_ID + "\",\"input\":\"hello\",\"encoding_format\":\"hex\"}"; + Response response = post(port, "/v1/embeddings", body, ""); + assertThat(response.code, is(400)); + assertThat(MAPPER.readTree(response.body).path("error").path("type").asText(), is("invalid_request_error")); + } + @Test public void embeddingsReachableWithoutV1Prefix() throws IOException { String body = "{\"model\":\"" + MODEL_ID + "\",\"input\":\"alias check\"}";