From 5683571757d352ac78263d3e63351c3d015ec8f9 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 7 Nov 2026 01:42:09 -0600 Subject: [PATCH 1/4] server: test faults for a failed prompt fill (red on 444f386cf) LLAMA_SERVER_TEST_FAIL_CHECKPOINTS fails the next N prompt checkpoint copies with std::bad_alloc at the point the copy allocates; LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS throws from the next N prompt fills after they add their tokens. Both are zero unless set. test_failed_fill.py reproduces the Windows node's abort (2026-11-07): two bad_alloc on slot 0 then slot 1, both released, then GGML_ASSERT(batch.slot_batched || batch.size() == 0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 14 ++++ tools/server/server-test-faults.h | 33 +++++++++ tools/server/tests/unit/test_failed_fill.py | 75 +++++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tools/server/server-test-faults.h create mode 100644 tools/server/tests/unit/test_failed_fill.py diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e80169b34519..c5eeb15587a6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -11,6 +11,7 @@ #include "server-stream.h" #include "server-page-adapters.h" #include "server-prefill.h" +#include "server-test-faults.h" #include "build-info.h" #include "common.h" @@ -943,6 +944,7 @@ struct server_context_impl { int trace = 0; // env: LLAMA_TRACE int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF + server_test_faults test_faults; // env: LLAMA_SERVER_TEST_FAIL_* int n_empty_consecutive = 0; @@ -1340,6 +1342,11 @@ struct server_context_impl { } } + test_faults = server_test_faults::from_env(); + if (test_faults.checkpoint_copies || test_faults.prompt_fills) { + SRV_WRN("test faults: checkpoint copies = %d, prompt fills = %d\n", test_faults.checkpoint_copies, test_faults.prompt_fills); + } + // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) { @@ -2522,6 +2529,9 @@ struct server_context_impl { // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); + if (server_test_faults::take(test_faults.checkpoint_copies)) { + throw std::bad_alloc(); // where the copy below allocates + } cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); // stash the draft's speculative state with the checkpoint @@ -4245,6 +4255,10 @@ struct server_context_impl { // the number of tokens added to the batch for the current slot const auto n_tokens_cur = batch.size() - n_tokens_prev; + if (n_tokens_cur > 0 && server_test_faults::take(test_faults.prompt_fills)) { + throw std::runtime_error("test fault: the prompt fill failed after adding its tokens"); + } + const auto n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; diff --git a/tools/server/server-test-faults.h b/tools/server/server-test-faults.h new file mode 100644 index 000000000000..573abe969e46 --- /dev/null +++ b/tools/server/server-test-faults.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +// Continuum fork: failures the server tests inject through the environment, so a test can show +// that a failure leaves the server consistent. Every count is zero unless its variable is set. +struct server_test_faults { + int32_t checkpoint_copies = 0; // LLAMA_SERVER_TEST_FAIL_CHECKPOINTS: the next N prompt checkpoint copies fail to allocate + int32_t prompt_fills = 0; // LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS: the next N prompt fills throw after adding their tokens + + static server_test_faults from_env() { + server_test_faults faults; + faults.checkpoint_copies = read("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS"); + faults.prompt_fills = read("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS"); + return faults; + } + + // true when this occurrence is one to fail, counting it + static bool take(int32_t & remaining) { + if (remaining <= 0) { + return false; + } + --remaining; + return true; + } + +private: + static int32_t read(const char * name) { + const char * value = std::getenv(name); + return value ? std::atoi(value) : 0; + } +}; diff --git a/tools/server/tests/unit/test_failed_fill.py b/tools/server/tests/unit/test_failed_fill.py new file mode 100644 index 000000000000..fd69a5c9d75f --- /dev/null +++ b/tools/server/tests/unit/test_failed_fill.py @@ -0,0 +1,75 @@ +import pytest +from utils import * + +# A slot whose prompt fill fails must leave the batch and its own prompt as it found them: the +# other slot completes, the server stays up, and the failed slot's next request trusts only the +# tokens its memory really holds. The faults are injected through LLAMA_SERVER_TEST_FAIL_*. + +server: ServerProcess + +PROMPT = ( + "Once upon a time in a land far away, there lived a brave knight " + "who traveled across mountains and rivers to find the legendary " + "golden sword hidden deep within the enchanted forest of whispers." +) + + +def complete(id_slot: int): + return server.make_request("POST", "/completion", data={ + "prompt": PROMPT, + "id_slot": id_slot, + "n_predict": 8, + "temperature": 0.0, + "cache_prompt": True, + }) + + +def complete_both(): + return parallel_function_calls([(complete, (0,)), (complete, (1,))]) + + +def configure(preset: ServerProcess) -> ServerProcess: + preset.n_slots = 2 + preset.n_predict = 8 + preset.temperature = 0.0 + preset.cache_ram = 0 # a retry must not be served from another slot's saved prompt + return preset + + +# what this catches: a checkpoint copy that cannot be allocated failed its slot after the slot had +# added its tokens to the batch; when every slot in the fill failed, the next decode found tokens +# with no batched slot and aborted (GGML_ASSERT(batch.slot_batched || batch.size() == 0)). +# A refused copy only makes a later restore cold, so both slots must complete. +def test_a_refused_checkpoint_copy_is_not_a_slot_failure(monkeypatch): + global server + server = configure(ServerPreset.tinygemma3()) # sliding-window attention: its prompts checkpoint + monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS", "2") # one copy per slot, as on the node + server.start() + + first, second = complete_both() + assert first.status_code == 200, first.body + assert second.status_code == 200, second.body + assert first.body["content"] == second.body["content"] + assert server.make_request("GET", "/health").status_code == 200 + + +# what this catches: a slot whose fill threw kept its tokens in the batch (no slot owning them) and +# in its prompt (claiming memory that was never written), so the next request with that prefix +# reused cells that do not exist. +def test_a_failed_fill_leaves_the_batch_and_its_prompt_clean(monkeypatch): + global server + server = configure(ServerPreset.tinyllama2()) + monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS", "1") + server.start() + + results = complete_both() + failed = [i for i, res in enumerate(results) if res.status_code != 200] + assert len(failed) == 1, [res.body for res in results] + good = results[1 - failed[0]] + assert server.make_request("GET", "/health").status_code == 200 + + # the failed slot's memory holds none of the prompt, so nothing of it may be reused + retry = complete(failed[0]) + assert retry.status_code == 200, retry.body + assert retry.body["timings"]["cache_n"] == 0 + assert retry.body["content"] == good.body["content"] From eae7df7d7e11274fd92078d7bed3cd72ebec2c55 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 7 Nov 2026 01:46:34 -0600 Subject: [PATCH 2/4] server: a failed prompt fill leaves the batch and its prompt clean A slot's prompt fill added its tokens to the batch and to slot.prompt.tokens, then create_checkpoint() allocated the state copy; a std::bad_alloc there reached iterate(), which released the slot. Its tokens stayed in the batch with no slot_batched owner (when every filling slot failed, the next decode aborted on GGML_ASSERT(batch.slot_batched || batch.size() == 0)), and its prompt kept tokens whose memory was never written, so the next request with that prefix reused cells that do not exist. - server_batch::get_mark()/rollback(): the batch's state before a slot adds to it, restored when its fill throws. - The prompt fill is one lambda; its caller rolls back the batch and the slot's prompt to the tokens whose memory is written, then rethrows to iterate(), which reports the error and releases the slot as before. - create_checkpoint() builds the copy aside and keeps it only when complete; a copy that cannot be allocated (std::bad_alloc) is refused and logged, and the fill continues: a later restore from there is cold. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 74 ++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c5eeb15587a6..7b965b708c80 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -172,6 +172,32 @@ struct server_batch { return (int32_t)tokens.size(); } + // what a slot finds in the batch before it adds to it + struct mark { + int32_t n_tokens; + size_t n_embd; + bool has_embd; + server_slot * slot_batched; + float alora_scale; + size_t alora_disabled_id; + }; + + mark get_mark() const { + return { size(), embd.size(), has_embd, slot_batched, alora_scale, alora_disabled_id }; + } + + // undo everything added since the mark, so a slot that fails leaves the batch as it found it + void rollback(const mark & m) { + GGML_ASSERT(!batch_rendered); + GGML_ASSERT(m.n_tokens <= size() && m.n_embd <= embd.size()); + tokens.resize(m.n_tokens); + embd.resize(m.n_embd); + has_embd = m.has_embd; + slot_batched = m.slot_batched; + alora_scale = m.alora_scale; + alora_disabled_id = m.alora_disabled_id; + } + void set_output(int32_t idx, bool output) { GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size()); tokens[idx].output = output; @@ -2520,7 +2546,8 @@ struct server_context_impl { slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); } - auto & cur = slot.prompt.checkpoints.emplace_back(); + // the copy is made aside and kept only once complete + common_prompt_checkpoint cur; cur.id_task = id_task; @@ -2529,18 +2556,26 @@ struct server_context_impl { // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); - if (server_test_faults::take(test_faults.checkpoint_copies)) { - throw std::bad_alloc(); // where the copy below allocates + // a copy that cannot be allocated only makes a later restore from here cold + try { + if (server_test_faults::take(test_faults.checkpoint_copies)) { + throw std::bad_alloc(); // where the copy below allocates + } + cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // stash the draft's speculative state with the checkpoint + common_speculative_get_state(spec.get(), slot.id, cur.data_spec); + } catch (const std::bad_alloc &) { + SLT_WRN(slot, "context checkpoint refused: its copy could not be allocated (n_tokens = %" PRId64 "), a restore from here will be cold\n", cur.n_tokens); + return; } - cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - // stash the draft's speculative state with the checkpoint - common_speculative_get_state(spec.get(), slot.id, cur.data_spec); + + const auto & kept = slot.prompt.checkpoints.emplace_back(std::move(cur)); SLT_TRC(slot, "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", - (int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min, - cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); + (int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, kept.pos_min, + kept.pos_max, kept.n_tokens, (float) kept.size() / 1024 / 1024); } // returns false to decline the task, it is offered again after the decode is done @@ -3733,7 +3768,9 @@ struct server_context_impl { for (const size_t index : prefill_order.order(slots.size(), params_base.kv_unified)) { prompt_slots.push_back(&slots[index]); } - iterate(prompt_slots, [&](server_slot & slot) { + // n_written: the slot's prompt tokens whose memory is written, set once the fill + // starts adding tokens that are only in the batch (not decoded yet) + const auto fill_prompt = [&](server_slot & slot, int32_t & n_written) { const auto batch_before = batch.size(); if (!add_ok || batch.size() >= n_batch) { return; // batch is full, skip remaining slots @@ -4186,6 +4223,7 @@ struct server_context_impl { slot.prompt.end_from = std::max(0, slot.task->n_tokens() - std::min(n_batch, 4 + n_ubatch)); // add prompt tokens for processing in the current batch + n_written = slot.prompt.n_tokens(); while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) { // get next token to process llama_token cur_tok = input_tokens[slot.prompt.n_tokens()]; @@ -4318,6 +4356,22 @@ struct server_context_impl { if (!slot_batched) { slot_batched = &slot; } + }; + + // a slot whose fill fails leaves the batch and its prompt as it found them: its + // tokens were never decoded, so no batched slot owns them and its memory lacks them + iterate(prompt_slots, [&](server_slot & slot) { + const auto batch_mark = batch.get_mark(); + int32_t n_written = -1; + try { + fill_prompt(slot, n_written); + } catch (...) { + batch.rollback(batch_mark); + if (n_written >= 0) { + slot.prompt.tokens.keep_first(n_written); + } + throw; // iterate() reports the error and releases the slot + } }); } } From eaacc70d8602f2c923f934127340ea32a8981f54 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 7 Nov 2026 02:00:15 -0600 Subject: [PATCH 3/4] server: one owner for checkpoint copies; a refused speculative copy drops the draft The speculative checkpoint (taken before a draft is verified, and on the draft context when a draft starts) allocated the same kind of state copy outside any guard: a std::bad_alloc there released a generating slot mid-reply, and the slot was still in `generating`, so handle_last_sampled_token() then aborted on GGML_ASSERT(task) (server-context.cpp:446). copy_checkpoint() is now the one place a checkpoint's state copy is allocated, for prompt and speculative checkpoints alike. A copy that cannot be allocated is refused by type (std::bad_alloc): the checkpoint is cleared, the refusal is logged, and the caller goes on without it. A prompt checkpoint's restore is cold; a speculative checkpoint's draft is dropped (or not started), so the step decodes without it and the reply is unchanged. Dropped drafts are not counted in n_draft_tokens. LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS injects the speculative refusal; the test generates on mamba-130m (recurrent: every verified draft is checkpointed) with ngram-simple and checks the faulted reply's tokens equal the clean one's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 80 +++++++++++++-------- tools/server/server-test-faults.h | 14 ++-- tools/server/tests/unit/test_failed_fill.py | 52 ++++++++++++++ 3 files changed, 113 insertions(+), 33 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 7b965b708c80..e14a80cd350e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1369,8 +1369,9 @@ struct server_context_impl { } test_faults = server_test_faults::from_env(); - if (test_faults.checkpoint_copies || test_faults.prompt_fills) { - SRV_WRN("test faults: checkpoint copies = %d, prompt fills = %d\n", test_faults.checkpoint_copies, test_faults.prompt_fills); + if (test_faults.any()) { + SRV_WRN("test faults: prompt checkpoints = %d, speculative checkpoints = %d, prompt fills = %d\n", + test_faults.prompt_checkpoints, test_faults.spec_checkpoints, test_faults.prompt_fills); } // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens @@ -2515,6 +2516,44 @@ struct server_context_impl { return (slot.task->n_tokens() + params_base.n_ctx_checkpoints - 1) / params_base.n_ctx_checkpoints; } + enum class checkpoint_kind { prompt, speculative }; + + // which parts of a sequence's state a checkpoint copies + struct checkpoint_parts { + bool tgt; + bool dft; + bool spec; // the speculative implementation's own state + }; + + // Continuum fork: the one place a checkpoint's state copy is allocated. A copy that cannot be + // allocated is refused, not a slot failure: the checkpoint is cleared, false is returned and + // the caller goes on without it (a restore from a prompt checkpoint is cold; a draft that a + // speculative checkpoint would protect is dropped, and the step decodes without it). + bool copy_checkpoint(const server_slot & slot, common_prompt_checkpoint & ckpt, checkpoint_kind kind, checkpoint_parts parts) { + const bool is_prompt = kind == checkpoint_kind::prompt; + try { + if (server_test_faults::take(is_prompt ? test_faults.prompt_checkpoints : test_faults.spec_checkpoints)) { + throw std::bad_alloc(); // where the copies below allocate + } + if (parts.tgt) { + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + } + if (parts.dft) { + ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + } + if (parts.spec) { + common_speculative_get_state(spec.get(), slot.id, ckpt.data_spec); + } + return true; + } catch (const std::bad_alloc &) { + SLT_WRN(slot, "%s checkpoint refused: its state copy could not be allocated (n_tokens = %" PRId64 "), %s\n", + is_prompt ? "context" : "speculative", ckpt.n_tokens, + is_prompt ? "a restore from here will be cold" : "the draft is dropped"); + ckpt.clear(); + return false; + } + } + // n_tokens_cur: the number of tokens added to the batch for the current slot void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { const int id_task = slot.task->id; @@ -2556,17 +2595,8 @@ struct server_context_impl { // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); - // a copy that cannot be allocated only makes a later restore from here cold - try { - if (server_test_faults::take(test_faults.checkpoint_copies)) { - throw std::bad_alloc(); // where the copy below allocates - } - cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - // stash the draft's speculative state with the checkpoint - common_speculative_get_state(spec.get(), slot.id, cur.data_spec); - } catch (const std::bad_alloc &) { - SLT_WRN(slot, "context checkpoint refused: its copy could not be allocated (n_tokens = %" PRId64 "), a restore from here will be cold\n", cur.n_tokens); + // with the draft's speculative state stashed beside it + if (!copy_checkpoint(slot, cur, checkpoint_kind::prompt, { /* tgt */ true, /* dft */ true, /* spec */ true })) { return; } @@ -3666,8 +3696,8 @@ struct server_context_impl { llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id), llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); - if (use_ckpt_dft) { - slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + if (use_ckpt_dft && !copy_checkpoint(slot, slot.spec_ckpt, checkpoint_kind::speculative, { false, true, false })) { + return; // no draft this step } slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); @@ -3699,8 +3729,6 @@ struct server_context_impl { auto & draft = slot.spec_draft; auto & ckpt = slot.spec_ckpt; - slot.stats.n_draft_tokens += draft.size(); - // TODO: avoid restoring the draft context and re-evaluating the drafted tokens when not needed [TAG_SPEC_AVOID_DRAFT_REEVAL] const bool use_ckpt_dft = ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL; @@ -3722,24 +3750,18 @@ struct server_context_impl { const bool use_ckpt_dft = (ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_RS && draft.size() > llama_n_rs_seq(ctx_dft)); - if (use_ckpt_tgt) { - //const int64_t t_start = ggml_time_us(); - - ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - - //const int64_t t_total = ggml_time_us() - t_start; - //printf("checkpoint total: %f ms\n", t_total / 1000.0); - + if ((use_ckpt_tgt || use_ckpt_dft) && + !copy_checkpoint(slot, ckpt, checkpoint_kind::speculative, { use_ckpt_tgt, use_ckpt_dft, false })) { + draft.clear(); // a draft that could not be rolled back is not verified + } else if (use_ckpt_tgt) { SLT_DBG(slot, "created speculative checkpoint (pos_min = %d, pos_max = %d, n_tokens = %d, size = %.3f MiB, draft = %.3f MiB)\n", ckpt.pos_min, ckpt.pos_max, slot.prompt.n_tokens(), (float) ckpt.size() / 1024 / 1024, (float) ckpt.data_dft.size() / 1024 / 1024); } - - if (use_ckpt_dft) { - ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - } } + + slot.stats.n_draft_tokens += draft.size(); }); // update the batch with the sampled/drafted tokens diff --git a/tools/server/server-test-faults.h b/tools/server/server-test-faults.h index 573abe969e46..d9cc091aad27 100644 --- a/tools/server/server-test-faults.h +++ b/tools/server/server-test-faults.h @@ -6,16 +6,22 @@ // Continuum fork: failures the server tests inject through the environment, so a test can show // that a failure leaves the server consistent. Every count is zero unless its variable is set. struct server_test_faults { - int32_t checkpoint_copies = 0; // LLAMA_SERVER_TEST_FAIL_CHECKPOINTS: the next N prompt checkpoint copies fail to allocate - int32_t prompt_fills = 0; // LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS: the next N prompt fills throw after adding their tokens + int32_t prompt_checkpoints = 0; // LLAMA_SERVER_TEST_FAIL_CHECKPOINTS: the next N prompt checkpoint copies fail to allocate + int32_t spec_checkpoints = 0; // LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS: the next N speculative checkpoint copies fail to allocate + int32_t prompt_fills = 0; // LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS: the next N prompt fills throw after adding their tokens static server_test_faults from_env() { server_test_faults faults; - faults.checkpoint_copies = read("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS"); - faults.prompt_fills = read("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS"); + faults.prompt_checkpoints = read("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS"); + faults.spec_checkpoints = read("LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS"); + faults.prompt_fills = read("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS"); return faults; } + bool any() const { + return prompt_checkpoints || spec_checkpoints || prompt_fills; + } + // true when this occurrence is one to fail, counting it static bool take(int32_t & remaining) { if (remaining <= 0) { diff --git a/tools/server/tests/unit/test_failed_fill.py b/tools/server/tests/unit/test_failed_fill.py index fd69a5c9d75f..6269b6082464 100644 --- a/tools/server/tests/unit/test_failed_fill.py +++ b/tools/server/tests/unit/test_failed_fill.py @@ -1,3 +1,5 @@ +import os +import tempfile import pytest from utils import * @@ -73,3 +75,53 @@ def test_a_failed_fill_leaves_the_batch_and_its_prompt_clean(monkeypatch): assert retry.status_code == 200, retry.body assert retry.body["timings"]["cache_n"] == 0 assert retry.body["content"] == good.body["content"] + + +def speculative_server() -> ServerProcess: + # a recurrent model: its memory cannot drop a rejected draft's tail, so every verified draft + # is protected by a speculative checkpoint of the target's state + preset = ServerProcess() + preset.model_hf_repo = "mradermacher/mamba-130m-hf-GGUF" + preset.model_hf_file = "mamba-130m-hf.Q4_K_M.gguf" + preset.model_alias = "mamba-130m" + preset.n_slots = 1 + preset.n_ctx = 2048 + preset.temperature = 0.0 + preset.spec_type = "ngram-simple" + preset.spec_draft_n_max = 8 + fd, preset.log_path = tempfile.mkstemp(suffix=".log") + os.close(fd) + return preset + + +def generate_repetitive(): + return server.make_request("POST", "/completion", data={ + # a period longer than the 12-token n-gram, so ngram-simple drafts from the history + "prompt": "the quick brown fox jumps over the lazy dog and then runs into the deep dark forest, " * 5 + "the quick", + "n_predict": 48, + "temperature": 0.0, + "return_tokens": True, + }) + + +# what this catches: the speculative checkpoint taken before a draft is verified allocates the +# same kind of state copy; a std::bad_alloc there released a generating slot mid-reply. Refused, +# the draft is dropped and the step decodes without it, so the reply is unchanged. +def test_a_refused_speculative_checkpoint_drops_the_draft_not_the_reply(monkeypatch): + global server + server = speculative_server() + server.start() + clean = generate_repetitive() + assert clean.status_code == 200, clean.body + assert clean.body["timings"]["draft_n"] > 0 # it drafted, so speculative checkpoints were taken + server.stop() + + server = speculative_server() + monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS", "1") + server.start() + faulted = generate_repetitive() + assert faulted.status_code == 200, faulted.body + assert faulted.body["tokens"] == clean.body["tokens"] + with open(server.log_path) as log: + assert "speculative checkpoint refused" in log.read() + assert server.make_request("GET", "/health").status_code == 200 From 5f3b59f3843022523dc045657e82e149fdc9b6fb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sat, 7 Nov 2026 02:08:53 -0600 Subject: [PATCH 4/4] server: test faults compile only into the server test build server_test_faults read LLAMA_SERVER_TEST_FAIL_* from the environment in every build, so a stray variable could fail a real slot. The faults now compile in only with -DLLAMA_SERVER_TEST_FAULTS=ON (a CMake option, off by default, that defines LLAMA_SERVER_TEST_FAULTS for server-context). Without it server_test_faults::enabled is false, every use sits under `if constexpr (server_test_faults::enabled)` and is discarded, and from_env() reads nothing: a default build carries no fault string or symbol. The server CI builds (server.yml, server-sanitize.yml, server-self-hosted.yml) set the option, the tests README says so, and test_failed_fill.py skips with that reason when the server it runs has no fault code. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .github/workflows/server-sanitize.yml | 1 + .github/workflows/server-self-hosted.yml | 6 ++-- .github/workflows/server.yml | 6 ++-- tools/server/CMakeLists.txt | 6 ++++ tools/server/server-context.cpp | 24 ++++++++----- tools/server/server-test-faults.h | 17 ++++++++- tools/server/tests/README.md | 4 ++- tools/server/tests/unit/test_failed_fill.py | 39 +++++++++++++++------ 8 files changed, 76 insertions(+), 27 deletions(-) diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index a0ffc70c1210..7d955bc35e3b 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -79,6 +79,7 @@ jobs: cmake -B build \ -DLLAMA_BUILD_BORINGSSL=ON \ -DGGML_SCHED_NO_REALLOC=ON \ + -DLLAMA_SERVER_TEST_FAULTS=ON \ -DGGML_SANITIZE_ADDRESS=${{ matrix.sanitizer == 'ADDRESS' }} \ -DGGML_SANITIZE_THREAD=${{ matrix.sanitizer == 'THREAD' }} \ -DGGML_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} \ diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index 675ddbaaa580..c7298c9e9b26 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -55,7 +55,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DGGML_SCHED_NO_REALLOC=ON + cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DLLAMA_SERVER_TEST_FAULTS=ON cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) --target llama-server - name: Python setup @@ -115,7 +115,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON + cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DLLAMA_SERVER_TEST_FAULTS=ON cmake --build build --config Release -j $(nproc) --target llama-server - name: Python setup @@ -202,7 +202,7 @@ jobs: - name: Build id: cmake_build run: | - cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON + cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_SERVER_TEST_FAULTS=ON cmake --build build --config Release -j $(nproc) --target llama-server - name: Python setup diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 93370e51c64f..ecbd9b18af40 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -90,7 +90,8 @@ jobs: id: cmake_build run: | cmake -B build \ - -DGGML_SCHED_NO_REALLOC=ON + -DGGML_SCHED_NO_REALLOC=ON \ + -DLLAMA_SERVER_TEST_FAULTS=ON cmake --build build --config Release -j $(nproc) --target llama-server test-server-prefill - name: Prefill scheduling test @@ -167,7 +168,8 @@ jobs: -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^ -DCMAKE_BUILD_TYPE=Release ^ -DLLAMA_BUILD_BORINGSSL=ON ^ - -DGGML_SCHED_NO_REALLOC=ON + -DGGML_SCHED_NO_REALLOC=ON ^ + -DLLAMA_SERVER_TEST_FAULTS=ON set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1 cmake --build build --config Release -j %NINJA_JOBS% --target llama-server test-server-prefill diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 141d44126731..67fb03964d7c 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -34,6 +34,12 @@ target_include_directories(${TARGET} PRIVATE ../mtmd) target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) +# the faults tools/server/tests inject (server-test-faults.h); only the server test build sets it +option(LLAMA_SERVER_TEST_FAULTS "llama-server: compile in the test fault injection (tests only, never a release)" OFF) +if (LLAMA_SERVER_TEST_FAULTS) + target_compile_definitions(${TARGET} PRIVATE LLAMA_SERVER_TEST_FAULTS) +endif() + # llama-server-impl: server logic, reusable by app set(TARGET llama-server-impl) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e14a80cd350e..2b9f6c5bed70 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -970,7 +970,7 @@ struct server_context_impl { int trace = 0; // env: LLAMA_TRACE int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF - server_test_faults test_faults; // env: LLAMA_SERVER_TEST_FAIL_* + server_test_faults test_faults; // the server test build only (LLAMA_SERVER_TEST_FAULTS) int n_empty_consecutive = 0; @@ -1368,10 +1368,12 @@ struct server_context_impl { } } - test_faults = server_test_faults::from_env(); - if (test_faults.any()) { - SRV_WRN("test faults: prompt checkpoints = %d, speculative checkpoints = %d, prompt fills = %d\n", - test_faults.prompt_checkpoints, test_faults.spec_checkpoints, test_faults.prompt_fills); + if constexpr (server_test_faults::enabled) { + test_faults = server_test_faults::from_env(); + if (test_faults.any()) { + SRV_WRN("test faults: prompt checkpoints = %d, speculative checkpoints = %d, prompt fills = %d\n", + test_faults.prompt_checkpoints, test_faults.spec_checkpoints, test_faults.prompt_fills); + } } // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens @@ -2532,8 +2534,10 @@ struct server_context_impl { bool copy_checkpoint(const server_slot & slot, common_prompt_checkpoint & ckpt, checkpoint_kind kind, checkpoint_parts parts) { const bool is_prompt = kind == checkpoint_kind::prompt; try { - if (server_test_faults::take(is_prompt ? test_faults.prompt_checkpoints : test_faults.spec_checkpoints)) { - throw std::bad_alloc(); // where the copies below allocate + if constexpr (server_test_faults::enabled) { + if (server_test_faults::take(is_prompt ? test_faults.prompt_checkpoints : test_faults.spec_checkpoints)) { + throw std::bad_alloc(); // where the copies below allocate + } } if (parts.tgt) { ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); @@ -4315,8 +4319,10 @@ struct server_context_impl { // the number of tokens added to the batch for the current slot const auto n_tokens_cur = batch.size() - n_tokens_prev; - if (n_tokens_cur > 0 && server_test_faults::take(test_faults.prompt_fills)) { - throw std::runtime_error("test fault: the prompt fill failed after adding its tokens"); + if constexpr (server_test_faults::enabled) { + if (n_tokens_cur > 0 && server_test_faults::take(test_faults.prompt_fills)) { + throw std::runtime_error("test fault: the prompt fill failed after adding its tokens"); + } } const auto n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; diff --git a/tools/server/server-test-faults.h b/tools/server/server-test-faults.h index d9cc091aad27..a46bfc8bd86e 100644 --- a/tools/server/server-test-faults.h +++ b/tools/server/server-test-faults.h @@ -1,20 +1,33 @@ #pragma once #include +#ifdef LLAMA_SERVER_TEST_FAULTS #include +#endif // Continuum fork: failures the server tests inject through the environment, so a test can show -// that a failure leaves the server consistent. Every count is zero unless its variable is set. +// that a failure leaves the server consistent. Compiled in only by the server test build +// (cmake -DLLAMA_SERVER_TEST_FAULTS=ON). In any other build `enabled` is false, every use sits +// under `if constexpr (server_test_faults::enabled)` and is discarded, and no environment is +// read, so a stray variable can never fail a real slot. struct server_test_faults { +#ifdef LLAMA_SERVER_TEST_FAULTS + static constexpr bool enabled = true; +#else + static constexpr bool enabled = false; +#endif + int32_t prompt_checkpoints = 0; // LLAMA_SERVER_TEST_FAIL_CHECKPOINTS: the next N prompt checkpoint copies fail to allocate int32_t spec_checkpoints = 0; // LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS: the next N speculative checkpoint copies fail to allocate int32_t prompt_fills = 0; // LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS: the next N prompt fills throw after adding their tokens static server_test_faults from_env() { server_test_faults faults; +#ifdef LLAMA_SERVER_TEST_FAULTS faults.prompt_checkpoints = read("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS"); faults.spec_checkpoints = read("LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS"); faults.prompt_fills = read("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS"); +#endif return faults; } @@ -31,9 +44,11 @@ struct server_test_faults { return true; } +#ifdef LLAMA_SERVER_TEST_FAULTS private: static int32_t read(const char * name) { const char * value = std::getenv(name); return value ? std::atoi(value) : 0; } +#endif }; diff --git a/tools/server/tests/README.md b/tools/server/tests/README.md index f566b43644b1..e98ced07ce1a 100644 --- a/tools/server/tests/README.md +++ b/tools/server/tests/README.md @@ -17,10 +17,12 @@ To mitigate it, you can increase values in `n_predict`, `kv_size`. ```shell cd ../../.. -cmake -B build +cmake -B build -DLLAMA_SERVER_TEST_FAULTS=ON cmake --build build --target llama-server ``` +`LLAMA_SERVER_TEST_FAULTS` compiles in the faults `unit/test_failed_fill.py` injects; without it those tests skip. Never set it for a build that serves. + 2. Start the test: `./tests.sh` It's possible to override some scenario steps values with environment variables: diff --git a/tools/server/tests/unit/test_failed_fill.py b/tools/server/tests/unit/test_failed_fill.py index 6269b6082464..c7db18afde45 100644 --- a/tools/server/tests/unit/test_failed_fill.py +++ b/tools/server/tests/unit/test_failed_fill.py @@ -1,11 +1,13 @@ import os import tempfile +import time import pytest from utils import * # A slot whose prompt fill fails must leave the batch and its own prompt as it found them: the # other slot completes, the server stays up, and the failed slot's next request trusts only the -# tokens its memory really holds. The faults are injected through LLAMA_SERVER_TEST_FAIL_*. +# tokens its memory really holds. The faults are injected through LLAMA_SERVER_TEST_FAIL_*, +# which only a server built with -DLLAMA_SERVER_TEST_FAULTS=ON reads. server: ServerProcess @@ -30,12 +32,32 @@ def complete_both(): return parallel_function_calls([(complete, (0,)), (complete, (1,))]) +def with_log(preset: ServerProcess) -> ServerProcess: + fd, preset.log_path = tempfile.mkstemp(suffix=".log") + os.close(fd) + return preset + + def configure(preset: ServerProcess) -> ServerProcess: preset.n_slots = 2 preset.n_predict = 8 preset.temperature = 0.0 preset.cache_ram = 0 # a retry must not be served from another slot's saved prompt - return preset + return with_log(preset) + + +def start_with_fault(monkeypatch, fault: str, count: int): + monkeypatch.setenv(fault, str(count)) + server.start() + # a server built without the faults never reads the variable, and says nothing about it + deadline = time.time() + 5 + while True: + with open(server.log_path) as log: + if "test faults:" in log.read(): + return + if time.time() > deadline: + pytest.skip("llama-server was built without -DLLAMA_SERVER_TEST_FAULTS=ON") + time.sleep(0.05) # what this catches: a checkpoint copy that cannot be allocated failed its slot after the slot had @@ -45,8 +67,7 @@ def configure(preset: ServerProcess) -> ServerProcess: def test_a_refused_checkpoint_copy_is_not_a_slot_failure(monkeypatch): global server server = configure(ServerPreset.tinygemma3()) # sliding-window attention: its prompts checkpoint - monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_CHECKPOINTS", "2") # one copy per slot, as on the node - server.start() + start_with_fault(monkeypatch, "LLAMA_SERVER_TEST_FAIL_CHECKPOINTS", 2) # one copy per slot, as on the node first, second = complete_both() assert first.status_code == 200, first.body @@ -61,8 +82,7 @@ def test_a_refused_checkpoint_copy_is_not_a_slot_failure(monkeypatch): def test_a_failed_fill_leaves_the_batch_and_its_prompt_clean(monkeypatch): global server server = configure(ServerPreset.tinyllama2()) - monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS", "1") - server.start() + start_with_fault(monkeypatch, "LLAMA_SERVER_TEST_FAIL_PROMPT_FILLS", 1) results = complete_both() failed = [i for i, res in enumerate(results) if res.status_code != 200] @@ -89,9 +109,7 @@ def speculative_server() -> ServerProcess: preset.temperature = 0.0 preset.spec_type = "ngram-simple" preset.spec_draft_n_max = 8 - fd, preset.log_path = tempfile.mkstemp(suffix=".log") - os.close(fd) - return preset + return with_log(preset) def generate_repetitive(): @@ -117,8 +135,7 @@ def test_a_refused_speculative_checkpoint_drops_the_draft_not_the_reply(monkeypa server.stop() server = speculative_server() - monkeypatch.setenv("LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS", "1") - server.start() + start_with_fault(monkeypatch, "LLAMA_SERVER_TEST_FAIL_SPEC_CHECKPOINTS", 1) faulted = generate_repetitive() assert faulted.status_code == 200, faulted.body assert faulted.body["tokens"] == clean.body["tokens"]