joelteply · Nov 1, 2026 · Nov 7, 2026
diff --git a/include/llama.h b/include/llama.h
@@ -1764,7 +1764,15 @@ extern "C" {
         uint32_t walk_horizon;
     };
 
-    LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
+    // false, with nothing changed and llama_opt_failure saying why, when the adapter to train cannot
+    // train: not attached at a non-zero scale, an A/B tensor that is not F32, or none of its tensors
+    // selected by the filter. A serving context refuses the learn instead of aborting.
+    LLAMA_API bool llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
+    // A serving context trains one shadow per mind: llama_opt_init with another adapter makes it the
+    // trained shadow, creating its optimizer the first time and keeping every shadow's moments across
+    // switches. llama_opt_release forgets one shadow's optimizer (before its adapter is unloaded);
+    // false when it had none.
+    LLAMA_API bool llama_opt_release(struct llama_context * lctx, const struct llama_adapter_lora * adapter);
     // Restore one shadow generation in place, preserving optimizer tensor identity. The caller
     // owns verified same-generation files/binding, stable adapter exclusion, and serialized queue
     // custody. Both payloads are prepared before mutation. An interrupted apply quarantines the

diff --git a/src/llama-context-opt.cpp b/src/llama-context-opt.cpp
@@ -79,8 +79,46 @@ bool llama_context::opt_fa_back_native() const {
     return true;
 }
 
-void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {
-    GGML_ASSERT(!opt_ctx);
+// Whether a tensor would be made a trainable parameter (llama_set_param's rule, without setting it).
+static bool llama_would_set_param(const struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) {
+    return tensor && tensor->type == GGML_TYPE_F32 && param_filter(tensor, userdata)
+        && strcmp(tensor->name, "token_embd.weight") != 0 && strcmp(tensor->name, "rope_freqs.weight") != 0;
+}
+
+// The one check every entry that trains an adapter makes before it changes anything: a learn on a
+// serving context that fails it is refused by name, never asserted (an F16 copy of a shadow killed
+// the serving process at the old asserts: Fable on fork #85, 2026-11-07, exit 134).
+const char * llama_context::opt_untrainable(const llama_adapter_lora * adapter, const struct llama_opt_params & lopt_params) const {
+    const auto attached = loras->find(const_cast<llama_adapter_lora *>(adapter));
+    if (attached == loras->end() || attached->second == 0.0f) {
+        return "the adapter to train is not attached at a non-zero scale";
+    }
+    bool selected = false;
+    for (const auto & [name, ab] : adapter->ab_map) {
+        if (ab.a->type != GGML_TYPE_F32 || ab.b->type != GGML_TYPE_F32) {
+            return "the adapter holds non-F32 A/B tensors (a trainable shadow is F32)";
+        }
+        selected = selected || llama_would_set_param(ab.a, lopt_params.param_filter, lopt_params.param_filter_ud)
+            || llama_would_set_param(ab.b, lopt_params.param_filter, lopt_params.param_filter_ud);
+    }
+    if (!selected) {
+        return "the parameter filter selected none of the adapter's tensors";
+    }
+    return nullptr;
+}
+
+bool llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {
+    if (lopt_params.adapter != nullptr) {
+        if (const char * why = opt_untrainable(lopt_params.adapter, lopt_params)) {
+            opt_failure = why;
+            return false;
+        }
+    }
+    // a serving context already configured for its first shadow: another mind's shadow is a switch
+    if (lopt_params.adapter != nullptr && !opt_minds.empty()) {
+        return opt_select(lopt_params.adapter, lopt_params);
+    }
+    GGML_ASSERT(!opt_configured());
     // The training window lives on this context, not the shared model (a serving context may
     // be running on the same weights).
     opt_n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx();
@@ -132,6 +170,7 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
     opt_params.optimizer       = lopt_params.optimizer_type;
     opt_ctx = ggml_opt_init(opt_params);
     ggml_opt_set_alloc_budget(opt_ctx, opt_memory_budget);
+    opt_params_base = opt_params;
 
     llama_opt_param_filter param_filter = lopt_params.param_filter;
     void * param_filter_ud              = lopt_params.param_filter_ud;
@@ -140,19 +179,17 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
         // LoRA-only: the adapter's A/B are the parameters, the base is frozen. An adapter
         // that is not attached is not in the graph, so it would train nothing while every
         // epoch reported a loss: refuse it here instead.
-        const auto it = loras->find(lopt_params.adapter);
-        GGML_ASSERT(it != loras->end() && it->second != 0.0f && "the LoRA to train must be attached with a non-zero scale");
+        // opt_untrainable above has refused an adapter that is not attached, not F32, or selects nothing
         int64_t n_params = 0;
         for (auto & [name, ab] : lopt_params.adapter->ab_map) {
-            GGML_ASSERT(ab.a->type == GGML_TYPE_F32 && ab.b->type == GGML_TYPE_F32 && "a trainable LoRA holds F32 A/B tensors");
             n_params += llama_set_param(ab.a, param_filter, param_filter_ud);
             n_params += llama_set_param(ab.b, param_filter, param_filter_ud);
         }
-        GGML_ASSERT(n_params > 0 && "the param filter selected none of the adapter's tensors");
         // the trained adapter is the shadow: labelled rows only, never a serving row
         opt_shadow = lopt_params.adapter;
+        opt_minds.emplace(lopt_params.adapter, opt_ctx);
         LLAMA_LOG_INFO("%s: LoRA-only training: %lld adapter tensors trainable, base frozen\n", __func__, (long long) n_params);
-        return;
+        return true;
     }
 
   //llama_set_param(model->tok_embd,        param_filter, param_filter_ud); // FIXME
@@ -176,6 +213,66 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
             llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud);
         }
     }
+    return true;
+}
+
+// Another mind's shadow on a configured serving context. Only the active shadow's tensors are
+// parameters, so a graph never carries a parameter the active optimizer was not built with; each
+// shadow keeps its own optimizer (moments, accumulators), so a switch back resumes where it was.
+bool llama_context::opt_select(const llama_adapter_lora * adapter, struct llama_opt_params lopt_params) {
+    if (adapter == opt_shadow) {
+        return true;
+    }
+    // checked before the previous shadow's parameters are cleared: a refused switch changes nothing
+    if (const char * why = opt_untrainable(adapter, lopt_params)) {
+        opt_failure = why;
+        return false;
+    }
+    if (opt_shadow != nullptr) {
+        for (const auto & [name, ab] : opt_shadow->ab_map) {
+            ab.a->flags &= ~GGML_TENSOR_FLAG_PARAM;
+            ab.b->flags &= ~GGML_TENSOR_FLAG_PARAM;
+        }
+    }
+    int64_t n_params = 0;
+    for (const auto & [name, ab] : adapter->ab_map) {
+        n_params += llama_set_param(ab.a, lopt_params.param_filter, lopt_params.param_filter_ud);
+        n_params += llama_set_param(ab.b, lopt_params.param_filter, lopt_params.param_filter_ud);
+    }
+    const auto mind = opt_minds.find(adapter);
+    if (mind != opt_minds.end()) {
+        opt_ctx = mind->second;
+    } else {
+        ggml_opt_params params = opt_params_base;
+        params.get_opt_pars    = lopt_params.get_opt_pars;
+        params.get_opt_pars_ud = lopt_params.get_opt_pars_ud;
+        params.optimizer       = lopt_params.optimizer_type;
+        opt_ctx = ggml_opt_init(params);
+        ggml_opt_set_alloc_budget(opt_ctx, opt_memory_budget);
+        opt_minds.emplace(adapter, opt_ctx);
+        LLAMA_LOG_INFO("%s: a new shadow's optimizer: %lld adapter tensors trainable (%zu shadows)\n", __func__,
+                (long long) n_params, opt_minds.size());
+    }
+    opt_shadow = adapter;
+    return true;
+}
+
+bool llama_context::opt_release(const llama_adapter_lora * adapter) {
+    const auto mind = opt_minds.find(adapter);
+    if (mind == opt_minds.end()) {
+        return false;
+    }
+    if (adapter == opt_shadow) {
+        for (const auto & [name, ab] : adapter->ab_map) {
+            ab.a->flags &= ~GGML_TENSOR_FLAG_PARAM;
+            ab.b->flags &= ~GGML_TENSOR_FLAG_PARAM;
+        }
+        opt_shadow = nullptr;
+        opt_ctx    = nullptr;
+    }
+    ggml_opt_free(mind->second);
+    opt_minds.erase(mind);
+    return true;
 }
 
 // THE WALK (Fable's design, continuum 2026-10-06; objective 2026-10-14): one pass over a window
@@ -749,10 +846,14 @@ bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userda
     return true;
 }
 
-void llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {
+bool llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {
     ctx->opt_alloc_failed.store(false);
     ctx->opt_failure.clear();
-    ctx->opt_init(model, lopt_params);
+    return ctx->opt_init(model, lopt_params);
+}
+
+bool llama_opt_release(struct llama_context * ctx, const struct llama_adapter_lora * adapter) {
+    return ctx->opt_release(adapter);
 }
 
 void llama_opt_stop(struct llama_context * ctx, bool stop) {

diff --git a/src/llama-context.cpp b/src/llama-context.cpp
@@ -493,7 +493,12 @@ llama_context::~llama_context() {
             }
         }
     }
-    ggml_opt_free(opt_ctx);
+    for (auto & [adapter, mind] : opt_minds) {
+        ggml_opt_free(mind);
+    }
+    if (opt_minds.empty()) {
+        ggml_opt_free(opt_ctx); // the base's own optimizer, when the base trained
+    }
 }
 
 void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs) {
@@ -580,7 +585,7 @@ void llama_context::sched_reserve() {
     // A training context's scheduler was sized for the training graph in opt_init, and the
     // optimizer holds it (ggml_opt_params.backend_sched). Re-creating it here, as the first
     // context decode of opt_epoch_iter's walk would, leaves the optimizer on a freed scheduler.
-    if (opt_ctx != nullptr) {
+    if (opt_configured()) {
         sched_need_reserve = false;
         return;
     }
@@ -2398,7 +2403,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
     // a training context (an optimizer exists), not this graph's flag: the walk decodes context
     // with cparams.training off, and a sched_reserve() run then must not shrink the scheduler
     // and the graph result below what the next training chunk needs
-    if (cparams.training || opt_ctx != nullptr) {
+    if (cparams.training || opt_configured()) {
         // each attention layer reads the cached context in front of the chunk (build_attn:
         // views of K and V, V's permute and cont, two concats, the mask's view and cont)
         res += 16u * model.hparams.n_layer();

diff --git a/src/llama-context.h b/src/llama-context.h
@@ -202,7 +202,13 @@ struct llama_context {
     // training
     //
 
-    void opt_init(struct llama_model * model, struct llama_opt_params lopt_params);
+    bool opt_init(struct llama_model * model, struct llama_opt_params lopt_params);
+    // why `adapter` cannot train on this context, or nullptr; checked before anything is changed
+    const char * opt_untrainable(const llama_adapter_lora * adapter, const struct llama_opt_params & lopt_params) const;
+    // make `adapter` the trained shadow, creating its optimizer the first time it trains
+    bool opt_select(const llama_adapter_lora * adapter, struct llama_opt_params lopt_params);
+    // forget `adapter`'s optimizer (its moments go); false when it has none
+    bool opt_release(const llama_adapter_lora * adapter);
 
     // TODO: more flexible combinations of logical/physical batch size and context size
     // set by llama_opt_stop from any thread; opt_epoch returns at the next data item
@@ -398,7 +404,16 @@ struct llama_context {
     std::vector<ggml_backend_ptr> backends;
 
     // training
+    // the ACTIVE optimizer: the one adapter-or-base this context's next training step updates
     ggml_opt_context_t opt_ctx = nullptr;
+    // one optimizer per shadow on a serving context (each mind's moments and accumulators, kept
+    // across switches); opt_ctx points at the active one. Empty when the base itself trains.
+    std::map<const llama_adapter_lora *, ggml_opt_context_t> opt_minds;
+    // the optimizer parameters every shadow's optimizer is created with (set by the first opt_init)
+    ggml_opt_params opt_params_base{};
+    // this context is configured for training (its scheduler and graph sized for it, and some
+    // optimizer holds that scheduler), whether or not a step is active
+    bool opt_configured() const { return opt_ctx != nullptr || !opt_minds.empty(); }
     // the training window this context's opt loop runs at; kept HERE, never written into the
     // model's hparams, because several contexts share one model (a serving context and a
     // training context on the same resident weights)

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
@@ -943,9 +943,8 @@ struct server_context_impl {
 
     common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
 
-    // LEARNING IN THE SERVING CONTEXT (SERVER_TASK_TYPE_LEARN): the shadow adapter llama_opt_init was
-    // given (one per server run for now), and the learning rate its next step reads
-    int32_t learn_lora_id = -1;
+    // LEARNING IN THE SERVING CONTEXT (SERVER_TASK_TYPE_LEARN): the learning rate the next step reads
+    // (each mind's shadow has its own optimizer in the context: llama_opt_init selects it)
     float   learn_lr      = 1e-4f;
     // LEARNING BARELY TOUCHES THE FLOW: while any other slot is working, learning may take at most
     // LEARN_DUTY of the loop's time (the step after a learn of t ms waits t*(1/LEARN_DUTY - 1) ms);
@@ -2198,9 +2197,6 @@ struct server_context_impl {
         if (L.lora_id < 0 || (size_t) L.lora_id >= loras.size() || loras[L.lora_id].ptr == nullptr) {
             return refused("no loaded adapter " + std::to_string(L.lora_id) + " to train (POST /lora-adapters/load it first)");
         }
-        if (learn_lora_id >= 0 && learn_lora_id != L.lora_id) {
-            return refused("this server trains adapter " + std::to_string(learn_lora_id) + "; one shadow per server run");
-        }
         const int32_t n0 = L.n_prefix - 1; // the last prompt token is re-decoded with the reply's first token as its label
         if (n0 < 0 || L.n_prefix > slot.prompt.n_tokens() || L.reply.empty()) {
             return refused("slot " + std::to_string(slot.id) + " holds " + std::to_string(slot.prompt.n_tokens())
@@ -2248,15 +2244,22 @@ struct server_context_impl {
         set[L.lora_id].scale = 1.0f;
         common_set_adapter_lora(ctx_tgt, set);
         learn_lr = L.lr;
-        if (learn_lora_id < 0) {
+        {
+            // her shadow is the trained one: its own optimizer, created the first time she learns
+            // and kept (moments and all) while other minds' shadows train in between
             llama_opt_params lopt{};
             lopt.param_filter    = llama_opt_param_filter_all;
             lopt.get_opt_pars    = learn_opt_pars;
             lopt.get_opt_pars_ud = &learn_lr;
             lopt.optimizer_type  = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
             lopt.adapter         = loras[L.lora_id].ptr;
-            llama_opt_init(ctx_tgt, model_tgt, lopt);
-            learn_lora_id = L.lora_id;
+            if (!llama_opt_init(ctx_tgt, model_tgt, lopt)) {
+                // refused before anything was trained or decoded: her own adapter set again, her
+                // slot untouched, and the server keeps serving (an F16 adapter killed it at an
+                // assert: Fable on fork #85, 2026-11-07)
+                common_set_adapter_lora(ctx_tgt, slot.lora);
+                return refused("adapter " + std::to_string(L.lora_id) + " cannot be trained: " + llama_opt_failure(ctx_tgt));
+            }
         }
         // drop what follows n0 (attention) and put her recurrent state back at n0
         if (cp != nullptr) {
@@ -3353,12 +3356,6 @@ struct server_context_impl {
                         send_error(task, "no loaded LoRA adapter with id " + std::to_string(id), ERROR_TYPE_INVALID_REQUEST);
                         break;
                     }
-                    if (id == learn_lora_id) {
-                        // Optimizer moments and opt_shadow still reference this adapter's tensors.
-                        send_error(task, "LoRA adapter " + std::to_string(id) +
-                                   " is bound to the learning optimizer; unload requires optimizer teardown", ERROR_TYPE_INVALID_REQUEST);
-                        break;
-                    }
                     llama_adapter_lora * ptr = loras[id].ptr;
                     if (loras[id].scale != 0.0f) {
                         send_error(task, "LoRA adapter " + std::to_string(id) + " is active by default (scale " +
@@ -3377,6 +3374,9 @@ struct server_context_impl {
                     if (ptr == nullptr) {
                         break;
                     }
+                    // a shadow that learned holds an optimizer referencing its tensors: release it
+                    // first (its moments go; a snapshot is the durable form of what it learned)
+                    llama_opt_release(ctx_tgt, ptr);
                     // An idle slot that last ran with it keeps KV computed WITH the adapter;
                     // once the adapter is gone its list would read as the base's, so that
                     // cache must not be reused.

diff --git a/tools/server/server-train.cpp b/tools/server/server-train.cpp
@@ -1066,7 +1066,10 @@ void server_trainer::run(json req, examples_data ex) {
     llama_opt_set_memory_budget(ctx, budget == 0 ? 0 : std::max<size_t>(1, budget > context_bytes ? budget - context_bytes : 1));
     // the exact walk's host memory per window: refused by name over the caller's cap
     llama_opt_set_walk_host_budget(ctx, (size_t) req.value("walk_host_budget_mib", (int64_t) 0) << 20);
-    llama_opt_init(ctx, model, lopt);
+    if (!llama_opt_init(ctx, model, lopt)) {
+        fail(std::string("the adapter cannot be trained: ") + llama_opt_failure(ctx));
+        return;
+    }
     llama_opt_set_step_callback(ctx, &server_trainer::before_window, this);
     {
         // from here an epoch can run: a cancel reaches this context directly