server: the lost-reply binding: a learn names its execution, base step and engine; a repeat is answered from its receipt, a stale one is fenced

ggml/include/ggml-opt.h

@@ -153,6 +153,14 @@ extern "C" {
     // set gradients to zero, initialize loss, and optionally reset the optimizer
     GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer);
 
+    // The optimizer steps taken (the iteration the next step uses, minus one): read before and after
+    // a call that may step, so a caller knows whether the weights moved even when the call failed.
+    GGML_API int64_t ggml_opt_steps(ggml_opt_context_t opt_ctx);
+    // Abandons the period in progress: the gradients accumulated since the last step are dropped (the
+    // next backward starts a fresh period from zero), so a failure part-way through a period never
+    // carries its partial gradient into the next step. The optimizer's moments and iteration stay.
+    GGML_API void ggml_opt_discard_period(ggml_opt_context_t opt_ctx);
+
     GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically
 
     // get underlying tensors that store data

ggml/src/ggml-opt.cpp

@@ -958,6 +958,16 @@ struct ggml_tensor * ggml_opt_leaf_grad(ggml_opt_context_t opt_ctx, struct ggml_
     return it == opt_ctx->leaf_grads.end() ? nullptr : it->second;
 }
 
+int64_t ggml_opt_steps(ggml_opt_context_t opt_ctx) {
+    return opt_ctx->iter - 1;
+}
+
+void ggml_opt_discard_period(ggml_opt_context_t opt_ctx) {
+    // the next ggml_opt_alloc zeroes the accumulators when a period starts fresh at opt_i 0
+    opt_ctx->period_fresh = true;
+    opt_ctx->opt_i        = 0;
+}
+
 void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer) {
     opt_ctx->checkpoint_ready = false;
     if (optimizer) {

include/llama.h

@@ -1860,6 +1860,11 @@ extern "C" {
     // A reply within one batch is one exact step; a longer one steps per batch, its gradient
     // stopped at each batch boundary. false: refused or failed by name (llama_opt_failure),
     // with the sequence restored.
+    // The optimizer steps the active shadow's optimizer has committed (0 before llama_opt_init). Read
+    // before and after llama_opt_learn: a learn whose optimizer stepped committed an update even when
+    // it returned false (its restore failed), so the caller's fence and receipt follow this count.
+    LLAMA_API int64_t llama_opt_steps(struct llama_context * lctx);
+
     LLAMA_API bool llama_opt_learn(
             struct llama_context * lctx,
             llama_seq_id           seq,

src/llama-context-opt.cpp

@@ -1015,6 +1015,12 @@ bool llama_context::opt_learn(llama_seq_id seq, llama_pos n_cached, const llama_
     opt_outputs_internal  = false;
     opt_learn_labels      = 0;
     opt_learn_labels_done = 0;
+    // A batch that failed part-way through a period leaves its partial gradient in the accumulators:
+    // drop the period, so the next learn's step never applies this input's gradient under another
+    // input's receipt (Fable on fork #88). Steps already taken by earlier batches stand.
+    if (!ok && opt_ctx != nullptr) {
+        ggml_opt_discard_period(opt_ctx);
+    }
     // RESTORE: the reply's cells and the recurrent state decoded again under her stable set
     bool restored = attn == nullptr || attn->seq_rm(seq, n_cached, -1);
     if (recr != nullptr) {
@@ -1034,6 +1040,14 @@ bool llama_context::opt_learn(llama_seq_id seq, llama_pos n_cached, const llama_
     return ok;
 }
 
+int64_t llama_context::opt_steps() const {
+    return opt_ctx != nullptr ? ggml_opt_steps(opt_ctx) : 0;
+}
+
+int64_t llama_opt_steps(struct llama_context * ctx) {
+    return ctx->opt_steps();
+}
+
 bool llama_opt_learn(
         struct llama_context * ctx,
         llama_seq_id           seq,

src/llama-context.h

@@ -209,6 +209,8 @@ struct llama_context {
     bool opt_select(const llama_adapter_lora * adapter, struct llama_opt_params lopt_params);
     // forget `adapter`'s optimizer (its moments go); false when it has none
     bool opt_release(const llama_adapter_lora * adapter);
+    // the active shadow's committed optimizer steps (0 before llama_opt_init)
+    int64_t opt_steps() const;
 
     // TODO: more flexible combinations of logical/physical batch size and context size
     // set by llama_opt_stop from any thread; opt_epoch returns at the next data item

tests/test-opt-dynamic-accum.cpp

@@ -513,11 +513,81 @@ static void run_refuses_misshaped_gradient() {
     ggml_free(ctx);
 }
 
+// what this catches (Fable on fork #88): a learn whose reply spans batches failing part-way through
+// a period left that batch's partial gradient in the accumulators, and the next learn's step applied
+// it under another input's receipt. ggml_opt_discard_period drops the period. With a period of two
+// graphs on loss = sum(w * x) (gradient x per graph): one graph, a discard, then one graph is the
+// first half of a NEW period, so nothing steps; the next graph ends it and moves w by exactly
+// -lr * 2x, as from an untouched optimizer. Without the discard the second graph already ends the
+// period. ggml_opt_steps counts the committed steps the caller fences on.
+static void run_discarded_period() {
+    ggml_backend_t cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
+    GGML_ASSERT(cpu != nullptr);
+    ggml_backend_t backends[] = { cpu };
+    ggml_backend_sched_t sched = ggml_backend_sched_new(backends, nullptr, 1, GGML_DEFAULT_GRAPH_SIZE, false, true);
+
+    ggml_init_params sp = { 8 * ggml_tensor_overhead(), nullptr, true };
+    ggml_context * ctx_static = ggml_init(sp);
+    ggml_tensor * w = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, 1);
+    ggml_set_param(w);
+    ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_static, cpu);
+    const float w0 = 1.0f;
+    ggml_backend_tensor_set(w, &w0, 0, sizeof(float));
+
+    ggml_opt_params params = ggml_opt_default_params(sched, GGML_OPT_LOSS_TYPE_SUM);
+    params.optimizer    = GGML_OPT_OPTIMIZER_TYPE_SGD;
+    params.get_opt_pars = sgd_pars;
+    params.opt_period   = 2;
+    ggml_opt_context_t opt_ctx = ggml_opt_init(params);
+    ggml_opt_result_t  result  = ggml_opt_result_init();
+    CHECK(ggml_opt_steps(opt_ctx) == 0);
+
+    auto eval = [&]() {
+        ggml_init_params cp = { GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + 4 * ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), nullptr, true };
+        ggml_context * ctx_compute = ggml_init(cp);
+        ggml_tensor * x   = ggml_new_tensor_1d(ctx_compute, GGML_TYPE_F32, 1);
+        ggml_tensor * out = ggml_mul(ctx_compute, w, x);
+        ggml_cgraph * gf  = ggml_new_graph_custom(ctx_compute, GGML_DEFAULT_GRAPH_SIZE, true);
+        ggml_build_forward_expand(gf, out);
+        ggml_opt_prepare_alloc(opt_ctx, ctx_compute, gf, x, out);
+        CHECK(ggml_opt_alloc(opt_ctx, true));
+        ggml_backend_tensor_set(x, &X, 0, sizeof(float));
+        ggml_opt_eval(opt_ctx, result);
+        ggml_free(ctx_compute);
+    };
+    auto w_now = [&]() {
+        float v;
+        ggml_backend_tensor_get(w, &v, 0, sizeof(float));
+        return v;
+    };
+
+    eval();                           // the failed learn's first graph: half a period, a gradient
+    ggml_opt_discard_period(opt_ctx); // its next graph failed: the period is dropped
+    eval();                           // the next learn's first graph: half of a fresh period
+    if (std::fabs(w_now() - w0) > 1e-7f || ggml_opt_steps(opt_ctx) != 0) {
+        fprintf(stderr, "a discarded period still counted: w %f (from %f), %lld steps\n", w_now(), w0, (long long) ggml_opt_steps(opt_ctx));
+        exit(1);
+    }
+    eval();                           // ends the fresh period: its two graphs, never three
+    if (std::fabs(w_now() - (w0 - LR * 2 * X)) > 1e-6f || ggml_opt_steps(opt_ctx) != 1) {
+        fprintf(stderr, "after a discarded period w moved to %f, expected %f\n", w_now(), w0 - LR * 2 * X);
+        exit(1);
+    }
+
+    ggml_opt_result_free(result);
+    ggml_opt_free(opt_ctx);
+    ggml_backend_buffer_free(buf);
+    ggml_free(ctx_static);
+    ggml_backend_sched_free(sched);
+    ggml_backend_free(cpu);
+}
+
 int main() {
     ggml_backend_load_all();
     run(1);
     run(2);
     run_manual();
+    run_discarded_period();
     run_refuses_graph_without_backward();
     run_trains_through_broadcast_divisor();
     run_refuses_misshaped_gradient();