diff --git a/apps/memos-local-plugin/core/memory/l2/README.md b/apps/memos-local-plugin/core/memory/l2/README.md index fff034ee0..374c739da 100644 --- a/apps/memos-local-plugin/core/memory/l2/README.md +++ b/apps/memos-local-plugin/core/memory/l2/README.md @@ -122,10 +122,16 @@ and reward backprop spreads similar V values across all step traces, so `mean(V_without)` collapses onto `mean(V_with)` and the V7 contrast formula evaluates to ≈ 0 for every policy regardless of its actual utility. Anchoring the without-set against a neutral 0.5 baseline -guarantees that genuinely-useful policies (V_with ≈ 0.7-0.85) score -positive and net-neutral or harmful ones don't. As real comparable -without-evidence accumulates, the prior gracefully dilutes and we -recover the original V7 §0.6 contrast formulation. +guarantees that genuinely-useful policies score positive and +net-neutral or harmful ones don't. Post-v2.0.7 the normalized +credit-assignment backprop (see `core/reward/backprop.ts`) leaves +per-trace V values in a ~0.02–0.5 band (matching the +`minTraceValue: 0.005` docstring in `core/config/defaults.ts`), so the +`adaptiveBaseline` floor (`gain.ts::MIN_ADAPTIVE_BASELINE = 0.005`) +lets `poolMean` flow through as the actual anchor across that band +rather than being pinned at a stale pre-v2.0.7 constant. As real +comparable without-evidence accumulates, the prior gracefully dilutes +and we recover the original V7 §0.6 contrast formulation. Use cases: diff --git a/apps/memos-local-plugin/core/memory/l2/gain.ts b/apps/memos-local-plugin/core/memory/l2/gain.ts index d2c322562..b2b903aac 100644 --- a/apps/memos-local-plugin/core/memory/l2/gain.ts +++ b/apps/memos-local-plugin/core/memory/l2/gain.ts @@ -60,7 +60,21 @@ export const V7_NEUTRAL_BASELINE = 0.5; * samples. Five is roughly "one short episode worth" of signal. */ export const WITHOUT_PRIOR_PSEUDOCOUNT = 5; -export const MIN_ADAPTIVE_BASELINE = 0.2; +/** + * Protective floor for {@link adaptiveBaseline}. Only guards against a + * degenerate near-zero `poolMean` (empty / all-noise pool) — the normal + * v2.0.7 backprop V distribution sits in ~0.02–0.5 (see + * `core/config/defaults.ts` `minTraceValue` docstring), so the adaptive + * branch of `adaptiveBaseline(poolMean) = poolMean` must be reachable + * across that entire band. Pre-v2.0.7 this constant was 0.2, which was + * appropriate for the old right-to-left decay reward scale where pool + * means clustered around 0.5–0.85; after the normalized-credit backprop + * rewrite it clamped ~every pool to a stale 0.2 and effectively froze + * policy/skill promotion (see issue #2364). 0.005 matches the v2.0.7 + * `minTraceValue` floor and only kicks in when the pool is truly empty + * or all traces sit below the trace-retention threshold. + */ +export const MIN_ADAPTIVE_BASELINE = 0.005; export interface ComputeGainOpts { tauSoftmax: number; diff --git a/apps/memos-local-plugin/tests/unit/memory/l2/gain.test.ts b/apps/memos-local-plugin/tests/unit/memory/l2/gain.test.ts index 9f5a3ffdd..4d1fbc23d 100644 --- a/apps/memos-local-plugin/tests/unit/memory/l2/gain.test.ts +++ b/apps/memos-local-plugin/tests/unit/memory/l2/gain.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it, vi } from "vitest"; -import { applyGain, computeGain, nextStatus, smoothGain } from "../../../../core/memory/l2/gain.js"; +import { adaptiveBaseline, applyGain, computeGain, MIN_ADAPTIVE_BASELINE, nextStatus, smoothGain } from "../../../../core/memory/l2/gain.js"; import type { PolicyId, TraceRow } from "../../../../core/types.js"; function mkTrace(value: number): TraceRow { @@ -60,6 +60,47 @@ describe("memory/l2/gain", () => { expect(g.baseline).toBeCloseTo(0.5, 5); }); + it("adaptiveBaseline tracks poolMean across the v2.0.7 backprop V distribution (regression for #2364)", () => { + // v2.0.7 backprop V values cluster in the 0.02–0.5 band (see + // core/config/defaults.ts "V values for typical multi-step turns are + // clustered around 0.02–0.5"). At these poolMeans the baseline must + // *actually* adapt instead of clamping to a fixed floor. + expect(adaptiveBaseline(0.02)).toBeCloseTo(0.02, 6); + expect(adaptiveBaseline(0.05)).toBeCloseTo(0.05, 6); + expect(adaptiveBaseline(0.15)).toBeCloseTo(0.15, 6); + // Bounds still hold at both ends. + expect(adaptiveBaseline(0.6)).toBeCloseTo(0.5, 6); + expect(adaptiveBaseline(0)).toBeCloseTo(MIN_ADAPTIVE_BASELINE, 6); + expect(adaptiveBaseline(Number.NaN)).toBeCloseTo(0.5, 6); + // The floor must be low enough that the entire v2.0.7 V band + // (documented as 0.02–0.5) is inside the adaptive branch, not below + // it — otherwise adaptiveBaseline collapses to a constant floor and + // policy/skill promotion stalls fleet-wide. + expect(MIN_ADAPTIVE_BASELINE).toBeLessThanOrEqual(0.02); + }); + + it("computeGain follows poolMean when the pool sits in the v2.0.7 backprop V band (regression for #2364)", () => { + // Typical v2.0.7 pool: with-set ~0.05–0.08, without-set ~0.02–0.05. + // Before the fix, baseline clamped to 0.2 and gain went strongly + // negative for these pools even though the with-set outperforms the + // without-set by the reward distribution's own scale. + const g = computeGain( + { + policyId: "po_v207" as PolicyId, + withTraces: [mkTrace(0.06), mkTrace(0.05)], + withoutTraces: [mkTrace(0.03), mkTrace(0.04)], + }, + { tauSoftmax: 0.5 }, + ); + expect(g.poolMean).toBeCloseTo(0.045, 5); + expect(g.baseline).toBeCloseTo(0.045, 5); + // With baseline = 0.045 the blended without-mean sits close to the + // empirical mean instead of being dragged up to 0.2, so the sign of + // the gain reflects the with vs. without contrast rather than the + // stale floor. + expect(g.gain).toBeGreaterThan(0); + }); + it("uses value-weighted mean for the with-set when count ≥ 3", () => { const g = computeGain( {