From 599f9b1b150e34278921403ad66b4a034af01732 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 11:53:46 +0200 Subject: [PATCH 01/10] feat: Keep a session out of the archive when its requests carry X-Cortex-Record: off Part of #1923 (PR 1 of 3: the proxy side). A request that asks (X-Cortex-Record: off) and names its own session with a session-id header stops that session's archiving from then on. Later requests of the session, with or without the header, are not archived either. Nothing already archived is erased: the opt-out never reaches back, and erasing stays explicit (DELETE). - Only a conversation id counts: a request filed by fallback, or one naming a shared or pending bucket, records as before. - The archive persists a content-free marker at the first flag (also while paused), so the opt-out survives restarts, prune, renames and store eviction, and an import never replaces a withheld session. - The header is stripped before forwarding on every listener, including ext_proc and its skip_hosts path. - Praxis mode: the forward proxy marks the session before a hop, and the store tells the archive, so rows that come back through /v1/ingest are withheld too. - The proxy still forwards and runs every plugin. The cost ledger and usage are unchanged. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/cost/ledger/norecord_test.go | 33 +++ core/listener/extproc/norecord_test.go | 87 ++++++++ core/listener/extproc/server.go | 12 +- core/listener/forwardproxy/norecord_test.go | 124 +++++++++++ core/listener/forwardproxy/server.go | 5 + .../internal/sessionevent/sessionevent.go | 5 + core/model/norecord.go | 38 ++++ core/model/norecord_test.go | 58 +++++ core/model/session.go | 14 ++ core/pipeline/context.go | 12 ++ core/pipeline/norecord.go | 53 +++++ core/pipeline/norecord_test.go | 40 ++++ core/pipeline/session_golden_test.go | 1 + .../testdata/session-event.golden.json | 1 + core/session/archive/archive.go | 50 ++++- core/session/archive/clear.go | 1 + core/session/archive/delete.go | 1 + core/session/archive/import.go | 6 + core/session/archive/norecord.go | 119 +++++++++++ core/session/archive/norecord2_test.go | 118 +++++++++++ core/session/archive/norecord_test.go | 200 ++++++++++++++++++ core/session/delete.go | 1 + core/session/fold.go | 19 ++ core/session/history.go | 1 + core/session/norecord_test.go | 97 +++++++++ core/session/store.go | 61 ++++++ core/session/summary.go | 5 + core/sessionapi/summary_test.go | 10 +- 28 files changed, 1166 insertions(+), 6 deletions(-) create mode 100644 core/cost/ledger/norecord_test.go create mode 100644 core/listener/extproc/norecord_test.go create mode 100644 core/listener/forwardproxy/norecord_test.go create mode 100644 core/model/norecord.go create mode 100644 core/model/norecord_test.go create mode 100644 core/pipeline/norecord.go create mode 100644 core/pipeline/norecord_test.go create mode 100644 core/session/archive/norecord.go create mode 100644 core/session/archive/norecord2_test.go create mode 100644 core/session/archive/norecord_test.go create mode 100644 core/session/norecord_test.go diff --git a/core/cost/ledger/norecord_test.go b/core/cost/ledger/norecord_test.go new file mode 100644 index 000000000..39eef2477 --- /dev/null +++ b/core/cost/ledger/norecord_test.go @@ -0,0 +1,33 @@ +package ledger + +import ( + "testing" + "time" +) + +// A session that asked not to be recorded (SessionEvent.NoRecord, #1923) is kept out of the +// session archive only. Its spend still counts here: ledger rows carry no content, and the cost +// views must stay complete. +func TestWriter_CountsAWithheldSession(t *testing.T) { + dir := t.TempDir() + now := at + w := newTestWriter(t, dir, func() time.Time { return now }) + + for i := 0; i < 2; i++ { + ev := costedEvent(t, "gw", "m", 0.25, 100, 50) + ev.NoRecord = true + w.Record("s1", ev) + } + now = at.Add(time.Minute) + later := costedEvent(t, "gw", "m", 0.10, 10, 5) + later.At, later.NoRecord = now, true + w.Record("s1", later) + if err := w.sync(); err != nil { + t.Fatalf("sync: %v", err) + } + + rows := readAllRows(t, dir) + if len(rows) != 1 || rows[0].Requests != 2 || rows[0].CostMicros != 500_000 { + t.Fatalf("rows = %+v, want one closed minute of 2 requests and 500000 micros", rows) + } +} diff --git a/core/listener/extproc/norecord_test.go b/core/listener/extproc/norecord_test.go new file mode 100644 index 000000000..d9c01bb21 --- /dev/null +++ b/core/listener/extproc/norecord_test.go @@ -0,0 +1,87 @@ +package extproc + +import ( + "context" + "net/http" + "slices" + "testing" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + + "github.com/rossoctl/cortex/core/listener/skiphost" + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// removed returns the RemoveHeaders of a request-headers or request-body response. +func removed(t *testing.T, resp *extprocv3.ProcessingResponse) []string { + t.Helper() + var cr *extprocv3.CommonResponse + switch r := resp.Response.(type) { + case *extprocv3.ProcessingResponse_RequestHeaders: + cr = r.RequestHeaders.Response + case *extprocv3.ProcessingResponse_RequestBody: + cr = r.RequestBody.Response + } + if cr == nil || cr.HeaderMutation == nil { + return nil + } + return cr.HeaderMutation.RemoveHeaders +} + +// A skip_hosts destination runs no pipeline, but the header is still for Cortex only: both the +// headers phase and the body phase remove it. +func TestNoRecord_ExtProcSkipHostRemovesTheHeader(t *testing.T) { + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + skip, err := skiphost.New([]string{"otel-collector.*.svc.cluster.local"}) + if err != nil { + t.Fatal(err) + } + srv, _ := newSkipServer(t, store, skip) + hdrs := func() *corev3.HeaderMap { + return makeHeaders(":authority", "otel-collector.ns.svc.cluster.local:8335", ":path", "/v1/traces", + "x-cortex-record", "off") + } + + stream := &mockStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{outboundRequest(hdrs())}} + _ = srv.Process(stream) + if len(stream.responses) != 1 || !slices.Contains(removed(t, stream.responses[0]), "x-cortex-record") { + t.Fatalf("headers phase: responses %d, RemoveHeaders %v, want x-cortex-record removed", + len(stream.responses), removed(t, stream.responses[0])) + } + + resp, pctx := srv.handleOutboundBody(&mockStream{ctx: context.Background()}, hdrs(), []byte(`{}`)) + if pctx != nil { + t.Fatal("body phase ran the pipeline for a skipped host") + } + if got := removed(t, resp); !slices.Contains(got, "x-cortex-record") { + t.Fatalf("body phase: RemoveHeaders %v, want x-cortex-record removed", got) + } +} + +// ext_proc forwards the wire headers and only the changes it emits. ResolveClient removed +// X-Cortex-Record from pctx.Headers before the listener cloned them, so the diff alone cannot +// see it: withHeaderMutation must remove it explicitly, whatever its value. +func TestNoRecord_ExtProcRemovesTheHeader(t *testing.T) { + for _, v := range []string{"off", "on"} { + h := http.Header{} + h.Set(model.RecordHeader, v) + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + orig := pctx.Headers.Clone() // as handleInbound and handleOutbound do + got := removed(t, withHeaderMutation(allowResponse(), pctx, orig)) + if !slices.Contains(got, "x-cortex-record") { + t.Fatalf("value %q: RemoveHeaders = %v, want x-cortex-record", v, got) + } + } + // No header: nothing extra is removed. + pctx := &pipeline.Context{Headers: http.Header{"User-Agent": {"curl/8"}}} + pctx.ResolveClient() + if got := removed(t, withHeaderMutation(allowResponse(), pctx, pctx.Headers.Clone())); slices.Contains(got, "x-cortex-record") { + t.Fatalf("RemoveHeaders = %v with no header sent", got) + } +} diff --git a/core/listener/extproc/server.go b/core/listener/extproc/server.go index 3ea7c6de8..f398be0a0 100644 --- a/core/listener/extproc/server.go +++ b/core/listener/extproc/server.go @@ -26,6 +26,7 @@ import ( "github.com/rossoctl/cortex/core/listener/internal/sessionevent" "github.com/rossoctl/cortex/core/listener/internal/sseframe" "github.com/rossoctl/cortex/core/listener/skiphost" + "github.com/rossoctl/cortex/core/model" "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/session" ) @@ -504,7 +505,9 @@ func (s *Server) handleOutbound(stream extprocv3.ExternalProcessor_ProcessServer // handleResponseBody) and the deferred RunFinish to no-op as well — // all four phases are skipped consistently. See ListenerConfig.SkipHosts. if s.SkipHosts.Match(pctx.Host) { - return passResponse(), nil + // No pipeline ran, so pctx.Headers is its own original and only the X-Cortex-Record + // removal is emitted: the header is for Cortex only, also toward a skipped host (#1923). + return withHeaderMutation(passResponse(), pctx, pctx.Headers), nil } if s.Sessions != nil { @@ -555,7 +558,7 @@ func (s *Server) handleOutboundBody(stream extprocv3.ExternalProcessor_ProcessSe if pat, matched := s.SkipHosts.MatchPattern(pctx.Host); matched { slog.Info("ext_proc: skip_hosts match (body phase) — bypassing pipeline + session recording", "host", pipeline.SanitizeLogValue(pctx.Host), "pattern", pat, "path", pipeline.SanitizeLogValue(pctx.Path)) - return allowBodyResponse(), nil + return withHeaderMutation(allowBodyResponse(), pctx, pctx.Headers), nil // as in handleOutbound } if s.Sessions != nil { @@ -1008,6 +1011,11 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte del = append(del, strings.ToLower(k)) // plugin removed it } } + // X-Cortex-Record is for Cortex only (#1923). ResolveClient removed it from pctx.Headers + // before orig was cloned, so the loop above cannot see it: remove it from the wire here. + if pctx.RecordHeaderSeen() { + del = append(del, strings.ToLower(model.RecordHeader)) + } if len(set) == 0 && len(del) == 0 { return resp } diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go new file mode 100644 index 000000000..1df8bf200 --- /dev/null +++ b/core/listener/forwardproxy/norecord_test.go @@ -0,0 +1,124 @@ +package forwardproxy + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// row is what a recorder saw of one event. +type row struct { + session string + phase pipeline.SessionPhase + noRecord bool +} + +// rows records every event's session, phase and NoRecord. +type rows struct { + mu sync.Mutex + got []row +} + +func (r *rows) Record(sid string, e *pipeline.SessionEvent) { + r.mu.Lock() + defer r.mu.Unlock() + r.got = append(r.got, row{sid, e.Phase, e.NoRecord}) +} + +func (r *rows) take() []row { + r.mu.Lock() + defer r.mu.Unlock() + out := r.got + r.got = nil + return out +} + +func noRecordProxy(t *testing.T) (proxy *httptest.Server, backendURL string, rec *rows, seen *[]string) { + t.Helper() + var got []string + var mu sync.Mutex + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + got = append(got, r.Header.Get(model.RecordHeader)) + mu.Unlock() + })) + t.Cleanup(backend.Close) + p, err := pipeline.New(nil) + if err != nil { + t.Fatal(err) + } + store := session.New(5*time.Minute, 100, 0) + t.Cleanup(store.Close) + rec = &rows{} + store.AddRecorder(rec) + srv := &Server{OutboundPipeline: pipeline.NewHolder(p), Sessions: store, Client: http.DefaultClient, + SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}} + proxy = httptest.NewServer(srv.Handler()) + t.Cleanup(proxy.Close) + return proxy, backend.URL, rec, &got +} + +// A request that names its own session and carries X-Cortex-Record: off is forwarded without the +// header, and both of its rows carry NoRecord. +func TestNoRecord_OwnSessionIsWithheldAndHeaderStripped(t *testing.T) { + proxy, backend, rec, seen := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{ + model.RecordHeader: {"off"}, + session.ClaudeCodeSessionHeader: {"11111111-2222-3333-4444-555555555555"}, + }) + if len(*seen) != 1 || (*seen)[0] != "" { + t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) + } + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if !r.noRecord || r.session != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("row %+v, want NoRecord under the request's own session", r) + } + } +} + +// A request that names no session is filed by fallback (the active session or the default +// bucket), so its flag must not withhold that bucket: it is recorded as usual. The header is +// still stripped. +func TestNoRecord_FallbackFiledRequestIsRecorded(t *testing.T) { + proxy, backend, rec, seen := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{model.RecordHeader: {"off"}}) + if len(*seen) != 1 || (*seen)[0] != "" { + t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) + } + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if r.noRecord { + t.Fatalf("row %+v under a fallback bucket carries NoRecord", r) + } + } +} + +// A request without the header is unchanged. +func TestNoRecord_UnflaggedRequestIsRecorded(t *testing.T) { + proxy, backend, rec, _ := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{ + session.ClaudeCodeSessionHeader: {"11111111-2222-3333-4444-555555555555"}, + }) + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if r.noRecord { + t.Fatalf("row %+v carries NoRecord with no header", r) + } + } +} diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index b43ea0250..a3d2b098d 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -733,6 +733,11 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // carries StatusCode). skip_hosts traffic never reaches here (the // !skipped guard above), so it stays suppressed by design. if hopped { + // The gateway records this request and its rows come back through /v1/ingest + // without the flag (#1923), so the store stamps them. + if pctx.WithholdsSession() { + s.Sessions.MarkNoRecord(sid) + } s.Hop.own(pctx, hopWire(pctx, r.Header, sid)) defer s.Hop.release(pctx) } else { diff --git a/core/listener/internal/sessionevent/sessionevent.go b/core/listener/internal/sessionevent/sessionevent.go index 2293fecae..c2f13bf27 100644 --- a/core/listener/internal/sessionevent/sessionevent.go +++ b/core/listener/internal/sessionevent/sessionevent.go @@ -77,6 +77,7 @@ func Deny(pctx *pipeline.Context, action pipeline.Action, dir pipeline.Direction SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), TLS: TLS(pctx), } } @@ -121,6 +122,7 @@ func Request(pctx *pipeline.Context, dir pipeline.Direction) pipeline.SessionEve SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), ClientProc: pctx.AttributedClientProc(), SessionHeader: pctx.NamingHeader(), TLS: TLS(pctx), @@ -162,6 +164,7 @@ func Response(pctx *pipeline.Context, dir pipeline.Direction, status int, fail * SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), TLS: TLS(pctx), } } @@ -208,6 +211,7 @@ func TunnelOpen(pctx *pipeline.Context, reason pipeline.TunnelReason) pipeline.S SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), } } @@ -243,6 +247,7 @@ func TunnelClose(pctx *pipeline.Context, reason pipeline.TunnelReason, status in SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), } } diff --git a/core/model/norecord.go b/core/model/norecord.go new file mode 100644 index 000000000..21ccb42e9 --- /dev/null +++ b/core/model/norecord.go @@ -0,0 +1,38 @@ +package model + +import ( + "net/http" + "strings" +) + +// RecordHeader is the request header a client sends to keep its session out of the +// session archive (#1923). RecordOff is the one value that does so. +// +// For Cortex only: a listener reads it before any plugin runs and removes it from the +// request it forwards, so the upstream never sees it. Named like the other X-Cortex-* +// headers. It is not a session-id header and is never in session.id_headers. +// +// Claude Code sends it from ANTHROPIC_CUSTOM_HEADERS, e.g. +// ANTHROPIC_CUSTOM_HEADERS="X-Cortex-Record: off". +const RecordHeader = "X-Cortex-Record" + +// RecordOff is the value of RecordHeader that withholds a session from the archive. +const RecordOff = "off" + +// RecordsOff reports whether h asks not to be recorded: RecordHeader is present with the +// value RecordOff, compared without case and padding. Any other value, or none, records as +// today, so a typo can never hide a session by accident. Several values: any one "off" counts, +// because the header can only withhold. +func RecordsOff(h http.Header) bool { + if h == nil { + return false + } + for _, v := range h.Values(RecordHeader) { + for _, part := range strings.Split(v, ",") { + if strings.EqualFold(strings.TrimSpace(part), RecordOff) { + return true + } + } + } + return false +} diff --git a/core/model/norecord_test.go b/core/model/norecord_test.go new file mode 100644 index 000000000..43fe63170 --- /dev/null +++ b/core/model/norecord_test.go @@ -0,0 +1,58 @@ +package model + +import ( + "net/http" + "strings" + "testing" +) + +func TestRecordsOff(t *testing.T) { + cases := []struct { + name string + vals []string + want bool + }{ + {"absent", nil, false}, + {"off", []string{"off"}, true}, + {"any case and padding", []string{" OFF "}, true}, + {"one of a list", []string{"on, off"}, true}, + {"one of several values", []string{"on", "off"}, true}, + {"on", []string{"on"}, false}, + {"a typo records", []string{"of"}, false}, + {"empty", []string{""}, false}, + {"false is not off", []string{"false"}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := http.Header{} + for _, v := range tc.vals { + h.Add(RecordHeader, v) + } + if got := RecordsOff(h); got != tc.want { + t.Fatalf("RecordsOff(%q) = %v, want %v", tc.vals, got, tc.want) + } + }) + } + if RecordsOff(nil) { + t.Fatal("RecordsOff(nil) = true") + } +} + +// NoRecord round-trips through the wire form and is absent from a recorded row. +func TestSessionEvent_NoRecordWire(t *testing.T) { + b, err := SessionEvent{NoRecord: true}.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var back SessionEvent + if err := back.UnmarshalJSON(b); err != nil { + t.Fatal(err) + } + if !back.NoRecord { + t.Fatalf("NoRecord lost in %s", b) + } + plain, _ := SessionEvent{}.MarshalJSON() + if got := string(plain); strings.Contains(got, "noRecord") { + t.Fatalf("a recorded row carries the key: %s", got) + } +} diff --git a/core/model/session.go b/core/model/session.go index 6f3d1539b..a445f3e50 100644 --- a/core/model/session.go +++ b/core/model/session.go @@ -447,6 +447,16 @@ type SessionEvent struct { // Use ImportedSource to make the value and Imported to test it. Source string + // NoRecord is true when the request asked not to be recorded: it carried + // RecordHeader with the value RecordOff (#1923). Read by Context.NoRecord, pinned + // before any plugin runs. The proxy still forwards the request and runs its plugins; + // the session archive keeps none of the session's events. + // + // Client-asserted, with the same caveat as Client above. It can only withhold a + // session from the archive, never put one there, so a forged value costs its sender + // their own recording. + NoRecord bool + // SubagentKind is the kind of a Bob child task, from Context.SubagentKind: // SubagentKindSubtask or SubagentKindSubagent. Set only on a Bob child's rows, and // empty everywhere else, Claude Code rows included (read Inference.AgentRole there). @@ -715,6 +725,8 @@ type sessionEventWire struct { ParentSubagent string `json:"parentSubagent,omitempty"` // omitempty: absent on every live event. Source LenientString `json:"source,omitempty"` + // omitempty: absent on every recorded row, which is every row the archive keeps. + NoRecord bool `json:"noRecord,omitempty"` // omitempty: absent on every row but a Bob child task's. SubagentKind string `json:"subagentKind,omitempty"` // omitempty: absent wherever the listener could not name the client's process, and on @@ -761,6 +773,7 @@ func (e SessionEvent) MarshalJSON() ([]byte, error) { Subagent: e.Subagent, ParentSubagent: e.ParentSubagent, Source: LenientString(e.Source), + NoRecord: e.NoRecord, SubagentKind: e.SubagentKind, Proc: e.Proc, ClientProc: e.ClientProc, @@ -809,6 +822,7 @@ func (e *SessionEvent) UnmarshalJSON(data []byte) error { Subagent: w.Subagent, ParentSubagent: w.ParentSubagent, Source: string(w.Source), + NoRecord: w.NoRecord, SubagentKind: w.SubagentKind, Proc: w.Proc, ClientProc: w.ClientProc, diff --git a/core/pipeline/context.go b/core/pipeline/context.go index c37f87d02..566458742 100644 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -171,6 +171,14 @@ type Context struct { // bobChild is true when subagent came from Bob's X-Task-Id, not Claude Code's header. bobChild bool + // noRecord and noRecordParsed memoize NoRecord's answer, on the same terms as client + // and clientParsed: filled by ResolveClient before any plugin runs. + noRecord bool + noRecordParsed bool + // recordHeaderSeen is true when the client sent model.RecordHeader with any value, so a + // listener that forwards by explicit mutation (ext_proc) knows to remove it. + recordHeaderSeen bool + // attributedProc, namingHeader and attributed hold SetAttribution's pin. attributedProc string clientProc string @@ -547,10 +555,14 @@ func (c *Context) ClientInfo() *EventClient { // X-Claude-Code-Agent-Id a plugin rewrote must not re-file a row under another subagent, // and recorders reading the memo must not be the ones filling it. // +// It pins NoRecord too, which also removes model.RecordHeader from Headers, so no plugin +// sees it and the forwarded request does not carry it. +// // Idempotent: the second call is the memo's own no-op. func (c *Context) ResolveClient() { _ = c.ClientInfo() _ = c.Subagent() + _ = c.NoRecord() } // SetSessionFacts is for listeners: pass what session.Store.ViewFacts returned. A zero diff --git a/core/pipeline/norecord.go b/core/pipeline/norecord.go new file mode 100644 index 000000000..f114940a1 --- /dev/null +++ b/core/pipeline/norecord.go @@ -0,0 +1,53 @@ +package pipeline + +import "github.com/rossoctl/cortex/core/model" + +// NoRecord reports whether the request asked not to be recorded (#1923): it carried +// model.RecordHeader set to model.RecordOff. Recorded as SessionEvent.NoRecord on every row +// the request produces, and the session archive keeps none of that session's events. +// +// The first call also deletes model.RecordHeader from Headers. The header is for Cortex +// only: plugins never see it, and the listeners that send the request on from Headers +// (httpx.SyncRequestHeaders) do not forward it. ResolveClient makes that first call at +// construction, before any plugin runs, so a plugin cannot set or clear it either. +// +// Memoized and pinned by ResolveClient on exactly the terms ClientInfo is; see there. +// +// CLIENT-ASSERTED AND UNAUTHENTICATED, like Subagent. It can only withhold the sender's own +// session from the archive. +func (c *Context) NoRecord() bool { + if c.noRecordParsed { + return c.noRecord + } + c.noRecordParsed = true + if c.Headers == nil { + return false + } + c.noRecord = model.RecordsOff(c.Headers) + c.recordHeaderSeen = len(c.Headers.Values(model.RecordHeader)) > 0 + c.Headers.Del(model.RecordHeader) + return c.noRecord +} + +// RecordHeaderSeen reports whether the client sent model.RecordHeader, with any value. NoRecord +// removed it from Headers, which is enough for a listener that forwards from Headers. One that +// forwards the wire headers and emits changes (ext_proc) must remove it explicitly. +func (c *Context) RecordHeaderSeen() bool { + _ = c.NoRecord() + return c.recordHeaderSeen +} + +// WithholdsSession reports whether this request's session is to be withheld from the archive: +// it asked not to be recorded (NoRecord) AND it names its own session with a session-id header +// (NamingHeader, pinned by SetAttribution). This is what listeners record as +// SessionEvent.NoRecord. +// +// THE OWN-ID RULE IS THE SAFETY. A request that names no session is filed by fallback, to the +// active session or the shared default bucket (attribution.FiledSession), so honouring its flag +// would stop recording a session that is not the sender's. Such a request is recorded as if it +// had not asked; its header is still stripped. A request whose header names another session can +// still stop that session's recording from then on, but never erase what is already archived: +// the session-id header is client-asserted, with session.IDFromHeaders' TRUST caveat. +func (c *Context) WithholdsSession() bool { + return c.NoRecord() && c.namingHeader != "" +} diff --git a/core/pipeline/norecord_test.go b/core/pipeline/norecord_test.go new file mode 100644 index 000000000..c3c9ec534 --- /dev/null +++ b/core/pipeline/norecord_test.go @@ -0,0 +1,40 @@ +package pipeline + +import ( + "net/http" + "testing" + + "github.com/rossoctl/cortex/core/model" +) + +// ResolveClient pins NoRecord from the headers as the client sent them, and removes the header +// so no plugin sees it and the forwarded request does not carry it. +func TestNoRecord_PinnedAndStripped(t *testing.T) { + h := http.Header{} + h.Set(model.RecordHeader, "off") + h.Set("User-Agent", "claude-cli/2.1.266 (external, cli)") + c := &Context{Headers: h} + c.ResolveClient() + if !c.NoRecord() { + t.Fatal("NoRecord = false for X-Cortex-Record: off") + } + if _, ok := c.Headers[http.CanonicalHeaderKey(model.RecordHeader)]; ok { + t.Fatal("the header is still in Headers after ResolveClient") + } + // A plugin that adds it back later changes nothing: the answer was pinned. + c.Headers.Set(model.RecordHeader, "on") + if !c.NoRecord() { + t.Fatal("a later header change unpinned NoRecord") + } +} + +func TestNoRecord_AbsentRecords(t *testing.T) { + c := &Context{Headers: http.Header{"User-Agent": {"curl/8"}}} + c.ResolveClient() + if c.NoRecord() { + t.Fatal("NoRecord = true with no header") + } + if (&Context{}).NoRecord() { + t.Fatal("NoRecord = true with nil Headers") + } +} diff --git a/core/pipeline/session_golden_test.go b/core/pipeline/session_golden_test.go index f3138681d..94892f434 100644 --- a/core/pipeline/session_golden_test.go +++ b/core/pipeline/session_golden_test.go @@ -84,6 +84,7 @@ func goldenEvent() SessionEvent { ClientProc: "/bin/zsh", Project: "~/src/app", Source: "imported:claude-code", + NoRecord: true, SessionHeader: "x-session-id", DialAddr: "203.0.113.7:443", } diff --git a/core/pipeline/testdata/session-event.golden.json b/core/pipeline/testdata/session-event.golden.json index 2d30799d0..d80852ce5 100644 --- a/core/pipeline/testdata/session-event.golden.json +++ b/core/pipeline/testdata/session-event.golden.json @@ -197,6 +197,7 @@ "subagent": "a1a2e36cb5946f31c", "parentSubagent": "a0974b0b8591dc76b", "source": "imported:claude-code", + "noRecord": true, "subagentKind": "subtask", "proc": "/bin/claude", "clientProc": "/bin/zsh", diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 0de17e37e..7c6958b33 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -131,7 +131,12 @@ type Archive struct { // recent clear, for AwaitClear; see clear.go. mu sync.Mutex lastSeq map[string]uint64 - pending *pendingClear + // noRecord holds, for each id a flagged event named (SessionEvent.NoRecord), the lowest Seq + // it carried. RecordCapture sets it before the event can be dropped, so the session is + // withheld even when that event never reaches the writer, and from that Seq on only: events + // queued before it are still written. See withhold. Under mu. + noRecord map[string]uint64 + pending *pendingClear // deletes is the most recent delete per session id, for AwaitDelete; see delete.go. nil // until the first. deletes map[string]*pendingClear @@ -203,6 +208,8 @@ const ( opClear opDelete opFunc + // opNoRecord marks a session as withheld (#1923), queued with its first flagged event. + opNoRecord ) // op is one unit of work for the writer goroutine. Events and renames share one FIFO, so a @@ -234,6 +241,7 @@ func Open(root string, opts ...Option) (*Archive, error) { maxOpen: defaultMaxOpen, tickEvery: defaultTickEvery, lastSeq: map[string]uint64{}, + noRecord: map[string]uint64{}, started: map[string]entryStart{}, sessions: map[string]*sessionState{}, index: map[string]*indexEntry{}, @@ -340,6 +348,9 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq } + if e.NoRecord { + a.markLocked(sessionID, e.Seq) + } // Every send happens under the store's write lock, so this check and the send below cannot // race another producer; the writer only ever makes room. if len(a.ops) >= cap(a.ops)-reserve { @@ -347,7 +358,9 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode return } o := op{kind: opEvent, id: sessionID, ev: *e} - if c != nil { + // A request that asked not to be recorded queues no bodies: the writer withholds its + // session anyway (withhold), so the copy would only take capture budget from others. + if c != nil && !e.NoRecord { if n := captureSize(c); a.captureBytes.Load()+n <= a.captureBudget { a.captureBytes.Add(n) o.cap = c @@ -525,6 +538,10 @@ func (a *Archive) take(o op) { case opFunc: o.f() close(o.done) + case opNoRecord: + // Also while paused: the mark is one small session.json, and it is what keeps the + // session out after a restart. + a.withhold(o.id, a.sessions[o.id], a.now()) } } @@ -555,6 +572,10 @@ func (a *Archive) write(id string, e *model.SessionEvent, c *model.RequestCaptur } now := a.now() s := a.sessions[id] + if a.withheld(id, s, e) { + a.withhold(id, s, now) + return + } fresh := s == nil if fresh { s = &sessionState{ @@ -670,6 +691,16 @@ func (a *Archive) writeFailed(s *sessionState, err error) { // id under the old directory, which adopt finishes at the next start. func (a *Archive) rename(oldID, newID string, after uint64) { s := a.sessions[oldID] + // A withheld session stays withheld under its new id (#1923), on every path below, the early + // returns included: the store's entry moves to newID whatever happens to the files here. + a.mu.Lock() + _, flaggedOld := a.noRecord[oldID] + marked := flaggedOld || (s != nil && s.meta.Summary.NoRecord()) + delete(a.noRecord, oldID) + if marked { + a.markLocked(newID, 0) // every event under the new id + } + a.mu.Unlock() if s == nil { return } @@ -684,6 +715,10 @@ func (a *Archive) rename(oldID, newID string, after uint64) { "from", oldID, "to", newID) return } + if marked { + // A flag only in memory (its event not written yet) moves with the session's files too. + s.meta.Summary.MarkNoRecord() + } if after > 0 && a.renameEntry(s, oldID, newID, newDir, after) { return } @@ -719,10 +754,15 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte } startAfter, before := s.startAfter, s.before s.startAfter, s.before = 0, nil + // Both folds below are rebuilt from segments, which do not hold the mark (#1923). + noRec := s.meta.Summary.NoRecord() if entry == len(segs) { if startAfter == after { a.closeWriter(s) s.meta.Summary, _ = foldSegments(s.dir, s.meta.Segments, oldID) + if noRec { + s.meta.Summary.MarkNoRecord() + } } else { // The fold also holds the history's events, which stay under oldID. The new id must // not take the history's hosts as its own. @@ -758,6 +798,9 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte meta: &sessionMeta{ID: newID, CreatedAt: createdAt, UpdatedAt: latestWrite(move), Summary: since, Segments: move}, lastWrite: s.lastWrite, } + if noRec { + ns.meta.Summary.MarkNoRecord() + } // session.json first: a crash before a segment moves leaves it listed where it is not, which // reconcile drops, rather than a directory loadMetas cannot read. if err := writeMeta(newDir, ns.meta); err != nil { @@ -900,7 +943,7 @@ func (a *Archive) prune(now time.Time) { } var gone []*sessionState for _, s := range a.sessions { - if s.w == nil && len(s.meta.Segments) == 0 { + if s.w == nil && len(s.meta.Segments) == 0 && !keepMarker(s, now, a.retainDays) { gone = append(gone, s) } } @@ -919,6 +962,7 @@ func (a *Archive) prune(now time.Time) { if a.lastSeq[id] <= lastBefore[s] { delete(a.lastSeq, id) } + delete(a.noRecord, id) // an expired marker, or a session whose history expired a.mu.Unlock() } for _, s := range a.sessions { diff --git a/core/session/archive/clear.go b/core/session/archive/clear.go index e0c24b1c3..c55c09dcc 100644 --- a/core/session/archive/clear.go +++ b/core/session/archive/clear.go @@ -53,6 +53,7 @@ func (a *Archive) Cleared() { a.mu.Lock() p.seqs = maps.Clone(a.lastSeq) a.pending = p + clear(a.noRecord) a.mu.Unlock() if a.closed.Load() { p.finish(ClearResult{}, errors.New("the session archive is closed")) diff --git a/core/session/archive/delete.go b/core/session/archive/delete.go index ed53b0a6f..fda2f35e3 100644 --- a/core/session/archive/delete.go +++ b/core/session/archive/delete.go @@ -22,6 +22,7 @@ func (a *Archive) Deleted(sessionID string) { a.mu.Lock() p.seqs = map[string]uint64{sessionID: a.lastSeq[sessionID]} delete(a.started, sessionID) + delete(a.noRecord, sessionID) if a.deletes == nil { a.deletes = map[string]*pendingClear{} } diff --git a/core/session/archive/import.go b/core/session/archive/import.go index 2269f8173..df360a9a3 100644 --- a/core/session/archive/import.go +++ b/core/session/archive/import.go @@ -65,7 +65,13 @@ func (a *Archive) writeSession(id string, events []model.SessionEvent, replace b // after the import (SeqSeeder), never from 1. a.mu.Lock() old := a.lastSeq[id] + _, flagged := a.noRecord[id] switch { + case flagged || (s != nil && s.meta.Summary.NoRecord()): + // The session asked not to be recorded (#1923). Its marker is the user's own choice, + // so no import adds to it or replaces it, also one whose events were imported first. + a.mu.Unlock() + return ErrLive case s == nil && old > 0: a.mu.Unlock() return ErrLive diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go new file mode 100644 index 000000000..c5345f79c --- /dev/null +++ b/core/session/archive/norecord.go @@ -0,0 +1,119 @@ +package archive + +import ( + "log/slog" + "path/filepath" + "time" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/session" +) + +// withhold keeps session id out of the archive from now on (#1923). It runs on the writer +// goroutine for an event that asked not to be recorded (SessionEvent.NoRecord, which listeners +// set only for a request that names its own session), and for every later event of a session +// already withheld, flagged or not: a session's header-less calls (telemetry, say) are filed to +// it by process or client affinity and carry no flag. +// +// FROM NOW ON, NEVER BACKWARDS. What the archive already holds for id stays: it was recorded +// before the opt-out, and the session-id header that names id is client-asserted, so a request +// must not be able to erase another session's history by naming it. The open segment is closed, +// so nothing more is written to it, and the session is marked. Erasing is the user's explicit +// DELETE /v1/sessions/{id}. +// +// For an id the archive does not hold, withhold writes a content-free marker: a session.json with +// no segments whose summary says noRecord and nothing else. The event itself is never folded, +// because the fold reads content (the title comes from the prompt). The mark is what keeps the +// opt-out after a restart, when the store's own mark is gone, and what makes an import refuse id +// (WriteSession returns ErrLive for an id the archive holds that is not imported). +// +// noRecord ids are also remembered in a.noRecord, under a.mu, by RecordCapture: an event dropped +// before the writer saw it (queue full, archive paused) must not leave the session unmarked. +func (a *Archive) withhold(id string, s *sessionState, now time.Time) { + if s != nil && s.meta.Summary.NoRecord() { + // Already marked. A marker's age is measured from the session's last event + // (keepMarker), so it is renewed, and written with the next metaEvery. + s.meta.UpdatedAt, s.metaDirty = now, true + return + } + if s == nil { + fold := session.NewSummaryFold() + s = &sessionState{ + dir: filepath.Join(a.data, dirName(id)), + meta: &sessionMeta{ID: id, CreatedAt: now, Summary: fold}, + } + a.sessions[id] = s + } else { + a.closeWriter(s) + } + s.meta.Summary.MarkNoRecord() + s.meta.UpdatedAt, s.lastWrite = now, now + a.writeMeta(s) + a.publish(s) +} + +// Withhold implements session.Withholder: the store tells the archive that sessionID is withheld +// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). It runs under +// the store's write lock, and never blocks. +func (a *Archive) Withhold(sessionID string) { + if a.closed.Load() { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.markLocked(sessionID, a.lastSeq[sessionID]+1) +} + +// markLocked marks sessionID withheld from Seq from on, in memory at once (a.noRecord), and on +// disk through an opNoRecord for the session's first mark, so a restart before its next event +// keeps it. a.mu held. The op takes the queue's normal capacity, never the reserve that renames, +// deletes and clears rely on: when the queue is full the mark stays in memory, and the session's +// next event writes it (withhold). +func (a *Archive) markLocked(sessionID string, from uint64) { + if !session.IsConversation(sessionID) { + return + } + old, ok := a.noRecord[sessionID] + if !ok || from < old { + a.noRecord[sessionID] = from + } + if ok { + return + } + if len(a.ops) >= cap(a.ops)-reserve { + slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event", + "session", sessionID) + return + } + select { + case a.ops <- op{kind: opNoRecord, id: sessionID}: + default: + slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event", + "session", sessionID) + } +} + +// withheld reports whether e, an event of id, is to be withheld: it asks it, or the session was +// marked on disk, or a flagged event the writer may not have seen yet came at or before it. +// Events numbered before that flagged one are still written: the opt-out never reaches back. +func (a *Archive) withheld(id string, s *sessionState, e *model.SessionEvent) bool { + // Never a shared bucket: the store already clears the flag there, and the archive does + // not trust that alone (session.IsConversation). + if !session.IsConversation(id) { + return false + } + if e.NoRecord || (s != nil && s.meta.Summary.NoRecord()) { + return true + } + a.mu.Lock() + defer a.mu.Unlock() + from, ok := a.noRecord[id] + return ok && e.Seq >= from +} + +// keepMarker reports whether prune keeps s, a session with nothing on disk, because it marks a +// withheld session. A marker lives retainDays after the session's last event, like a segment: +// after that, the agent's own transcript has usually expired too. Open makes retainDays positive. +func keepMarker(s *sessionState, now time.Time, retainDays int) bool { + return s.meta.Summary.NoRecord() && now.Sub(s.meta.UpdatedAt) < time.Duration(retainDays)*24*time.Hour +} diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go new file mode 100644 index 000000000..dbd54db60 --- /dev/null +++ b/core/session/archive/norecord2_test.go @@ -0,0 +1,118 @@ +package archive + +import ( + "errors" + "testing" + "time" + + "github.com/rossoctl/cortex/core/session" +) + +// Review round 2, item 1: an imported session that a live request later withholds is never +// replaced by an import, also after a restart, when its Source still says imported and its +// numbering matches the disk. +func TestNoRecord_AnImportNeverReplacesAWithheldImportedSession(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + if err := a.WriteSession("s1", importedEvents(4), false); err != nil { + t.Fatalf("import: %v", err) + } + r := newRecorder(a) + r.seq["s1"] = 4 + r.record("s1", flagged(synthSession(320, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + defer a2.Close() + if err := a2.WriteSession("s1", importedEvents(6), true); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(replace) on a withheld imported session = %v, want ErrLive", err) + } +} + +// Review round 2, item 3: the default bucket and a pending bucket are never withheld, even when a +// flagged event reaches the archive under their id. They hold many clients' calls. +func TestNoRecord_SharedBucketsAreNeverWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + for _, id := range []string{session.DefaultSessionID, session.PendingSessionID("claude-code")} { + r.record(id, flagged(synthSession(321, 1, 512))...) + } + a.Close() + for _, id := range []string{session.DefaultSessionID, session.PendingSessionID("claude-code")} { + if got := onDisk(t, root, id); len(got) != 3 { + t.Fatalf("%s holds %d events, want all 3 recorded", id, len(got)) + } + } +} + +// Review round 3, item 1: praxis mode. The store mark for a hopped request (MarkNoRecord) is +// dropped when the store evicts the entry, and the gateway's ingested rows carry no flag. The +// archive must already hold its own mark by then, so those rows are withheld. +func TestNoRecord_HopMarkSurvivesEviction(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + st := session.New(time.Hour, 0, 1) // one session: the next one evicts the first + defer st.Close() + st.AddRecorder(a) + evs := synthSession(330, 2, 512) + for _, e := range evs[:3] { + st.Append("s1", e) // recorded before the opt-out + } + st.MarkNoRecord("s1") // a hopped request with X-Cortex-Record: off + st.Append("s2", synthSession(331, 1, 512)[0]) + for _, e := range evs[3:] { + st.Append("s1", e) // the gateway's rows, ingested without the flag + } + settle(a) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != 3 { + t.Fatalf("s1 holds %d events, want the 3 recorded before the mark", len(got)) + } +} + +// Review round 3, item 4: a flag the writer has not seen yet, held only in memory (its event and +// its opNoRecord lost to a full queue), still follows a rename to the new id. +func TestNoRecord_AMemoryOnlyFlagFollowsARename(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + a.gate = make(chan struct{}) + r := newRecorder(a) + evs := synthSession(332, 3, 512) // 9 events + r.record("s1", evs[:6]...) // fills the queue to the reserve + r.record("s1", flagged(evs[6:7])...) // dropped, and its opNoRecord does not fit + a.Rekeyed("s1", "s2") // into the reserve + close(a.gate) + settle(a) + r.seq["s2"] = r.seq["s1"] + r.record("s2", evs[7:]...) + a.Close() + if got := onDisk(t, root, "s2"); len(got) != 6 { + t.Fatalf("s2 holds %d events, want the 6 recorded before the flag", len(got)) + } + a2 := openTest(t, root, newClock()) + defer a2.Close() + if err := a2.WriteSession("s2", importedEvents(4), true); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(s2) after the rename = %v, want ErrLive: the mark was not persisted", err) + } +} + +// Review round 2, item 4: a flagged event dropped while the archive is paused still persists the +// mark, so a restart before the session's next event keeps it withheld. +func TestNoRecord_ADroppedFlagIsPersistedBeforeARestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + a.paused.Store(true) + newRecorder(a).record("s1", flagged(synthSession(322, 1, 512)[:1])...) + settle(a) + a.Close() + + a2 := openTest(t, root, newClock()) + r := newRecorder(a2) + r.seq["s1"] = 1 + r.record("s1", synthSession(323, 1, 512)...) + a2.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none", segs) + } +} diff --git a/core/session/archive/norecord_test.go b/core/session/archive/norecord_test.go new file mode 100644 index 000000000..224c76395 --- /dev/null +++ b/core/session/archive/norecord_test.go @@ -0,0 +1,200 @@ +package archive + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// flagged marks every event as asking not to be recorded, as the listeners do when a request +// carries X-Cortex-Record: off. +func flagged(evs []pipeline.SessionEvent) []pipeline.SessionEvent { + for i := range evs { + evs[i].NoRecord = true + } + return evs +} + +// segmentsIn lists the segment files under id's directory. +func segmentsIn(t *testing.T, root, id string) []string { + t.Helper() + entries, err := os.ReadDir(filepath.Join(root, dataDirName, dirName(id))) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + var out []string + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".jsonl.zst") { + out = append(out, e.Name()) + } + } + return out +} + +// A flagged session writes no segment, only a session.json that marks it as not recorded and +// holds nothing else: no events, no title, no tokens. +func TestNoRecord_WritesOnlyAContentFreeMarker(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(301, 2, 512))...) + a.Close() + + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatalf("no marker: %v", err) + } + sum := m.Summary.Summary("s1") + if !sum.NoRecord || sum.EventCount != 0 || sum.Title != "" || sum.TotalTokens != 0 || len(m.Segments) != 0 { + t.Fatalf("marker = %+v with %d segments, want noRecord and nothing else", sum, len(m.Segments)) + } +} + +// Header-less events filed to a withheld session carry no flag. They are withheld too. +func TestNoRecord_LaterUnflaggedEventsAreWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(302, 2, 512) + r.record("s1", flagged(evs[:2])...) + r.record("s1", evs[2:]...) + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } +} + +// The opt-out applies from the first flagged event on and never backwards: what the archive +// already held stays (a session-id header is client-asserted, so a request naming another +// session must not erase it), and nothing after the flag is added. +func TestNoRecord_KeepsWhatCameBeforeTheFlagAndAddsNothing(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(303, 2, 512) + r.record("s1", evs[:2]...) + r.record("s1", flagged(evs[2:3])...) + r.record("s1", evs[3:]...) + a.Close() + got := onDisk(t, root, "s1") + if len(got) != 2 || got[0].Seq != 1 || got[1].Seq != 2 { + t.Fatalf("s1 holds %d events, want the 2 recorded before the flag", len(got)) + } + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatal(err) + } + if !m.Summary.NoRecord() { + t.Fatal("the session is not marked after the flag") + } +} + +// A withheld session stays withheld when the store renames it, on the path that rebuilds the +// fold from segments (an entry re-created after the marker, then renamed whole). +func TestNoRecord_RenameKeepsTheMark(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(309, 1, 512))...) + settle(a) + a.inWriter(func() { a.begin("s1", 3) }) + a.inWriter(func() { a.rename("s1", "s2", 3) }) + // Right after the rename and before any event of s2, so the in-memory mark cannot stand in + // for a fold that lost it: the summary under the new id, and session.json on disk. + var marked bool + a.inWriter(func() { s := a.sessions["s2"]; marked = s != nil && s.meta.Summary.NoRecord() }) + if !marked { + t.Fatal("the renamed session's summary is not marked") + } + if m, err := readMeta(filepath.Join(root, dataDirName, dirName("s2"))); err != nil || !m.Summary.NoRecord() { + t.Fatalf("session.json under the new id: %v, marked %v", err, err == nil && m.Summary.NoRecord()) + } + r.seq["s2"] = 3 + r.record("s2", synthSession(310, 1, 512)...) + a.Close() + if segs := segmentsIn(t, root, "s2"); len(segs) != 0 { + t.Fatalf("segments %v under the new id, want none", segs) + } + a2 := openTest(t, root, newClock()) + defer a2.Close() + if err := a2.WriteSession("s2", importedEvents(4), false); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(s2) after the rename = %v, want ErrLive", err) + } +} + +// A flagged event the writer never sees (here, while the archive is paused) still marks the +// session, so the header-less events after it are withheld. +func TestNoRecord_ADroppedFlaggedEventStillMarks(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(311, 1, 512) + a.paused.Store(true) + r.record("s1", flagged(evs[:1])...) + settle(a) + a.paused.Store(false) + r.record("s1", evs[1:]...) + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } +} + +// The marker keeps the opt-out across a restart, when the store's own mark is gone: an +// unflagged event for the id is still withheld. +func TestNoRecord_SurvivesARestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(304, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + r := newRecorder(a2) + r.seq["s1"] = a2.LastSeq("s1") + r.record("s1", synthSession(305, 1, 512)...) + a2.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none", segs) + } +} + +// An import refuses a withheld session, also after a restart: otherwise agentop import would +// bring it in from the agent's own transcript. +func TestNoRecord_AnImportRefusesTheSession(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(306, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + defer a2.Close() + for _, replace := range []bool{false, true} { + if err := a2.WriteSession("s1", importedEvents(4), replace); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(replace=%v) = %v, want ErrLive", replace, err) + } + } +} + +// Another session is untouched by one that opted out. +func TestNoRecord_OtherSessionsAreRecorded(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(307, 1, 512))...) + r.record("s2", synthSession(308, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s2"); len(got) != 3 { + t.Fatalf("s2 holds %d events, want 3", len(got)) + } + for _, e := range onDisk(t, root, "s2") { + if e.NoRecord { + t.Fatalf("a recorded event carries NoRecord: seq %d", e.Seq) + } + } +} diff --git a/core/session/delete.go b/core/session/delete.go index 9f250bccf..b27948f79 100644 --- a/core/session/delete.go +++ b/core/session/delete.go @@ -27,6 +27,7 @@ type Deleter interface { func (s *Store) Delete(sessionID string) bool { s.mu.Lock() defer s.mu.Unlock() + delete(s.noRecord, sessionID) _, held := s.sessions[sessionID] if held { s.removeLocked(sessionID) diff --git a/core/session/fold.go b/core/session/fold.go index 14207ca65..25a397cfd 100644 --- a/core/session/fold.go +++ b/core/session/fold.go @@ -283,8 +283,18 @@ type SummaryFold struct { // (Source ""). Summary marks the session as imported only while live is false. imported string live bool + // noRecord is true once any event asked not to be recorded (SessionEvent.NoRecord, #1923), + // or once MarkNoRecord was called. Sticky: a later event without the flag does not clear it. + noRecord bool } +// MarkNoRecord marks the session as not recorded without folding an event, for the archive's +// content-free marker of an opted-out session (#1923). +func (f *SummaryFold) MarkNoRecord() { f.noRecord = true } + +// NoRecord reports whether the session asked not to be recorded; see SummaryFold.noRecord. +func (f *SummaryFold) NoRecord() bool { return f.noRecord } + // InferenceHosts is a copy of the fold's table, and false when the fold holds history it has no // table for. func (f *SummaryFold) InferenceHosts() (model.InferenceHosts, bool) { @@ -345,6 +355,9 @@ func (f *SummaryFold) Add(sessionID string, e *model.SessionEvent) { } else if f.imported == "" { f.imported = e.Source } + if e.NoRecord { + f.noRecord = true + } } // promptContext is the fold's CTX figure, nil when nothing can be said. @@ -372,6 +385,7 @@ func (f *SummaryFold) Summary(sessionID string) SessionSummary { LastRequestAt: f.lastRequestAt, Agent: pickAgent(sessionID, f.owner, nil, f.agents), Projects: slices.Clone(f.projects), + NoRecord: f.noRecord, } if !f.live { sum.Source = f.imported @@ -414,6 +428,9 @@ type foldJSON struct { // before they existed holds no imported events, so it decodes as not imported. Imported string `json:"imported,omitempty"` Live bool `json:"live,omitempty"` + // NoRecord is SummaryFold's field of the same name. Absent from every recorded session's + // session.json, which decodes to recorded. + NoRecord bool `json:"noRecord,omitempty"` } // hostsDoc is SummaryFold's InferenceHosts table in session.json. V is the hostRuleVersion that @@ -467,6 +484,7 @@ func (f *SummaryFold) MarshalJSON() ([]byte, error) { Projects: f.projects, Imported: f.imported, Live: f.live, + NoRecord: f.noRecord, } if !f.hostsUnknown { j.InferenceHosts = &hostsDoc{V: hostRuleVersion, Hosts: f.hosts} @@ -502,6 +520,7 @@ func (f *SummaryFold) UnmarshalJSON(b []byte) error { // session.json written before Live existed. live: j.Live || (j.EventCount > 0 && j.Imported == ""), hostsUnknown: j.InferenceHosts == nil || j.InferenceHosts.V != hostRuleVersion, + noRecord: j.NoRecord, } if j.TitleRank != nil { f.titleRank = *j.TitleRank diff --git a/core/session/history.go b/core/session/history.go index 1f464378f..d75e8712a 100644 --- a/core/session/history.go +++ b/core/session/history.go @@ -100,6 +100,7 @@ func (s *Store) Clear() int { n := len(s.sessions) s.sessions = make(map[string]*entry) s.activeID = "" + s.noRecord = nil // nil, as a store nothing has claimed from holds them; every writer allocates on first use. for _, r := range s.recorders { if c, ok := r.(Clearer); ok { diff --git a/core/session/norecord_test.go b/core/session/norecord_test.go new file mode 100644 index 000000000..6eeb2b5ab --- /dev/null +++ b/core/session/norecord_test.go @@ -0,0 +1,97 @@ +package session + +import ( + "slices" + "testing" + "time" + + "github.com/rossoctl/cortex/core/model" +) + +// A session shows NoRecord from its first flagged event on, and a later event without the flag +// (a header-less call filed to it) does not clear it. Another session is not marked. +func TestNoRecord_SummaryIsSticky(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest}) + s.Append("s2", model.SessionEvent{Phase: model.SessionRequest}) + + got := map[string]bool{} + for _, sum := range s.ListSessions() { + got[sum.ID] = sum.NoRecord + } + if !got["s1"] || got["s2"] { + t.Fatalf("NoRecord by session = %v, want s1 only", got) + } +} + +// MarkNoRecord stamps every later event of that session, for rows the listener does not build +// (praxis mode's /v1/ingest), and no other session's. +func TestNoRecord_MarkNoRecordStampsLaterEvents(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + rec := &flagRecorder{} + s.AddRecorder(rec) + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest}) + s.MarkNoRecord("s1") + s.Append("s1", model.SessionEvent{Phase: model.SessionResponse}) + s.Append("s2", model.SessionEvent{Phase: model.SessionRequest}) + if want := []bool{false, true, false}; !slices.Equal(rec.flags, want) { + t.Fatalf("NoRecord by event = %v, want %v", rec.flags, want) + } +} + +// A header may name the default bucket or a pending bucket. Neither is one conversation, so +// the store clears the flag there and MarkNoRecord ignores them: no recorder sees a withheld +// shared bucket. +func TestNoRecord_SharedBucketsAreNeverWithheld(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + rec := &flagRecorder{} + s.AddRecorder(rec) + pending := PendingSessionID("claude-code") + s.Append(DefaultSessionID, model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.Append(pending, model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.MarkNoRecord(DefaultSessionID) + s.MarkNoRecord(pending) + s.Append(DefaultSessionID, model.SessionEvent{Phase: model.SessionRequest}) + s.Append(pending, model.SessionEvent{Phase: model.SessionRequest}) + if want := []bool{false, false, false, false}; !slices.Equal(rec.flags, want) { + t.Fatalf("NoRecord by event = %v, want %v", rec.flags, want) + } + for _, sum := range s.ListSessions() { + if sum.NoRecord { + t.Fatalf("%s is marked NoRecord", sum.ID) + } + } +} + +// flagRecorder keeps each event's NoRecord in order. The store calls it under its write lock. +type flagRecorder struct{ flags []bool } + +func (r *flagRecorder) Record(_ string, e *model.SessionEvent) { r.flags = append(r.flags, e.NoRecord) } + +// The archive's fold keeps the mark through session.json. +func TestNoRecord_FoldRoundTrips(t *testing.T) { + f := NewSummaryFold() + f.Add("s1", &model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + f.Add("s1", &model.SessionEvent{Phase: model.SessionRequest}) + b, err := f.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var back SummaryFold + if err := back.UnmarshalJSON(b); err != nil { + t.Fatal(err) + } + if !back.NoRecord() || !back.Summary("s1").NoRecord { + t.Fatalf("NoRecord lost through %s", b) + } + plain, _ := NewSummaryFold().MarshalJSON() + var p SummaryFold + _ = p.UnmarshalJSON(plain) + if p.NoRecord() { + t.Fatal("an unmarked fold decodes as NoRecord") + } +} diff --git a/core/session/store.go b/core/session/store.go index 6a7c48cdd..c7409240f 100644 --- a/core/session/store.go +++ b/core/session/store.go @@ -85,6 +85,11 @@ type entry struct { // count rather than a set because a trim has to know when the LAST event in a unit leaves. units map[string]int + // noRecord is true once an event of this entry asked not to be recorded + // (SessionEvent.NoRecord, #1923). Sticky, and read by ListSessions as + // SessionSummary.NoRecord. The archive keeps its own sticky mark, which survives a restart. + noRecord bool + // money is each event's decoded contribution, PARALLEL TO Events: same length, same // order, same trim. // @@ -225,6 +230,10 @@ type Store struct { // AddRecorder, before the store serves traffic. recorders []Recorder + // noRecord holds the session ids MarkNoRecord named: every event appended to one of them + // is stamped SessionEvent.NoRecord before the recorders see it (#1923). Protected by mu. + noRecord map[string]bool + // streams holds the live byte counter of each open inference response, by // request ID (see TrackStream). It has its own lock so that the hot path // does not take mu. @@ -309,6 +318,39 @@ func (s *Store) backgroundCleanup() { } } +// MarkNoRecord stamps SessionEvent.NoRecord on every event appended to sessionID from now on, so +// the session archive withholds them (#1923). For a request whose rows the listener does not +// build itself: in praxis mode the gateway records a hopped request and its rows come back +// through /v1/ingest without the flag, the header having been stripped before the hop. +// +// The caller must have checked pipeline.Context.WithholdsSession: the request named sessionID +// with its own session-id header. Never for a fallback bucket. +func (s *Store) MarkNoRecord(sessionID string) { + if !IsConversation(sessionID) { // see appendLocked + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.noRecord == nil { + s.noRecord = map[string]bool{} + } + s.noRecord[sessionID] = true + // The archive marks it too, at once: this store mark goes when the entry is evicted, and a + // row ingested after that would carry no flag. + for _, r := range s.recorders { + if w, ok := r.(Withholder); ok { + w.Withhold(sessionID) + } + } +} + +// Withholder is optionally implemented by a Recorder that keeps a session's events (the +// session archive): Withhold tells it, from now on, to keep none of sessionID's (#1923). Called +// under the store's write lock, so it must not block or call back into the store. +type Withholder interface { + Withhold(sessionID string) +} + // Append adds an event to the named session. Creates the session if it // doesn't exist. Updates activeID to this session. Evicts the oldest event if the // session exceeds maxEvents — which by default it cannot, maxEvents being unset. @@ -491,6 +533,19 @@ func (s *Store) appendLocked(sessionID string, b *Bucket, event model.SessionEve // without needing to know which session it was appended to — critical for // outbound events that have no protocol-native session field. event.SessionID = sessionID + if s.noRecord[sessionID] { + event.NoRecord = true + } + // Only one conversation's own session can opt out (#1923). The default bucket and a pending + // bucket hold many clients' calls, and a header can name them ("default" passes the + // session-id rules): a flag there would withhold everyone filed to it, and a pending mark + // would follow an Adopt onto a session that never asked. Such an event records as usual. + if event.NoRecord && !IsConversation(sessionID) { + event.NoRecord = false + } + if event.NoRecord { + sess.noRecord = true + } // Stamp the cursor before publishLocked below, so a subscriber streaming events // live sees the same Seq a paging client would get for that event. If this were @@ -974,6 +1029,10 @@ func (s *Store) rekeyLocked(oldID, newID string) bool { sess.ID = newID s.sessions[newID] = sess delete(s.sessions, oldID) + if s.noRecord[oldID] { // the mark follows the session (#1923) + delete(s.noRecord, oldID) + s.noRecord[newID] = true + } if s.activeID == oldID { s.activeID = newID } @@ -1000,6 +1059,8 @@ func (s *Store) rekeyLocked(oldID, newID string) bool { // held. func (s *Store) removeLocked(id string) { delete(s.sessions, id) + // The archive's own mark withholds the session from here on (#1923). + delete(s.noRecord, id) if s.activeID == id { s.activeID = "" } diff --git a/core/session/summary.go b/core/session/summary.go index c35bd911e..84f8f9446 100644 --- a/core/session/summary.go +++ b/core/session/summary.go @@ -215,6 +215,10 @@ type SessionSummary struct { // Source is "imported:" while every event of the session is imported, and "" // otherwise. Only the archive's fold sets it: store entries are live. Source string `json:"source,omitempty"` + // NoRecord is true when the session asked not to be recorded (#1923): the archive keeps + // none of its events, only a marker that it opted out. A live session shows it from its + // first flagged request. + NoRecord bool `json:"noRecord,omitempty"` // LinkedTo, on a pending bucket, is the sessions its process named that OwnerAt gives a // call of the bucket, each with the span of its naming requests (Link says how close that // is); LinkedAll says every request in the bucket falls before one of them. Set by Link when @@ -275,6 +279,7 @@ func (s *Store) ListSessions() []SessionSummary { Title: sess.Title, Agent: pickAgent(id, sess.owner, nil, sess.agents), Projects: slices.Clone(sess.projects), + NoRecord: sess.noRecord, // Still a walk, and deliberately left as one: it is a pointer deref per event // with no allocation, where the money figures below needed a JSON unmarshal. TotalTokens: sumTokens(sess.Events), diff --git a/core/sessionapi/summary_test.go b/core/sessionapi/summary_test.go index a149ec0e1..89bc6be16 100644 --- a/core/sessionapi/summary_test.go +++ b/core/sessionapi/summary_test.go @@ -99,6 +99,7 @@ func fullEvent() pipeline.SessionEvent { SessionHeader: "X-Claude-Code-Session-Id", Project: "~/src/app", Source: "imported:claude-code", + NoRecord: true, } } @@ -235,6 +236,11 @@ func TestSummarizeEvent_DropsPayloadsKeepsTimelineFields(t *testing.T) { if got.Source != full.Source { t.Errorf("source = %q, want %q", got.Source, full.Source) } + // noRecord is kept for the same reason: a bool that marks a row of a session the archive + // withholds (#1923). + if got.NoRecord != full.NoRecord { + t.Errorf("noRecord = %v, want %v", got.NoRecord, full.NoRecord) + } if got.Direction != full.Direction || got.Phase != full.Phase { t.Error("direction/phase did not survive") } @@ -444,7 +450,9 @@ func TestSummarizeEvent_ShapeIsGuarded(t *testing.T) { // so the struct copy keeps it; asserted above. // 36 since Source, TIMELINE data: agentop marks an imported row by it (#1871). A // scalar, so the struct copy keeps it; asserted above. - {"SessionEvent", reflect.TypeOf(pipeline.SessionEvent{}), 36}, + // 37 since NoRecord, TIMELINE data: it marks a row of a withheld session (#1923). A + // scalar, so the struct copy keeps it; asserted above. + {"SessionEvent", reflect.TypeOf(pipeline.SessionEvent{}), 37}, // 25 since AgentRole, which is TIMELINE data: agentop's CONTEXT gauge reads it on every // row the timeline serves. It needs no assertion of its own in the projection test // beyond the equality one there — a scalar survives the struct copy, unlike the two From 2a1a2e858274ddc082390e1003b55941ebd3af62 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 12:25:43 +0200 Subject: [PATCH 02/10] fix: Keep the header-derived session id out of the archive's queue-full log lines CodeQL go/clear-text-logging: the session id comes from a client's request header and flowed into two slog.Warn calls in markLocked. The lines are about the queue, not the session, so they no longer name it. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/session/archive/norecord.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index c5345f79c..4c237590e 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -80,16 +80,16 @@ func (a *Archive) markLocked(sessionID string, from uint64) { if ok { return } + // No session id in these lines: it comes from a client's request header (CodeQL + // go/clear-text-logging), and the line is about the queue, not the session. if len(a.ops) >= cap(a.ops)-reserve { - slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event", - "session", sessionID) + slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event") return } select { case a.ops <- op{kind: opNoRecord, id: sessionID}: default: - slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event", - "session", sessionID) + slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event") } } From e0dc212d392e2eaeab2a67715bbc0c3fc792ce03 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 13:56:29 +0200 Subject: [PATCH 03/10] fix: Let the archive's writer own the opt-out, and never drop a flagged event of an unmarked session From the review of #1959. Most findings came from one state: a mark that lived only in memory when the queue was full, to be persisted later. That state is gone. - The writer owns the opt-out. A session is withheld when its fold is marked noRecord, and only the writer goroutine marks a fold, in the same FIFO as renames, deletes, clears and prune. The archive's and the store's noRecord maps are removed, and with them the races against a delete, a clear, prune, a dropped rename, shutdown and the store's eviction. - A flagged event of a session the writer has not marked yet is never dropped. RecordCapture skips the drop check for it and, when the queue is full, reserve included, waits for the writer, outside a.mu: the writer never takes the store's lock. Privacy over latency: at most once per opted-out session, and only while the archive is saturated. Withhold (for praxis mode's ingested rows) queues its op the same way. - write runs the withhold check before the pause check, so the marker is written while paused. A marker whose write failed stays dirty, is retried, and makes Close return an error. - rename keeps the mark through the fold. Where the move is refused because the target is taken, or no part of the history moved, the new id is marked in place, on the writer. - Tunnel rows never take the flag from the CONNECT, which names no session, and a bridged tunnel's open row takes the flag of the decrypted request it is filed with. A withheld session that goes through the TLS bridge leaves no segment on disk. - noRecord decodes leniently (model.LenientBool), so a producer's wrong type costs the flag, not the row. After a restart, a live row takes the archive's mark (withPriorLocked). - Tests: a flagged event waits on a full queue instead of being dropped; a rename followed by DELETE leaves nothing, and CLEAR removes the mark; a bridged request with off leaves no segment; tunnel rows carry no flag; noRecord of a wrong type keeps the row; MarkNoRecord tells the archive. With each of the three guarantees removed, its test fails. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/listener/forwardproxy/norecord_test.go | 51 ++++++++++ core/listener/forwardproxy/tunnel.go | 4 + .../internal/sessionevent/norecord_test.go | 29 ++++++ .../internal/sessionevent/sessionevent.go | 7 +- core/model/norecord_test.go | 14 +++ core/model/session.go | 22 ++++- core/session/archive/archive.go | 84 +++++++++------- core/session/archive/clear.go | 1 - core/session/archive/delete.go | 1 - core/session/archive/import.go | 3 +- core/session/archive/norecord.go | 97 +++++++++---------- core/session/archive/norecord2_test.go | 82 +++++++++++++--- core/session/delete.go | 1 - core/session/history.go | 4 +- core/session/norecord_test.go | 30 ++++-- core/session/store.go | 32 ++---- 16 files changed, 317 insertions(+), 145 deletions(-) create mode 100644 core/listener/internal/sessionevent/norecord_test.go diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go index 1df8bf200..0b97929fc 100644 --- a/core/listener/forwardproxy/norecord_test.go +++ b/core/listener/forwardproxy/norecord_test.go @@ -1,8 +1,11 @@ package forwardproxy import ( + "io/fs" "net/http" "net/http/httptest" + "path/filepath" + "strings" "sync" "testing" "time" @@ -10,6 +13,7 @@ import ( "github.com/rossoctl/cortex/core/model" "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/session" + "github.com/rossoctl/cortex/core/session/archive" ) // row is what a recorder saw of one event. @@ -106,6 +110,53 @@ func TestNoRecord_FallbackFiledRequestIsRecorded(t *testing.T) { } } +// The normal laptop path for an inference host is the TLS bridge: the CONNECT names nothing, and +// the bridged tunnel's open row is filed with the first decrypted request. When that request +// withholds its session, the open row shares its fate, so the archive holds no segment at all +// for the session (#1923), not even the CONNECT row with its project. +func TestNoRecord_BridgedRequestLeavesNoSegment(t *testing.T) { + root := t.TempDir() + arch, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + store.AddRecorder(arch) + proxyAddr, target, ca, done := bridgingProxyWith(t, store, func(s *Server) { + s.SessionIDHeaders = []string{session.ClaudeCodeSessionHeader} + }) + + raw, br, _ := sendConnect(t, proxyAddr, target) + tc := bridgedTLS(t, raw, br, target, ca) + req, _ := http.NewRequest(http.MethodGet, "https://"+target+"/v1/messages", nil) + req.Header.Set(session.ClaudeCodeSessionHeader, "11111111-2222-3333-4444-555555555555") + req.Header.Set(model.RecordHeader, "off") + if resp := bridgedRoundTrip(t, tc, req); resp.StatusCode != http.StatusOK { + t.Fatalf("bridged request status %d, want 200", resp.StatusCode) + } + _ = tc.Close() + waitDone(t, done) + + open, next := openThen(t, store, "11111111-2222-3333-4444-555555555555") + if !open.NoRecord || !next.NoRecord { + t.Fatalf("open row NoRecord=%v, request row NoRecord=%v, want both", open.NoRecord, next.NoRecord) + } + if err := arch.Close(); err != nil { + t.Fatal(err) + } + var segs []string + _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err == nil && strings.HasSuffix(p, ".jsonl.zst") { + segs = append(segs, p) + } + return nil + }) + if len(segs) != 0 { + t.Fatalf("segments on disk %v, want none", segs) + } +} + // A request without the header is unchanged. func TestNoRecord_UnflaggedRequestIsRecorded(t *testing.T) { proxy, backend, rec, _ := noRecordProxy(t) diff --git a/core/listener/forwardproxy/tunnel.go b/core/listener/forwardproxy/tunnel.go index 04e97cdac..9f693e254 100644 --- a/core/listener/forwardproxy/tunnel.go +++ b/core/listener/forwardproxy/tunnel.go @@ -887,6 +887,10 @@ func (t *tunnelLog) recordWithLocked(sid string, ev pipeline.SessionEvent, c *pi t.deferred, t.opened = false, true open := t.openEv open.At = ev.At + // The open row shares the request's fate (#1923): it goes into the same bucket, only to be + // folded into ev, and it comes first. Without this, a withheld session's archive would write + // the open row (with its project) before it ever saw the request's flag. + open.NoRecord = ev.NoRecord t.bucket = t.s.Sessions.AppendPairCapture(sid, open, ev, c) return true } diff --git a/core/listener/internal/sessionevent/norecord_test.go b/core/listener/internal/sessionevent/norecord_test.go new file mode 100644 index 000000000..388bc83bb --- /dev/null +++ b/core/listener/internal/sessionevent/norecord_test.go @@ -0,0 +1,29 @@ +package sessionevent + +import ( + "net/http" + "testing" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" +) + +// A request that names its own session and asks not to be recorded flags its request and +// response rows, but never a tunnel row: a bridged tunnel's open row can be filed later under the +// session its first decrypted request names, which need not be the one the CONNECT named (#1923). +func TestNoRecord_TunnelRowsCarryNoFlag(t *testing.T) { + h := http.Header{} + h.Set(model.RecordHeader, "off") + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + pctx.SetAttribution("", "", "X-Claude-Code-Session-Id") + if !pctx.WithholdsSession() { + t.Fatal("precondition: the request withholds its own session") + } + if !Request(pctx, pipeline.Outbound).NoRecord || !Response(pctx, pipeline.Outbound, 200, nil).NoRecord { + t.Fatal("request and response rows must carry NoRecord") + } + if TunnelOpen(pctx, "").NoRecord || TunnelClose(pctx, "", 200, nil, 0, 0).NoRecord { + t.Fatal("a tunnel row carries NoRecord") + } +} diff --git a/core/listener/internal/sessionevent/sessionevent.go b/core/listener/internal/sessionevent/sessionevent.go index c2f13bf27..1f094ce93 100644 --- a/core/listener/internal/sessionevent/sessionevent.go +++ b/core/listener/internal/sessionevent/sessionevent.go @@ -180,6 +180,11 @@ func DenyResponse(pctx *pipeline.Context, action pipeline.Action, dir pipeline.D // TunnelOpen builds a tunnel's open row. reason says why the bytes stayed // opaque. The caller knows it, so it is a parameter and not derived here from // the host shape. +// +// No NoRecord on either tunnel row (#1923): a bridged tunnel's open row can be +// filed later under the session its first decrypted request names, which need +// not be the one the CONNECT named. The rows carry no content, and a withheld +// session's archive withholds them anyway. func TunnelOpen(pctx *pipeline.Context, reason pipeline.TunnelReason) pipeline.SessionEvent { return pipeline.SessionEvent{ At: time.Now(), @@ -211,7 +216,6 @@ func TunnelOpen(pctx *pipeline.Context, reason pipeline.TunnelReason) pipeline.S SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), - NoRecord: pctx.WithholdsSession(), } } @@ -247,7 +251,6 @@ func TunnelClose(pctx *pipeline.Context, reason pipeline.TunnelReason, status in SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), - NoRecord: pctx.WithholdsSession(), } } diff --git a/core/model/norecord_test.go b/core/model/norecord_test.go index 43fe63170..84c2bd517 100644 --- a/core/model/norecord_test.go +++ b/core/model/norecord_test.go @@ -38,6 +38,20 @@ func TestRecordsOff(t *testing.T) { } } +// A producer that sends noRecord with the wrong type loses the flag, never the row: it decodes as +// false, the safe direction for a field that can only withhold. +func TestSessionEvent_NoRecordIsLenient(t *testing.T) { + for _, v := range []string{`"yes"`, `1`, `null`, `{}`} { + var e SessionEvent + if err := e.UnmarshalJSON([]byte(`{"at":"2026-11-18T09:00:00Z","direction":"outbound","phase":"request","host":"h","noRecord":` + v + `}`)); err != nil { + t.Fatalf("noRecord=%s: %v, want the row decoded", v, err) + } + if e.NoRecord || e.Host != "h" { + t.Fatalf("noRecord=%s: NoRecord=%v Host=%q, want false and the row's other fields", v, e.NoRecord, e.Host) + } + } +} + // NoRecord round-trips through the wire form and is absent from a recorded row. func TestSessionEvent_NoRecordWire(t *testing.T) { b, err := SessionEvent{NoRecord: true}.MarshalJSON() diff --git a/core/model/session.go b/core/model/session.go index a445f3e50..eea1dc9e2 100644 --- a/core/model/session.go +++ b/core/model/session.go @@ -725,8 +725,9 @@ type sessionEventWire struct { ParentSubagent string `json:"parentSubagent,omitempty"` // omitempty: absent on every live event. Source LenientString `json:"source,omitempty"` - // omitempty: absent on every recorded row, which is every row the archive keeps. - NoRecord bool `json:"noRecord,omitempty"` + // omitempty: absent on every recorded row, which is every row the archive keeps. Lenient, as + // Source is: a producer (cortex-gateway, say) that sends a wrong type must not cost the row. + NoRecord LenientBool `json:"noRecord,omitempty"` // omitempty: absent on every row but a Bob child task's. SubagentKind string `json:"subagentKind,omitempty"` // omitempty: absent wherever the listener could not name the client's process, and on @@ -773,7 +774,7 @@ func (e SessionEvent) MarshalJSON() ([]byte, error) { Subagent: e.Subagent, ParentSubagent: e.ParentSubagent, Source: LenientString(e.Source), - NoRecord: e.NoRecord, + NoRecord: LenientBool(e.NoRecord), SubagentKind: e.SubagentKind, Proc: e.Proc, ClientProc: e.ClientProc, @@ -822,7 +823,7 @@ func (e *SessionEvent) UnmarshalJSON(data []byte) error { Subagent: w.Subagent, ParentSubagent: w.ParentSubagent, Source: string(w.Source), - NoRecord: w.NoRecord, + NoRecord: bool(w.NoRecord), SubagentKind: w.SubagentKind, Proc: w.Proc, ClientProc: w.ClientProc, @@ -1099,6 +1100,19 @@ func (s *LenientString) UnmarshalJSON(b []byte) error { return nil } +// LenientBool decodes JSON true as true and any other JSON value as false, for the reason +// LenientString gives. Used where false is the safe default: noRecord can only withhold. +type LenientBool bool + +func (b *LenientBool) UnmarshalJSON(raw []byte) error { + var v bool + if json.Unmarshal(raw, &v) != nil { + v = false + } + *b = LenientBool(v) + return nil +} + // Imported tells if an import wrote e. A live event gives false. func (e *SessionEvent) Imported() bool { return strings.HasPrefix(e.Source, ImportedSourcePrefix) diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 7c6958b33..f561d638f 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -131,12 +131,7 @@ type Archive struct { // recent clear, for AwaitClear; see clear.go. mu sync.Mutex lastSeq map[string]uint64 - // noRecord holds, for each id a flagged event named (SessionEvent.NoRecord), the lowest Seq - // it carried. RecordCapture sets it before the event can be dropped, so the session is - // withheld even when that event never reaches the writer, and from that Seq on only: events - // queued before it are still written. See withhold. Under mu. - noRecord map[string]uint64 - pending *pendingClear + pending *pendingClear // deletes is the most recent delete per session id, for AwaitDelete; see delete.go. nil // until the first. deletes map[string]*pendingClear @@ -241,7 +236,6 @@ func Open(root string, opts ...Option) (*Archive, error) { maxOpen: defaultMaxOpen, tickEvery: defaultTickEvery, lastSeq: map[string]uint64{}, - noRecord: map[string]uint64{}, started: map[string]entryStart{}, sessions: map[string]*sessionState{}, index: map[string]*indexEntry{}, @@ -343,17 +337,18 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if a.closed.Load() { return } + // A flagged event of a session the writer has not marked yet is never dropped (#1923): the + // writer's mark is the opt-out's only state, so losing the event would lose the opt-out. + // Read before a.mu, which the reader index never nests inside. + mustDeliver := e.NoRecord && session.IsConversation(sessionID) && !a.markedInIndex(sessionID) a.mu.Lock() - defer a.mu.Unlock() if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq } - if e.NoRecord { - a.markLocked(sessionID, e.Seq) - } // Every send happens under the store's write lock, so this check and the send below cannot // race another producer; the writer only ever makes room. - if len(a.ops) >= cap(a.ops)-reserve { + if !mustDeliver && len(a.ops) >= cap(a.ops)-reserve { + a.mu.Unlock() a.dropped.Add(1) return } @@ -378,9 +373,25 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode st.queued = true a.started[sessionID] = st } + a.mu.Unlock() default: - a.dropped.Add(1) - a.releaseCapture(o.cap) + if !mustDeliver { + a.mu.Unlock() + a.dropped.Add(1) + a.releaseCapture(o.cap) + return + } + if starting && !st.queued { + st.queued = true + a.started[sessionID] = st + } + a.mu.Unlock() + // The queue is full, reserve included. Wait for the writer, outside a.mu: the writer + // takes a.mu and never the store's lock, so it always makes room. The store's write + // lock is held, so no other producer sends in between and the order holds. Privacy over + // latency: this waits at most once per opted-out session, and only while the archive is + // saturated. + a.deliver(o) } } @@ -566,16 +577,18 @@ func (a *Archive) begin(id string, after uint64) { // write appends one event, with its request's capture c when it has one, to its session's open // segment, opening one if needed. func (a *Archive) write(id string, e *model.SessionEvent, c *model.RequestCapture) { - if a.paused.Load() { - a.dropped.Add(1) - return - } now := a.now() s := a.sessions[id] + // Before the pause check: the marker is one small session.json, and it is the opt-out's only + // state, so it is written while paused too (#1923). if a.withheld(id, s, e) { a.withhold(id, s, now) return } + if a.paused.Load() { + a.dropped.Add(1) + return + } fresh := s == nil if fresh { s = &sessionState{ @@ -691,35 +704,34 @@ func (a *Archive) writeFailed(s *sessionState, err error) { // id under the old directory, which adopt finishes at the next start. func (a *Archive) rename(oldID, newID string, after uint64) { s := a.sessions[oldID] - // A withheld session stays withheld under its new id (#1923), on every path below, the early - // returns included: the store's entry moves to newID whatever happens to the files here. - a.mu.Lock() - _, flaggedOld := a.noRecord[oldID] - marked := flaggedOld || (s != nil && s.meta.Summary.NoRecord()) - delete(a.noRecord, oldID) - if marked { - a.markLocked(newID, 0) // every event under the new id - } - a.mu.Unlock() if s == nil { return } + // A withheld session stays withheld under its new id (#1923): the store's entry moves to newID + // whatever happens to the files here. On the paths that move the fold, renameEntry and the + // move below keep its mark; where the move is refused, the new id is marked in place. All on + // the writer, in FIFO order with any later delete or clear of newID. + marked := s.meta.Summary.NoRecord() newDir := filepath.Join(a.data, dirName(newID)) if _, taken := a.sessions[newID]; taken { slog.Warn("session archive: rename target already has history; keeping it under the old id", "from", oldID, "to", newID) + if marked { + a.withhold(newID, a.sessions[newID], a.now()) + } return } if _, err := os.Stat(newDir); err == nil { slog.Warn("session archive: rename target directory exists; keeping history under the old id", "from", oldID, "to", newID) - return - } - if marked { - // A flag only in memory (its event not written yet) moves with the session's files too. - s.meta.Summary.MarkNoRecord() + return // a directory the archive does not own: nothing is written into it } if after > 0 && a.renameEntry(s, oldID, newID, newDir, after) { + // The history stayed under oldID, whole or in part. If no part moved, newID has no state + // yet, and gets the mark as a marker. + if marked && a.sessions[newID] == nil { + a.withhold(newID, nil, a.now()) + } return } a.closeWriterBeforeRename(s) // a no-op except on Windows: see rename_windows.go @@ -962,7 +974,6 @@ func (a *Archive) prune(now time.Time) { if a.lastSeq[id] <= lastBefore[s] { delete(a.lastSeq, id) } - delete(a.noRecord, id) // an expired marker, or a session whose history expired a.mu.Unlock() } for _, s := range a.sessions { @@ -1020,6 +1031,11 @@ func (a *Archive) shutdown() error { for _, s := range a.sessions { if s.metaDirty { a.writeMeta(s) + // A withheld session's marker is its opt-out: report one that could not be written, + // rather than restart without it (#1923). No session id: it comes from a client header. + if s.metaDirty && s.meta.Summary.NoRecord() { + errs = append(errs, errors.New("a withheld session's marker could not be written")) + } } } return errors.Join(errs...) diff --git a/core/session/archive/clear.go b/core/session/archive/clear.go index c55c09dcc..e0c24b1c3 100644 --- a/core/session/archive/clear.go +++ b/core/session/archive/clear.go @@ -53,7 +53,6 @@ func (a *Archive) Cleared() { a.mu.Lock() p.seqs = maps.Clone(a.lastSeq) a.pending = p - clear(a.noRecord) a.mu.Unlock() if a.closed.Load() { p.finish(ClearResult{}, errors.New("the session archive is closed")) diff --git a/core/session/archive/delete.go b/core/session/archive/delete.go index fda2f35e3..ed53b0a6f 100644 --- a/core/session/archive/delete.go +++ b/core/session/archive/delete.go @@ -22,7 +22,6 @@ func (a *Archive) Deleted(sessionID string) { a.mu.Lock() p.seqs = map[string]uint64{sessionID: a.lastSeq[sessionID]} delete(a.started, sessionID) - delete(a.noRecord, sessionID) if a.deletes == nil { a.deletes = map[string]*pendingClear{} } diff --git a/core/session/archive/import.go b/core/session/archive/import.go index df360a9a3..0690ea6ce 100644 --- a/core/session/archive/import.go +++ b/core/session/archive/import.go @@ -65,9 +65,8 @@ func (a *Archive) writeSession(id string, events []model.SessionEvent, replace b // after the import (SeqSeeder), never from 1. a.mu.Lock() old := a.lastSeq[id] - _, flagged := a.noRecord[id] switch { - case flagged || (s != nil && s.meta.Summary.NoRecord()): + case s != nil && s.meta.Summary.NoRecord(): // The session asked not to be recorded (#1923). Its marker is the user's own choice, // so no import adds to it or replaces it, also one whose events were imported first. a.mu.Unlock() diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index 4c237590e..39acbccdc 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -1,7 +1,6 @@ package archive import ( - "log/slog" "path/filepath" "time" @@ -9,11 +8,17 @@ import ( "github.com/rossoctl/cortex/core/session" ) -// withhold keeps session id out of the archive from now on (#1923). It runs on the writer -// goroutine for an event that asked not to be recorded (SessionEvent.NoRecord, which listeners -// set only for a request that names its own session), and for every later event of a session -// already withheld, flagged or not: a session's header-less calls (telemetry, say) are filed to -// it by process or client affinity and carry no flag. +// THE WRITER OWNS THE OPT-OUT (#1923). A session is withheld when its fold is marked noRecord, +// and only the writer goroutine marks a fold, in the same FIFO as renames, deletes, clears and +// prune, so they cannot reorder against it. There is no other copy of the state: RecordCapture +// never drops a flagged event of a session the writer has not marked yet, and Withhold never +// drops its op (deliver), so nothing has to be remembered elsewhere until the writer gets to it. + +// withhold keeps session id out of the archive from now on. It runs on the writer goroutine for an +// event that asked not to be recorded (SessionEvent.NoRecord, which listeners set only for a +// request that names its own session), for an opNoRecord (Withhold), and for every later event of +// a session already withheld, flagged or not: a session's header-less calls (telemetry, say) are +// filed to it by process or client affinity and carry no flag. // // FROM NOW ON, NEVER BACKWARDS. What the archive already holds for id stays: it was recorded // before the opt-out, and the session-id header that names id is client-asserted, so a request @@ -24,11 +29,10 @@ import ( // For an id the archive does not hold, withhold writes a content-free marker: a session.json with // no segments whose summary says noRecord and nothing else. The event itself is never folded, // because the fold reads content (the title comes from the prompt). The mark is what keeps the -// opt-out after a restart, when the store's own mark is gone, and what makes an import refuse id -// (WriteSession returns ErrLive for an id the archive holds that is not imported). +// opt-out after a restart, and what makes an import refuse id (WriteSession returns ErrLive). // -// noRecord ids are also remembered in a.noRecord, under a.mu, by RecordCapture: an event dropped -// before the writer saw it (queue full, archive paused) must not leave the session unmarked. +// metaDirty is set before the write, so a failed write is retried at the next metaEvery and at +// shutdown, and Close reports a marker it could not write. func (a *Archive) withhold(id string, s *sessionState, now time.Time) { if s != nil && s.meta.Summary.NoRecord() { // Already marked. A marker's age is measured from the session's last event @@ -47,68 +51,57 @@ func (a *Archive) withhold(id string, s *sessionState, now time.Time) { a.closeWriter(s) } s.meta.Summary.MarkNoRecord() - s.meta.UpdatedAt, s.lastWrite = now, now + s.meta.UpdatedAt, s.lastWrite, s.metaDirty = now, now, true a.writeMeta(s) a.publish(s) } // Withhold implements session.Withholder: the store tells the archive that sessionID is withheld -// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). It runs under -// the store's write lock, and never blocks. +// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). The op it +// queues is never dropped (deliver), and it precedes those rows in the FIFO, because the store +// calls this before the request is hopped. It runs under the store's write lock. func (a *Archive) Withhold(sessionID string) { - if a.closed.Load() { + if a.closed.Load() || !session.IsConversation(sessionID) || a.markedInIndex(sessionID) { return } - a.mu.Lock() - defer a.mu.Unlock() - a.markLocked(sessionID, a.lastSeq[sessionID]+1) + a.deliver(op{kind: opNoRecord, id: sessionID}) } -// markLocked marks sessionID withheld from Seq from on, in memory at once (a.noRecord), and on -// disk through an opNoRecord for the session's first mark, so a restart before its next event -// keeps it. a.mu held. The op takes the queue's normal capacity, never the reserve that renames, -// deletes and clears rely on: when the queue is full the mark stays in memory, and the session's -// next event writes it (withhold). -func (a *Archive) markLocked(sessionID string, from uint64) { - if !session.IsConversation(sessionID) { - return - } - old, ok := a.noRecord[sessionID] - if !ok || from < old { - a.noRecord[sessionID] = from - } - if ok { - return - } - // No session id in these lines: it comes from a client's request header (CodeQL - // go/clear-text-logging), and the line is about the queue, not the session. - if len(a.ops) >= cap(a.ops)-reserve { - slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event") +// deliver queues o and, when the queue is full, waits for the writer rather than drop it. Only +// for the ops an opt-out depends on. The caller holds the store's write lock and must not hold +// a.mu: the writer takes a.mu, and it never takes the store's lock, so it always makes room. It +// gives up only when the archive closes. +func (a *Archive) deliver(o op) { + select { + case a.ops <- o: return + default: } select { - case a.ops <- op{kind: opNoRecord, id: sessionID}: - default: - slog.Warn("session archive: the queue is full; a session that asked not to be recorded is withheld in memory until its next event") + case a.ops <- o: + case <-a.done: + a.releaseCapture(o.cap) } } -// withheld reports whether e, an event of id, is to be withheld: it asks it, or the session was -// marked on disk, or a flagged event the writer may not have seen yet came at or before it. -// Events numbered before that flagged one are still written: the opt-out never reaches back. +// markedInIndex reports whether the writer has published id as withheld. A false answer can be +// stale (a mark not published yet), which only makes the caller deliver an op it could have +// dropped. +func (a *Archive) markedInIndex(id string) bool { + a.idxMu.RLock() + defer a.idxMu.RUnlock() + e := a.index[id] + return e != nil && e.summary.NoRecord +} + +// withheld reports whether e, an event of id, is to be withheld: it asks it, or the session is +// marked. Never for a shared bucket: the store already clears the flag there, and the archive +// does not trust that alone (session.IsConversation). func (a *Archive) withheld(id string, s *sessionState, e *model.SessionEvent) bool { - // Never a shared bucket: the store already clears the flag there, and the archive does - // not trust that alone (session.IsConversation). if !session.IsConversation(id) { return false } - if e.NoRecord || (s != nil && s.meta.Summary.NoRecord()) { - return true - } - a.mu.Lock() - defer a.mu.Unlock() - from, ok := a.noRecord[id] - return ok && e.Seq >= from + return e.NoRecord || (s != nil && s.meta.Summary.NoRecord()) } // keepMarker reports whether prune keeps s, a session with nothing on disk, because it marks a diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index dbd54db60..26a075577 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -71,29 +71,81 @@ func TestNoRecord_HopMarkSurvivesEviction(t *testing.T) { } } -// Review round 3, item 4: a flag the writer has not seen yet, held only in memory (its event and -// its opNoRecord lost to a full queue), still follows a rename to the new id. -func TestNoRecord_AMemoryOnlyFlagFollowsARename(t *testing.T) { +// The writer's mark is the opt-out's only state, so a flagged event of a session not marked yet is +// never dropped: with the queue full, reserve included, it waits for the writer instead. +func TestNoRecord_AFlaggedEventIsNeverDroppedWhileUnmarked(t *testing.T) { root := t.TempDir() a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) a.gate = make(chan struct{}) r := newRecorder(a) - evs := synthSession(332, 3, 512) // 9 events - r.record("s1", evs[:6]...) // fills the queue to the reserve - r.record("s1", flagged(evs[6:7])...) // dropped, and its opNoRecord does not fit - a.Rekeyed("s1", "s2") // into the reserve + evs := synthSession(332, 3, 512) // 9 events + r.record("s1", evs[:6]...) // to the reserve + for len(a.ops) < cap(a.ops) { // and the reserve too + a.Rekeyed("filler", "filler2") + } + dropped := a.Stats().DroppedEvents + done := make(chan struct{}) + go func() { + r.record("s1", flagged(evs[6:7])...) + close(done) + }() + select { + case <-done: + t.Fatal("a flagged event of an unmarked session returned with the queue full: it was dropped") + case <-time.After(200 * time.Millisecond): + } close(a.gate) + <-done + if got := a.Stats().DroppedEvents; got != dropped { + t.Fatalf("dropped events %d → %d, want the flagged one delivered", dropped, got) + } settle(a) - r.seq["s2"] = r.seq["s1"] - r.record("s2", evs[7:]...) + r.record("s1", evs[7:]...) a.Close() - if got := onDisk(t, root, "s2"); len(got) != 6 { - t.Fatalf("s2 holds %d events, want the 6 recorded before the flag", len(got)) + if got := onDisk(t, root, "s1"); len(got) != 6 { + t.Fatalf("s1 holds %d events, want the 6 recorded before the flag", len(got)) } - a2 := openTest(t, root, newClock()) - defer a2.Close() - if err := a2.WriteSession("s2", importedEvents(4), true); !errors.Is(err, ErrLive) { - t.Fatalf("WriteSession(s2) after the rename = %v, want ErrLive: the mark was not persisted", err) +} + +// A rename moves a marked session's mark inside the rename op, so a DELETE queued after it +// removes the new id for good: no marker comes back behind the delete. +func TestNoRecord_RenameThenDeleteLeavesNothing(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(333, 1, 512))...) + settle(a) + a.Rekeyed("s1", "s2") + a.Deleted("s2") + if _, err := a.AwaitDelete("s2", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + settle(a) + r.seq["s2"] = 0 + r.record("s2", synthSession(334, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s2"); len(got) != 3 { + t.Fatalf("s2 holds %d events after DELETE, want the 3 recorded since: the mark came back", len(got)) + } +} + +// A clear removes every mark with the history: the user's explicit erase. Later unflagged events +// record again. +func TestNoRecord_ClearRemovesTheMark(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(335, 1, 512))...) + settle(a) + a.Cleared() + if _, err := a.AwaitClear(5 * time.Second); err != nil { + t.Fatalf("clear: %v", err) + } + r.seq["s1"] = 0 + r.record("s1", synthSession(336, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != 3 { + t.Fatalf("s1 holds %d events after CLEAR, want the 3 recorded since", len(got)) } } diff --git a/core/session/delete.go b/core/session/delete.go index b27948f79..9f250bccf 100644 --- a/core/session/delete.go +++ b/core/session/delete.go @@ -27,7 +27,6 @@ type Deleter interface { func (s *Store) Delete(sessionID string) bool { s.mu.Lock() defer s.mu.Unlock() - delete(s.noRecord, sessionID) _, held := s.sessions[sessionID] if held { s.removeLocked(sessionID) diff --git a/core/session/history.go b/core/session/history.go index d75e8712a..7a476aedc 100644 --- a/core/session/history.go +++ b/core/session/history.go @@ -100,7 +100,6 @@ func (s *Store) Clear() int { n := len(s.sessions) s.sessions = make(map[string]*entry) s.activeID = "" - s.noRecord = nil // nil, as a store nothing has claimed from holds them; every writer allocates on first use. for _, r := range s.recorders { if c, ok := r.(Clearer); ok { @@ -180,6 +179,9 @@ func (s *Store) withPriorLocked(sum *SessionSummary, id string, sess *entry) { f := p.Fold sum.EventCount += f.events sum.TotalTokens += f.tokens + // The archive's mark is the opt-out (#1923): a withheld session shows it after a restart, + // before its next flagged event. + sum.NoRecord = sum.NoRecord || f.noRecord // CostSum's addition for the sums, OR for the flags: a total that clamped on either side is a // bound, whichever side it was. cost, avoided := sess.cost, sess.avoided diff --git a/core/session/norecord_test.go b/core/session/norecord_test.go index 6eeb2b5ab..af30dcebe 100644 --- a/core/session/norecord_test.go +++ b/core/session/norecord_test.go @@ -26,22 +26,34 @@ func TestNoRecord_SummaryIsSticky(t *testing.T) { } } -// MarkNoRecord stamps every later event of that session, for rows the listener does not build -// (praxis mode's /v1/ingest), and no other session's. -func TestNoRecord_MarkNoRecordStampsLaterEvents(t *testing.T) { +// MarkNoRecord tells every Withholder (the archive, which owns the opt-out) and marks the live +// entry's summary, for rows the listener does not build (praxis mode's /v1/ingest). It never +// names a shared bucket. +func TestNoRecord_MarkNoRecordTellsTheArchive(t *testing.T) { s := New(time.Hour, 0, 0) defer s.Close() - rec := &flagRecorder{} - s.AddRecorder(rec) + w := &withholder{} + s.AddRecorder(w) s.Append("s1", model.SessionEvent{Phase: model.SessionRequest}) s.MarkNoRecord("s1") - s.Append("s1", model.SessionEvent{Phase: model.SessionResponse}) - s.Append("s2", model.SessionEvent{Phase: model.SessionRequest}) - if want := []bool{false, true, false}; !slices.Equal(rec.flags, want) { - t.Fatalf("NoRecord by event = %v, want %v", rec.flags, want) + s.MarkNoRecord("s2") // no live entry yet: the archive still hears it + s.MarkNoRecord(DefaultSessionID) + if want := []string{"s1", "s2"}; !slices.Equal(w.ids, want) { + t.Fatalf("Withhold calls %v, want %v", w.ids, want) + } + for _, sum := range s.ListSessions() { + if sum.ID == "s1" && !sum.NoRecord { + t.Fatal("s1's live summary is not marked") + } } } +// withholder is a Recorder that also implements Withholder, as the archive does. +type withholder struct{ ids []string } + +func (w *withholder) Record(string, *model.SessionEvent) {} +func (w *withholder) Withhold(id string) { w.ids = append(w.ids, id) } + // A header may name the default bucket or a pending bucket. Neither is one conversation, so // the store clears the flag there and MarkNoRecord ignores them: no recorder sees a withheld // shared bucket. diff --git a/core/session/store.go b/core/session/store.go index c7409240f..22881b4a7 100644 --- a/core/session/store.go +++ b/core/session/store.go @@ -230,10 +230,6 @@ type Store struct { // AddRecorder, before the store serves traffic. recorders []Recorder - // noRecord holds the session ids MarkNoRecord named: every event appended to one of them - // is stamped SessionEvent.NoRecord before the recorders see it (#1923). Protected by mu. - noRecord map[string]bool - // streams holds the live byte counter of each open inference response, by // request ID (see TrackStream). It has its own lock so that the hot path // does not take mu. @@ -318,10 +314,12 @@ func (s *Store) backgroundCleanup() { } } -// MarkNoRecord stamps SessionEvent.NoRecord on every event appended to sessionID from now on, so -// the session archive withholds them (#1923). For a request whose rows the listener does not -// build itself: in praxis mode the gateway records a hopped request and its rows come back -// through /v1/ingest without the flag, the header having been stripped before the hop. +// MarkNoRecord withholds sessionID from the archive from now on (#1923), for a request whose rows +// the listener does not build itself: in praxis mode the gateway records a hopped request, and +// its rows come back through /v1/ingest without the flag, the header having been stripped before +// the hop. It tells every Withholder (the archive), whose mark is the opt-out's only state, and +// marks the live entry's summary when there is one. Call it before the request is hopped: the +// archive's op then precedes every row that comes back. // // The caller must have checked pipeline.Context.WithholdsSession: the request named sessionID // with its own session-id header. Never for a fallback bucket. @@ -331,12 +329,9 @@ func (s *Store) MarkNoRecord(sessionID string) { } s.mu.Lock() defer s.mu.Unlock() - if s.noRecord == nil { - s.noRecord = map[string]bool{} + if sess := s.sessions[sessionID]; sess != nil { + sess.noRecord = true } - s.noRecord[sessionID] = true - // The archive marks it too, at once: this store mark goes when the entry is evicted, and a - // row ingested after that would carry no flag. for _, r := range s.recorders { if w, ok := r.(Withholder); ok { w.Withhold(sessionID) @@ -346,7 +341,7 @@ func (s *Store) MarkNoRecord(sessionID string) { // Withholder is optionally implemented by a Recorder that keeps a session's events (the // session archive): Withhold tells it, from now on, to keep none of sessionID's (#1923). Called -// under the store's write lock, so it must not block or call back into the store. +// under the store's write lock. It may wait for its own writer, never for the store. type Withholder interface { Withhold(sessionID string) } @@ -533,9 +528,6 @@ func (s *Store) appendLocked(sessionID string, b *Bucket, event model.SessionEve // without needing to know which session it was appended to — critical for // outbound events that have no protocol-native session field. event.SessionID = sessionID - if s.noRecord[sessionID] { - event.NoRecord = true - } // Only one conversation's own session can opt out (#1923). The default bucket and a pending // bucket hold many clients' calls, and a header can name them ("default" passes the // session-id rules): a flag there would withhold everyone filed to it, and a pending mark @@ -1029,10 +1021,6 @@ func (s *Store) rekeyLocked(oldID, newID string) bool { sess.ID = newID s.sessions[newID] = sess delete(s.sessions, oldID) - if s.noRecord[oldID] { // the mark follows the session (#1923) - delete(s.noRecord, oldID) - s.noRecord[newID] = true - } if s.activeID == oldID { s.activeID = newID } @@ -1059,8 +1047,6 @@ func (s *Store) rekeyLocked(oldID, newID string) bool { // held. func (s *Store) removeLocked(id string) { delete(s.sessions, id) - // The archive's own mark withholds the session from here on (#1923). - delete(s.noRecord, id) if s.activeID == id { s.activeID = "" } From aba849cd891c1a42d58b6d99fd741cadbf11c258 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 14:04:16 +0200 Subject: [PATCH 04/10] fix: Never set X-Cortex-Record through ext_proc, and tighten the opt-out's docs and tests CodeRabbit's minor comments on #1959: - ext_proc emits no set mutation for X-Cortex-Record, also when a plugin wrote it back, so the removal is never contradicted. - The docs of SessionEvent.NoRecord, Context.NoRecord and SessionSummary.NoRecord say the opt-out applies from then on, and that what the archive held before stays. - Tests: the header's removal is checked right after ResolveClient, before NoRecord can remove it itself; the forward proxy checks that the upstream header is absent, not just empty; ext_proc checks the response count before indexing; the fold round trip checks both errors. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/listener/extproc/norecord_test.go | 8 +++++--- core/listener/extproc/server.go | 4 +++- core/listener/forwardproxy/norecord_test.go | 11 ++++++++--- core/model/session.go | 14 +++++++------- core/pipeline/norecord.go | 5 +++-- core/pipeline/norecord_test.go | 8 +++++--- core/session/norecord_test.go | 9 +++++++-- core/session/summary.go | 7 ++++--- 8 files changed, 42 insertions(+), 24 deletions(-) diff --git a/core/listener/extproc/norecord_test.go b/core/listener/extproc/norecord_test.go index d9c01bb21..6880e2356 100644 --- a/core/listener/extproc/norecord_test.go +++ b/core/listener/extproc/norecord_test.go @@ -49,9 +49,11 @@ func TestNoRecord_ExtProcSkipHostRemovesTheHeader(t *testing.T) { stream := &mockStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{outboundRequest(hdrs())}} _ = srv.Process(stream) - if len(stream.responses) != 1 || !slices.Contains(removed(t, stream.responses[0]), "x-cortex-record") { - t.Fatalf("headers phase: responses %d, RemoveHeaders %v, want x-cortex-record removed", - len(stream.responses), removed(t, stream.responses[0])) + if len(stream.responses) != 1 { + t.Fatalf("headers phase: %d responses, want 1", len(stream.responses)) + } + if got := removed(t, stream.responses[0]); !slices.Contains(got, "x-cortex-record") { + t.Fatalf("headers phase: RemoveHeaders %v, want x-cortex-record removed", got) } resp, pctx := srv.handleOutboundBody(&mockStream{ctx: context.Background()}, hdrs(), []byte(`{}`)) diff --git a/core/listener/extproc/server.go b/core/listener/extproc/server.go index f398be0a0..4db084c00 100644 --- a/core/listener/extproc/server.go +++ b/core/listener/extproc/server.go @@ -971,7 +971,9 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte var set []*corev3.HeaderValueOption var del []string for k, vv := range pctx.Headers { - if skip(k) || slices.Equal(orig[k], vv) { + // Never forward X-Cortex-Record (#1923), also when a plugin wrote it back: it is removed + // below, and a set beside the remove would leave the outcome to Envoy's ordering. + if skip(k) || slices.Equal(orig[k], vv) || strings.EqualFold(k, model.RecordHeader) { continue } if len(vv) == 0 { diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go index 0b97929fc..a861bd87e 100644 --- a/core/listener/forwardproxy/norecord_test.go +++ b/core/listener/forwardproxy/norecord_test.go @@ -49,7 +49,12 @@ func noRecordProxy(t *testing.T) (proxy *httptest.Server, backendURL string, rec var mu sync.Mutex backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() - got = append(got, r.Header.Get(model.RecordHeader)) + // "" when the header is not there at all, not just empty. + v := "" + if vs := r.Header.Values(model.RecordHeader); len(vs) > 0 { + v = strings.Join(vs, ",") + } + got = append(got, v) mu.Unlock() })) t.Cleanup(backend.Close) @@ -76,7 +81,7 @@ func TestNoRecord_OwnSessionIsWithheldAndHeaderStripped(t *testing.T) { model.RecordHeader: {"off"}, session.ClaudeCodeSessionHeader: {"11111111-2222-3333-4444-555555555555"}, }) - if len(*seen) != 1 || (*seen)[0] != "" { + if len(*seen) != 1 || (*seen)[0] != "" { t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) } got := rec.take() @@ -96,7 +101,7 @@ func TestNoRecord_OwnSessionIsWithheldAndHeaderStripped(t *testing.T) { func TestNoRecord_FallbackFiledRequestIsRecorded(t *testing.T) { proxy, backend, rec, seen := noRecordProxy(t) postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{model.RecordHeader: {"off"}}) - if len(*seen) != 1 || (*seen)[0] != "" { + if len(*seen) != 1 || (*seen)[0] != "" { t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) } got := rec.take() diff --git a/core/model/session.go b/core/model/session.go index eea1dc9e2..4ea89d0a0 100644 --- a/core/model/session.go +++ b/core/model/session.go @@ -447,14 +447,14 @@ type SessionEvent struct { // Use ImportedSource to make the value and Imported to test it. Source string - // NoRecord is true when the request asked not to be recorded: it carried - // RecordHeader with the value RecordOff (#1923). Read by Context.NoRecord, pinned - // before any plugin runs. The proxy still forwards the request and runs its plugins; - // the session archive keeps none of the session's events. + // NoRecord is true when the request withholds its own session from the archive: it + // carried RecordHeader with the value RecordOff and named its session with a session-id + // header (#1923; Context.WithholdsSession, pinned before any plugin runs). The proxy + // still forwards the request and runs its plugins. The archive keeps none of the + // session's events from then on; what it held before the opt-out stays. // - // Client-asserted, with the same caveat as Client above. It can only withhold a - // session from the archive, never put one there, so a forged value costs its sender - // their own recording. + // Client-asserted, with the same caveat as Client above. It can stop a session's + // recording, never put one there or erase what is recorded. NoRecord bool // SubagentKind is the kind of a Bob child task, from Context.SubagentKind: diff --git a/core/pipeline/norecord.go b/core/pipeline/norecord.go index f114940a1..546af2938 100644 --- a/core/pipeline/norecord.go +++ b/core/pipeline/norecord.go @@ -3,8 +3,9 @@ package pipeline import "github.com/rossoctl/cortex/core/model" // NoRecord reports whether the request asked not to be recorded (#1923): it carried -// model.RecordHeader set to model.RecordOff. Recorded as SessionEvent.NoRecord on every row -// the request produces, and the session archive keeps none of that session's events. +// model.RecordHeader set to model.RecordOff. With an own session id (WithholdsSession), it is +// recorded as SessionEvent.NoRecord on the request's rows, and the archive keeps none of that +// session's events from then on; what it held before stays. // // The first call also deletes model.RecordHeader from Headers. The header is for Cortex // only: plugins never see it, and the listeners that send the request on from Headers diff --git a/core/pipeline/norecord_test.go b/core/pipeline/norecord_test.go index c3c9ec534..66bf7eb50 100644 --- a/core/pipeline/norecord_test.go +++ b/core/pipeline/norecord_test.go @@ -15,12 +15,14 @@ func TestNoRecord_PinnedAndStripped(t *testing.T) { h.Set("User-Agent", "claude-cli/2.1.266 (external, cli)") c := &Context{Headers: h} c.ResolveClient() - if !c.NoRecord() { - t.Fatal("NoRecord = false for X-Cortex-Record: off") - } + // Before calling NoRecord here: its first call would remove the header itself, so this + // checks that ResolveClient made it. if _, ok := c.Headers[http.CanonicalHeaderKey(model.RecordHeader)]; ok { t.Fatal("the header is still in Headers after ResolveClient") } + if !c.NoRecord() { + t.Fatal("NoRecord = false for X-Cortex-Record: off") + } // A plugin that adds it back later changes nothing: the answer was pinned. c.Headers.Set(model.RecordHeader, "on") if !c.NoRecord() { diff --git a/core/session/norecord_test.go b/core/session/norecord_test.go index af30dcebe..3b466bfcf 100644 --- a/core/session/norecord_test.go +++ b/core/session/norecord_test.go @@ -100,9 +100,14 @@ func TestNoRecord_FoldRoundTrips(t *testing.T) { if !back.NoRecord() || !back.Summary("s1").NoRecord { t.Fatalf("NoRecord lost through %s", b) } - plain, _ := NewSummaryFold().MarshalJSON() + plain, err := NewSummaryFold().MarshalJSON() + if err != nil { + t.Fatal(err) + } var p SummaryFold - _ = p.UnmarshalJSON(plain) + if err := p.UnmarshalJSON(plain); err != nil { + t.Fatal(err) + } if p.NoRecord() { t.Fatal("an unmarked fold decodes as NoRecord") } diff --git a/core/session/summary.go b/core/session/summary.go index 84f8f9446..4070a92d2 100644 --- a/core/session/summary.go +++ b/core/session/summary.go @@ -215,9 +215,10 @@ type SessionSummary struct { // Source is "imported:" while every event of the session is imported, and "" // otherwise. Only the archive's fold sets it: store entries are live. Source string `json:"source,omitempty"` - // NoRecord is true when the session asked not to be recorded (#1923): the archive keeps - // none of its events, only a marker that it opted out. A live session shows it from its - // first flagged request. + // NoRecord is true when the session asked not to be recorded (#1923): the archive keeps none + // of its events from then on, and what it held before stays. A session with nothing before + // is only a marker that it opted out. A live session shows it from its first flagged request, + // and after a restart from the archive's mark. NoRecord bool `json:"noRecord,omitempty"` // LinkedTo, on a pending bucket, is the sessions its process named that OwnerAt gives a // call of the bucket, each with the span of its naming requests (Link says how close that From 4ee46300213744184029ed1b35751e2ba21f0577 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 14:33:15 +0200 Subject: [PATCH 05/10] fix: Deliver every flagged event and Withhold, below the reserve, and mark a renamed session in every case From a review of the writer-owned opt-out: - No "already marked" shortcut. A DELETE, CLEAR or prune still queued can remove a mark the reader index still shows, so every flagged event of a conversation and every Withhold is delivered, never dropped. Withholding a marked session is cheap. - deliver waits for room below the reserve and never takes it. The reserve stays for renames, deletes and clears, so a burst of flagged events can't crowd them out. The trade is as before, privacy over latency: a request of an opted-out session waits only while the archive is saturated, without bound on a disk that never recovers. An op lost to the archive closing is counted and logged. - rename marks the new id when its directory already exists, as when the target is taken: the new id's next event would build its state on that directory anyway. A partial rename keeps the mark on the old id. - Tests: Withhold while a DELETE is queued, and a flagged event while a CLEAR is queued, both with a gated writer; flagged events leave the reserve to a rename. With the shortcut restored, or with flagged events allowed into the reserve, the matching test fails. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/session/archive/archive.go | 44 ++++++++------- core/session/archive/norecord.go | 53 +++++++++++------- core/session/archive/norecord2_test.go | 76 ++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 40 deletions(-) diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index f561d638f..37a32ec4a 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -337,10 +337,10 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if a.closed.Load() { return } - // A flagged event of a session the writer has not marked yet is never dropped (#1923): the - // writer's mark is the opt-out's only state, so losing the event would lose the opt-out. - // Read before a.mu, which the reader index never nests inside. - mustDeliver := e.NoRecord && session.IsConversation(sessionID) && !a.markedInIndex(sessionID) + // A flagged event of a conversation is never dropped (#1923): the writer's mark is the + // opt-out's only state, and a DELETE, CLEAR or prune still queued may be about to remove it, + // so no event may assume the session is already marked. See deliver. + mustDeliver := e.NoRecord && session.IsConversation(sessionID) a.mu.Lock() if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq @@ -367,31 +367,26 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if starting && !st.queued { o.after = st.after } - select { - case a.ops <- o: + if mustDeliver { if starting && !st.queued { st.queued = true a.started[sessionID] = st } a.mu.Unlock() - default: - if !mustDeliver { - a.mu.Unlock() - a.dropped.Add(1) - a.releaseCapture(o.cap) - return - } + a.deliver(o) // outside a.mu: it may wait for the writer, which takes a.mu + return + } + select { + case a.ops <- o: if starting && !st.queued { st.queued = true a.started[sessionID] = st } a.mu.Unlock() - // The queue is full, reserve included. Wait for the writer, outside a.mu: the writer - // takes a.mu and never the store's lock, so it always makes room. The store's write - // lock is held, so no other producer sends in between and the order holds. Privacy over - // latency: this waits at most once per opted-out session, and only while the archive is - // saturated. - a.deliver(o) + default: + a.mu.Unlock() + a.dropped.Add(1) + a.releaseCapture(o.cap) } } @@ -724,7 +719,12 @@ func (a *Archive) rename(oldID, newID string, after uint64) { if _, err := os.Stat(newDir); err == nil { slog.Warn("session archive: rename target directory exists; keeping history under the old id", "from", oldID, "to", newID) - return // a directory the archive does not own: nothing is written into it + // newID's next event would build its state on that directory anyway (write), so mark it + // there now, as for a taken target: unflagged events under newID must not be recorded. + if marked { + a.withhold(newID, nil, a.now()) + } + return } if after > 0 && a.renameEntry(s, oldID, newID, newDir, after) { // The history stayed under oldID, whole or in part. If no part moved, newID has no state @@ -836,6 +836,10 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte } // A CLONE: before was published, and the old id's live fold is Added to from here on. s.meta.Segments, s.meta.Summary, s.meta.UpdatedAt = keep, before.Clone(), latestWrite(keep) + if noRec { + // before predates the opt-out, so it does not hold the mark; the old id keeps it (#1923). + s.meta.Summary.MarkNoRecord() + } a.writeMeta(s) if ns.metaDirty { a.writeMeta(ns) diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index 39acbccdc..b9b1a25f7 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -1,6 +1,7 @@ package archive import ( + "log/slog" "path/filepath" "time" @@ -11,8 +12,8 @@ import ( // THE WRITER OWNS THE OPT-OUT (#1923). A session is withheld when its fold is marked noRecord, // and only the writer goroutine marks a fold, in the same FIFO as renames, deletes, clears and // prune, so they cannot reorder against it. There is no other copy of the state: RecordCapture -// never drops a flagged event of a session the writer has not marked yet, and Withhold never -// drops its op (deliver), so nothing has to be remembered elsewhere until the writer gets to it. +// never drops a flagged event of a conversation, and Withhold never drops its op (deliver), so +// nothing has to be remembered elsewhere until the writer gets to it. // withhold keeps session id out of the archive from now on. It runs on the writer goroutine for an // event that asked not to be recorded (SessionEvent.NoRecord, which listeners set only for a @@ -61,37 +62,49 @@ func (a *Archive) withhold(id string, s *sessionState, now time.Time) { // queues is never dropped (deliver), and it precedes those rows in the FIFO, because the store // calls this before the request is hopped. It runs under the store's write lock. func (a *Archive) Withhold(sessionID string) { - if a.closed.Load() || !session.IsConversation(sessionID) || a.markedInIndex(sessionID) { + if a.closed.Load() || !session.IsConversation(sessionID) { return } + // Every time, also for a session the writer has marked already: a DELETE, CLEAR or prune + // still queued may be about to remove that mark. Withholding a marked session is cheap. a.deliver(op{kind: opNoRecord, id: sessionID}) } -// deliver queues o and, when the queue is full, waits for the writer rather than drop it. Only -// for the ops an opt-out depends on. The caller holds the store's write lock and must not hold -// a.mu: the writer takes a.mu, and it never takes the store's lock, so it always makes room. It -// gives up only when the archive closes. +// deliver queues o, an op an opt-out depends on, and never drops it: when the queue has no room +// below the reserve it waits for the writer. The reserve stays for renames, deletes and clears, +// which must not be crowded out by flagged events. +// +// PRIVACY OVER LATENCY, a deliberate trade: while the archive is saturated (its writer a full +// queue behind, e.g. on a stalled disk) a request of an opted-out session waits here, without +// bound if the disk never recovers. An unsaturated archive never waits. +// +// No deadlock: the caller holds the store's write lock and must not hold a.mu. The writer takes +// a.mu and never the store's lock, so it keeps draining, also while paused. Only producers that +// hold the store's lock send, so once there is room below the reserve the send cannot block, and +// the FIFO order of the store's appends holds. It gives up only when the archive closes, and +// says so. func (a *Archive) deliver(o op) { - select { - case a.ops <- o: - return - default: + for len(a.ops) >= cap(a.ops)-reserve { + select { + case <-a.done: + a.undelivered(o) + return + case <-time.After(time.Millisecond): + } } select { case a.ops <- o: case <-a.done: - a.releaseCapture(o.cap) + a.undelivered(o) } } -// markedInIndex reports whether the writer has published id as withheld. A false answer can be -// stale (a mark not published yet), which only makes the caller deliver an op it could have -// dropped. -func (a *Archive) markedInIndex(id string) bool { - a.idxMu.RLock() - defer a.idxMu.RUnlock() - e := a.index[id] - return e != nil && e.summary.NoRecord +// undelivered counts and reports an opt-out op lost to the archive closing. No session id: it +// comes from a client's request header. +func (a *Archive) undelivered(o op) { + a.releaseCapture(o.cap) + a.dropped.Add(1) + slog.Warn("session archive: closed before a request that asked not to be recorded reached the writer; nothing of it was written") } // withheld reports whether e, an event of id, is to be withheld: it asks it, or the session is diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index 26a075577..e387cf42c 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/session" ) @@ -107,6 +108,81 @@ func TestNoRecord_AFlaggedEventIsNeverDroppedWhileUnmarked(t *testing.T) { } } +// A DELETE still queued when a praxis Withhold arrives: the session was marked, but the delete is +// about to remove the mark, so Withhold must not take "already marked" for granted. The rows +// ingested after it are withheld. +func TestNoRecord_WithholdWhileADeleteIsQueued(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(340, 1, 512))...) + settle(a) // marked and published + a.gate = make(chan struct{}) + a.Deleted("s1") // queued, not applied yet + a.Withhold("s1") // a hopped request withholds s1 again + r.record("s1", synthSession(341, 1, 512)...) // the gateway's rows, unflagged + close(a.gate) + if _, err := a.AwaitDelete("s1", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the rows after the delete were archived", segs) + } +} + +// The same for a flagged event behind a queued CLEAR: it is delivered, and the session is marked +// again after the clear. +func TestNoRecord_FlaggedEventWhileAClearIsQueued(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(342, 1, 512))...) + settle(a) + a.gate = make(chan struct{}) + a.Cleared() + evs := synthSession(343, 1, 512) + r.record("s1", flagged(evs[:1])...) + r.record("s1", evs[1:]...) + close(a.gate) + if _, err := a.AwaitClear(5 * time.Second); err != nil { + t.Fatalf("clear: %v", err) + } + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the opt-out after the clear was lost", segs) + } +} + +// Flagged events wait below the reserve, never in it: the reserve stays for renames, deletes and +// clears, which a burst of flagged events must not crowd out. +func TestNoRecord_FlaggedEventsLeaveTheReserveAlone(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + a.gate = make(chan struct{}) + r := newRecorder(a) + r.record("s1", synthSession(344, 2, 512)[:6]...) // up to the reserve + // More than the reserve holds, counting the slot the gated writer has already taken off + // the queue: if flagged events could use the reserve, they would fill it. + evs := make([]pipeline.SessionEvent, reserve+2) + for i := range evs { + evs[i].NoRecord = true + } + done := make(chan struct{}) + go func() { + r.record("s2", evs...) + close(done) + }() + time.Sleep(200 * time.Millisecond) + a.Rekeyed("s1", "s3") // must still find a reserve slot + if got := a.Stats().DroppedRenames; got != 0 { + t.Fatalf("dropped %d renames: flagged events took the reserve", got) + } + close(a.gate) + <-done + a.Close() +} + // A rename moves a marked session's mark inside the rename op, so a DELETE queued after it // removes the new id for good: no marker comes back behind the delete. func TestNoRecord_RenameThenDeleteLeavesNothing(t *testing.T) { From 7091cf6205ab4bd2a07302c14e4c7e859c2ef918 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Wed, 18 Nov 2026 17:21:50 +0200 Subject: [PATCH 06/10] fix: Withhold an opaque CONNECT's own session, close the deliver/Close race, and trim expired markers From the second review of #1959: - An opaque CONNECT that names its own session and sends off flags its open row, but only when the row files under the session it named (appendTunnelOpen). A row filed under another session carries no flag. - deliver and the writer's stop are ordered by a.mu: the writer sets stopped after its drain for Close and drains once more, and deliver refuses an op once stopped is set and counts it as undelivered. No op is queued where nothing reads it. - rename sets metaDirty before writing the moved session.json, so a failed write is retried and reported. A split keeps a withheld session's renewal time in both halves. - When retention leaves a withheld session with no segments, prune reduces its fold to the mark, so the expired history's title and counts don't outlive it. - Tests: the opaque CONNECT files under its own session (flagged) and under another (not flagged); deliver refuses after the writer stopped; an expired history leaves only the mark; the reserve test waits on a signal from deliver instead of a sleep; the bridged test registers its cleanup early and fails on scan errors; the store test asserts the session is listed. With the CONNECT condition halved, the stopped check removed, or flagged events let into the reserve, the matching test fails. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/listener/forwardproxy/norecord_test.go | 86 ++++++++++++++++++++- core/listener/forwardproxy/transparent.go | 9 ++- core/session/archive/archive.go | 57 ++++++++++---- core/session/archive/norecord.go | 38 ++++++++- core/session/archive/norecord2_test.go | 60 +++++++++++++- core/session/norecord_test.go | 11 ++- 6 files changed, 238 insertions(+), 23 deletions(-) diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go index a861bd87e..978de391f 100644 --- a/core/listener/forwardproxy/norecord_test.go +++ b/core/listener/forwardproxy/norecord_test.go @@ -1,7 +1,10 @@ package forwardproxy import ( + "bufio" + "fmt" "io/fs" + "net" "net/http" "net/http/httptest" "path/filepath" @@ -125,6 +128,9 @@ func TestNoRecord_BridgedRequestLeavesNoSegment(t *testing.T) { if err != nil { t.Fatal(err) } + // Closed explicitly below before the scan; this covers an early exit. A second Close is a + // no-op. + t.Cleanup(func() { _ = arch.Close() }) store := session.New(5*time.Minute, 100, 0) defer store.Close() store.AddRecorder(arch) @@ -151,17 +157,91 @@ func TestNoRecord_BridgedRequestLeavesNoSegment(t *testing.T) { t.Fatal(err) } var segs []string - _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { - if err == nil && strings.HasSuffix(p, ".jsonl.zst") { + if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err // an unread subtree must fail the test, not pass it + } + if strings.HasSuffix(p, ".jsonl.zst") { segs = append(segs, p) } return nil - }) + }); err != nil { + t.Fatalf("scan %s: %v", root, err) + } if len(segs) != 0 { t.Fatalf("segments on disk %v, want none", segs) } } +// sendConnectWith is sendConnect with extra request headers on the CONNECT. +func sendConnectWith(t *testing.T, proxyAddr, target string, h http.Header) (net.Conn, *bufio.Reader) { + t.Helper() + raw, err := net.Dial("tcp", proxyAddr) + if err != nil { + t.Fatalf("dial proxy: %v", err) + } + t.Cleanup(func() { _ = raw.Close() }) + _ = raw.SetDeadline(time.Now().Add(10 * time.Second)) + var b strings.Builder + fmt.Fprintf(&b, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n", target, target) + for k, vs := range h { + for _, v := range vs { + fmt.Fprintf(&b, "%s: %s\r\n", k, v) + } + } + b.WriteString("\r\n") + if _, err := raw.Write([]byte(b.String())); err != nil { + t.Fatalf("write CONNECT: %v", err) + } + br := bufio.NewReader(raw) + resp, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("read CONNECT response: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("CONNECT status = %d, want 200", resp.StatusCode) + } + return raw, br +} + +// An opaque (not bridged) CONNECT that names its own session and sends off withholds that +// session: its open row files under the named session at once, so it carries the flag. When the +// row files under another session (the active one, with no pin), it carries none, so no other +// session is stopped. +func TestNoRecord_OpaqueConnectWithholdsOnlyItsOwnSession(t *testing.T) { + const own = "11111111-2222-3333-4444-555555555555" + hdr := http.Header{session.ClaudeCodeSessionHeader: {own}, model.RecordHeader: {"off"}} + for _, tc := range []struct { + name string + affinity bool // pins the row to the session the CONNECT names + active string // a session active before the CONNECT, or "" + filed string // where the open row files + want bool + }{ + {"files under its own session", true, "", own, true}, + {"files under another session", false, "sess-A", "sess-A", false}, + } { + t.Run(tc.name, func(t *testing.T) { + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + if tc.active != "" { + store.Append(tc.active, pipeline.SessionEvent{At: time.Now(), Direction: pipeline.Outbound, Phase: pipeline.SessionRequest}) + } + s := connectServer(t, store, nil) // no bridge: an opaque tunnel + s.SessionIDHeaders = []string{session.ClaudeCodeSessionHeader} + s.ClientAffinity = tc.affinity + proxyAddr, done := connectProxy(t, s) + raw, br := sendConnectWith(t, proxyAddr, pingPongOrigin(t), hdr) + pingPong(t, raw, br) + waitDone(t, done) + open, _ := onePair(t, store, tc.filed) + if open.NoRecord != tc.want { + t.Fatalf("open row under %s: NoRecord=%v, want %v", tc.filed, open.NoRecord, tc.want) + } + }) + } +} + // A request without the header is unchanged. func TestNoRecord_UnflaggedRequestIsRecorded(t *testing.T) { proxy, backend, rec, _ := noRecordProxy(t) diff --git a/core/listener/forwardproxy/transparent.go b/core/listener/forwardproxy/transparent.go index 1a414d17e..80ef2d334 100644 --- a/core/listener/forwardproxy/transparent.go +++ b/core/listener/forwardproxy/transparent.go @@ -336,7 +336,14 @@ func (s *Server) appendTunnelOpen(pctx *pipeline.Context, ev pipeline.SessionEve if s.ClientAffinity || s.processesOn() || pctx.SessionSource == pipeline.SessionSourceConnection { pinned = pctx.OutboundSessionID } - return s.Sessions.AppendBucket(attribution.FiledSession(pinned, s.Sessions.ActiveSession), ev) + sid := attribution.FiledSession(pinned, s.Sessions.ActiveSession) + // A CONNECT that names its own session and asks not to be recorded withholds that session + // (#1923), but only when its row files under the session it named: by these rules the row can + // go to the active session instead, and a flag there would stop another session's recording. + if pctx.WithholdsSession() && sid == session.IDFromHeaders(pctx.Headers, s.SessionIDHeaders) { + ev.NoRecord = true + } + return s.Sessions.AppendBucket(sid, ev) } // recordTunnelClosed emits the SessionResponse row for a tunnel that ended, or that diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 37a32ec4a..b573c017a 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -115,9 +115,11 @@ type Archive struct { maxOpen int tickEvery time.Duration - // Test seams: gate holds the writer before each op; segHook sees each new segment writer. - gate chan struct{} - segHook func(*segmentWriter) + // Test seams: gate holds the writer before each op; segHook sees each new segment writer; + // deliverWait is called while deliver waits for room. + gate chan struct{} + segHook func(*segmentWriter) + deliverWait func() ops chan op stop chan struct{} @@ -131,6 +133,9 @@ type Archive struct { // recent clear, for AwaitClear; see clear.go. mu sync.Mutex lastSeq map[string]uint64 + // stopped is set by the writer once it has drained for Close: deliver then refuses an + // opt-out op rather than queue one nothing will read. Under mu. + stopped bool pending *pendingClear // deletes is the most recent delete per session id, for AwaitDelete; see delete.go. nil // until the first. @@ -507,20 +512,31 @@ func (a *Archive) run() { case <-t.C: a.tick() case <-a.stop: - for drained := false; !drained; { - select { - case o := <-a.ops: - a.take(o) - default: - drained = true - } - } + a.drain() + // From here no opt-out op is accepted (deliver checks stopped under a.mu), so one + // more drain takes every op that was (#1923). + a.mu.Lock() + a.stopped = true + a.mu.Unlock() + a.drain() a.closeErr = a.shutdown() return } } } +// drain takes every op queued now, without waiting for more. +func (a *Archive) drain() { + for { + select { + case o := <-a.ops: + a.take(o) + default: + return + } + } +} + func (a *Archive) take(o op) { if a.gate != nil { <-a.gate @@ -736,6 +752,8 @@ func (a *Archive) rename(oldID, newID string, after uint64) { } a.closeWriterBeforeRename(s) // a no-op except on Windows: see rename_windows.go s.meta.ID = newID + // Dirty before the write: a failure is retried, and Close reports a withheld session's (#1923). + s.metaDirty = true a.writeMeta(s) if err := os.Rename(s.dir, newDir); err != nil { slog.Warn("session archive: could not move a renamed session's directory; it will be moved at the next start", @@ -810,8 +828,12 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte meta: &sessionMeta{ID: newID, CreatedAt: createdAt, UpdatedAt: latestWrite(move), Summary: since, Segments: move}, lastWrite: s.lastWrite, } + renewed := s.meta.UpdatedAt // a withheld session's last event, which wrote no segment if noRec { ns.meta.Summary.MarkNoRecord() + // The marker lives retainDays after the session's last event (keepMarker), not after its + // last segment. + ns.meta.UpdatedAt = later(ns.meta.UpdatedAt, renewed) } // session.json first: a crash before a segment moves leaves it listed where it is not, which // reconcile drops, rather than a directory loadMetas cannot read. @@ -837,8 +859,10 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte // A CLONE: before was published, and the old id's live fold is Added to from here on. s.meta.Segments, s.meta.Summary, s.meta.UpdatedAt = keep, before.Clone(), latestWrite(keep) if noRec { - // before predates the opt-out, so it does not hold the mark; the old id keeps it (#1923). + // before predates the opt-out, so it does not hold the mark; the old id keeps it (#1923), + // and the renewal time with it. s.meta.Summary.MarkNoRecord() + s.meta.UpdatedAt = later(s.meta.UpdatedAt, renewed) } a.writeMeta(s) if ns.metaDirty { @@ -959,9 +983,14 @@ func (a *Archive) prune(now time.Time) { } var gone []*sessionState for _, s := range a.sessions { - if s.w == nil && len(s.meta.Segments) == 0 && !keepMarker(s, now, a.retainDays) { - gone = append(gone, s) + if s.w != nil || len(s.meta.Segments) > 0 { + continue + } + if keepMarker(s, now, a.retainDays) { + a.markerOnly(s) // its history expired: keep the mark, not the history's fold + continue } + gone = append(gone, s) } a.forget(gone) for _, s := range gone { diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index b9b1a25f7..c7905ab49 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -85,6 +85,9 @@ func (a *Archive) Withhold(sessionID string) { // says so. func (a *Archive) deliver(o op) { for len(a.ops) >= cap(a.ops)-reserve { + if a.deliverWait != nil { + a.deliverWait() // tests: the producer is waiting for room + } select { case <-a.done: a.undelivered(o) @@ -92,9 +95,20 @@ func (a *Archive) deliver(o op) { case <-time.After(time.Millisecond): } } + // The send and the writer's stop are ordered by a.mu: an op is either queued before the + // writer's last drain, and taken by it, or refused here and counted. Non-blocking: there is + // room below the reserve, and no other producer sends while the store's lock is held. + a.mu.Lock() + if a.stopped { + a.mu.Unlock() + a.undelivered(o) + return + } select { case a.ops <- o: - case <-a.done: + a.mu.Unlock() + default: + a.mu.Unlock() a.undelivered(o) } } @@ -117,6 +131,28 @@ func (a *Archive) withheld(id string, s *sessionState, e *model.SessionEvent) bo return e.NoRecord || (s != nil && s.meta.Summary.NoRecord()) } +// later returns the later of a and b. +func later(a, b time.Time) time.Time { + if b.After(a) { + return b + } + return a +} + +// markerOnly reduces s, a withheld session that has nothing left on disk, to its marker: the +// fold drops what the expired segments contributed (title, counts, projects), so nothing +// derived from the prompt outlives them, and keeps only the mark. prune calls it for a marker it +// keeps. +func (a *Archive) markerOnly(s *sessionState) { + if sum := s.meta.Summary.Summary(s.meta.ID); sum.EventCount == 0 && sum.Title == "" { + return // already only a marker: no rewrite on every prune + } + fold := session.NewSummaryFold() + fold.MarkNoRecord() + s.meta.Summary, s.metaDirty = fold, true + a.publish(s) +} + // keepMarker reports whether prune keeps s, a session with nothing on disk, because it marks a // withheld session. A marker lives retainDays after the session's last event, like a segment: // after that, the agent's own transcript has usually expired too. Open makes retainDays positive. diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index e387cf42c..7ab1425e4 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -2,6 +2,8 @@ package archive import ( "errors" + "path/filepath" + "sync" "testing" "time" @@ -168,13 +170,22 @@ func TestNoRecord_FlaggedEventsLeaveTheReserveAlone(t *testing.T) { for i := range evs { evs[i].NoRecord = true } + waiting := make(chan struct{}) + var once sync.Once + a.deliverWait = func() { once.Do(func() { close(waiting) }) } done := make(chan struct{}) go func() { r.record("s2", evs...) close(done) }() - time.Sleep(200 * time.Millisecond) - a.Rekeyed("s1", "s3") // must still find a reserve slot + select { + case <-waiting: // a flagged event reached the wait for room + case <-done: + // Every flagged event was queued without waiting: they took the reserve. + case <-time.After(10 * time.Second): + t.Fatal("the flagged producer never reached deliver's wait") + } + a.Rekeyed("s1", "s3") // with the writer still gated: must still find a reserve slot if got := a.Stats().DroppedRenames; got != 0 { t.Fatalf("dropped %d renames: flagged events took the reserve", got) } @@ -183,6 +194,51 @@ func TestNoRecord_FlaggedEventsLeaveTheReserveAlone(t *testing.T) { a.Close() } +// Review: when retention expires the last segment of a withheld session, the marker it keeps +// holds only the mark, not the expired history's title and counts. +func TestNoRecord_AnExpiredHistoryLeavesOnlyTheMark(t *testing.T) { + root := t.TempDir() + clk := newClock() + a := openTest(t, root, clk, WithRetentionDays(1)) + r := newRecorder(a) + evs := synthSession(345, 1, 512) + r.record("s1", evs[:2]...) // history before the opt-out, with a title + settle(a) + clk.advance(30 * time.Hour) // past retention for the segment + r.record("s1", flagged(evs[2:])...) // the opt-out, today + settle(a) + a.inWriter(func() { a.prune(clk.now()) }) + a.Close() + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatalf("the marker is gone: %v", err) + } + if sum := m.Summary.Summary("s1"); !sum.NoRecord || sum.EventCount != 0 || sum.Title != "" || len(m.Segments) != 0 { + t.Fatalf("after retention: %+v with %d segments, want only the mark", sum, len(m.Segments)) + } +} + +// Review: an op that reaches deliver after the writer stopped reading the queue for Close is +// refused and counted, never queued where nothing reads it. +func TestNoRecord_DeliverRefusesAfterTheWriterStopped(t *testing.T) { + a := openTest(t, t.TempDir(), newClock()) + defer a.Close() + a.mu.Lock() + a.stopped = true // as the writer sets it after its drain for Close + a.mu.Unlock() + before := a.Stats().DroppedEvents + a.Withhold("s1") + if got := a.Stats().DroppedEvents; got != before+1 { + t.Fatalf("dropped %d → %d, want the op counted as undelivered", before, got) + } + if n := len(a.ops); n != 0 { + t.Fatalf("%d ops queued after the writer stopped, want none", n) + } + a.mu.Lock() + a.stopped = false + a.mu.Unlock() +} + // A rename moves a marked session's mark inside the rename op, so a DELETE queued after it // removes the new id for good: no marker comes back behind the delete. func TestNoRecord_RenameThenDeleteLeavesNothing(t *testing.T) { diff --git a/core/session/norecord_test.go b/core/session/norecord_test.go index 3b466bfcf..972a7aac8 100644 --- a/core/session/norecord_test.go +++ b/core/session/norecord_test.go @@ -41,11 +41,18 @@ func TestNoRecord_MarkNoRecordTellsTheArchive(t *testing.T) { if want := []string{"s1", "s2"}; !slices.Equal(w.ids, want) { t.Fatalf("Withhold calls %v, want %v", w.ids, want) } + found := false for _, sum := range s.ListSessions() { - if sum.ID == "s1" && !sum.NoRecord { - t.Fatal("s1's live summary is not marked") + if sum.ID == "s1" { + found = true + if !sum.NoRecord { + t.Fatal("s1's live summary is not marked") + } } } + if !found { + t.Fatal("s1 is not in the session list") + } } // withholder is a Recorder that also implements Withholder, as the archive does. From 72952a725f83b18a48102acfb465b5b0a1ab43f3 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Thu, 19 Nov 2026 10:07:17 +0200 Subject: [PATCH 07/10] fix: Keep an opt-out at its place in the archive queue instead of waiting under the store lock From the third review of #1959 (clawgenti's must-fix, and Hai's decision): a flagged event or Withhold no longer waits for the archive's writer under the session store's write lock, so a stuck writer never blocks other sessions' traffic. - Each op queued under the store's lock gets a position (sendLocked). With room below the reserve, an opt-out is queued as before. With none, it is kept in optOuts, tagged with the last position queued, and the writer applies it right after that op: after any DELETE, CLEAR or rename queued earlier, before the session's later rows. deliver and its wait loop are removed. - One entry per session. A repeat opt-out adds a position only after a rename, delete or clear queued since its last one. Entries go when applied; Close applies what is left before writing markers. - Bounded at the queue's depth. Past it the archive fails closed: no conversation is recorded until a restart, imports are refused, and Stats reports FailedClosed. - An opaque CONNECT's opt-out compares the filing session with the id pinned before the gate plugins ran (Context.SetNamedSession), not a re-read of the headers a plugin can rewrite. - markerOnly also reduces the prior summary to the mark, so the store's list row shows no expired title or counts. - Tests: clawgenti's probe (a stuck writer at the reserve line; a flagged Append, another session's Append and ListSessions all return within 2 s), a kept opt-out's order against a queued DELETE, one entry per session, a kept opt-out across Close and restart, failing closed, the rewritten-header CONNECT, and markerOnly's prior summary. With the wait reinstated, the opt-out applied before its place, the re-tag after a DELETE removed, or the header re-read restored, the matching test fails. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/listener/forwardproxy/identity.go | 10 + core/listener/forwardproxy/norecord_test.go | 29 ++ core/listener/forwardproxy/transparent.go | 4 +- core/listener/forwardproxy/websocket.go | 1 + core/pipeline/attributed.go | 14 + core/pipeline/context.go | 3 + core/session/archive/archive.go | 111 +++++-- core/session/archive/clear.go | 11 +- core/session/archive/delete.go | 11 +- core/session/archive/import.go | 9 +- core/session/archive/norecord.go | 162 +++++++---- core/session/archive/norecord2_test.go | 303 ++++++++++++++++++-- 12 files changed, 536 insertions(+), 132 deletions(-) diff --git a/core/listener/forwardproxy/identity.go b/core/listener/forwardproxy/identity.go index 5638daaa5..74acc605a 100644 --- a/core/listener/forwardproxy/identity.go +++ b/core/listener/forwardproxy/identity.go @@ -80,10 +80,19 @@ func (s *Server) attributeForRecording(pctx *pipeline.Context, trusted string, c at = s.attribute(clientHeaders, chain) } pctx.SetAttribution(at.Proc, at.ClientProc, at.Header) + pctx.SetNamedSession(namedSession(at)) pctx.SetProject(s.projectFor(trusted, m, chain, at)) return at.RecordSession(trusted, s.activeSession) } +// namedSession is the session id a session-id header named in at, "" when none did. +func namedSession(at attribution.Attribution) string { + if at.Header == "" { + return "" + } + return at.Session +} + // tunnelSessionID is recordingSessionID for a tunnel's own row. Under client affinity // resolved is "" only for affinity's ambiguous answer or for no identity at all, and // neither sends a tunnel row to default: a CONNECT rarely carries a User-Agent, so the @@ -177,6 +186,7 @@ func (s *Server) hydrateSession(pctx *pipeline.Context, trusted string, clientHe at = s.attribute(clientHeaders, chain) } pctx.SetAttribution(at.Proc, at.ClientProc, at.Header) + pctx.SetNamedSession(namedSession(at)) pctx.SetProject(s.projectFor(trusted, m, chain, at)) sid, source := at.PluginSession(trusted, s.activeSession) if sid != "" { diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go index 978de391f..fa6ed23a4 100644 --- a/core/listener/forwardproxy/norecord_test.go +++ b/core/listener/forwardproxy/norecord_test.go @@ -242,6 +242,35 @@ func TestNoRecord_OpaqueConnectWithholdsOnlyItsOwnSession(t *testing.T) { } } +// The CONNECT's opt-out compares the filing session with the id pinned before the gate plugins +// ran, not with a re-read of pctx.Headers: a plugin that removes or rewrites the session-id header +// must not make an own-session opt-out record its open row. +func TestNoRecord_OpaqueConnectIgnoresARewrittenHeader(t *testing.T) { + const own = "11111111-2222-3333-4444-555555555555" + for _, rewritten := range []string{"", "99999999-2222-3333-4444-555555555555"} { + store := session.New(5*time.Minute, 100, 0) + h := http.Header{session.ClaudeCodeSessionHeader: {own}, model.RecordHeader: {"off"}} + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + pctx.SetAttribution("", "", session.ClaudeCodeSessionHeader) + pctx.SetNamedSession(own) + pctx.OutboundSessionID = own + // A gate plugin rewrites the header after the pin. + if rewritten == "" { + h.Del(session.ClaudeCodeSessionHeader) + } else { + h.Set(session.ClaudeCodeSessionHeader, rewritten) + } + s := &Server{Sessions: store, ClientAffinity: true, SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}} + s.appendTunnelOpen(pctx, pipeline.SessionEvent{At: time.Now(), Direction: pipeline.Outbound, Phase: pipeline.SessionRequest}) + v := store.View(own) + if v == nil || len(v.Events) != 1 || !v.Events[0].NoRecord { + t.Fatalf("header rewritten to %q: rows under %s %+v, want one flagged open row", rewritten, own, v) + } + store.Close() + } +} + // A request without the header is unchanged. func TestNoRecord_UnflaggedRequestIsRecorded(t *testing.T) { proxy, backend, rec, _ := noRecordProxy(t) diff --git a/core/listener/forwardproxy/transparent.go b/core/listener/forwardproxy/transparent.go index 80ef2d334..b1a06d99a 100644 --- a/core/listener/forwardproxy/transparent.go +++ b/core/listener/forwardproxy/transparent.go @@ -340,7 +340,9 @@ func (s *Server) appendTunnelOpen(pctx *pipeline.Context, ev pipeline.SessionEve // A CONNECT that names its own session and asks not to be recorded withholds that session // (#1923), but only when its row files under the session it named: by these rules the row can // go to the active session instead, and a flag there would stop another session's recording. - if pctx.WithholdsSession() && sid == session.IDFromHeaders(pctx.Headers, s.SessionIDHeaders) { + // The named id is the one pinned before the gate plugins ran (NamedSession), not a re-read + // of pctx.Headers, which a plugin can rewrite. + if pctx.WithholdsSession() && sid == pctx.NamedSession() { ev.NoRecord = true } return s.Sessions.AppendBucket(sid, ev) diff --git a/core/listener/forwardproxy/websocket.go b/core/listener/forwardproxy/websocket.go index 88f5ff4e9..e500e9474 100644 --- a/core/listener/forwardproxy/websocket.go +++ b/core/listener/forwardproxy/websocket.go @@ -838,6 +838,7 @@ func (ws *wsSocket) newTurnContext(body []byte) *pipeline.Context { // process on the same session, and without it no turn row names the header that named // the session — the rows #1535's fold counts procs and owner from. pctx.SetAttribution(hs.AttributedProc(), hs.AttributedClientProc(), hs.NamingHeader()) + pctx.SetNamedSession(hs.NamedSession()) pctx.SetProject(hs.Project()) return pctx } diff --git a/core/pipeline/attributed.go b/core/pipeline/attributed.go index 3758a9914..234491c27 100644 --- a/core/pipeline/attributed.go +++ b/core/pipeline/attributed.go @@ -27,6 +27,20 @@ func (c *Context) AttributedProc() string { return c.attributedProc } // none did. Recorded as SessionEvent.SessionHeader on the request's own row. func (c *Context) NamingHeader() string { return c.namingHeader } +// SetNamedSession pins the session id the request named with its session-id header (the one +// NamingHeader names), "" when none did. FIRST CALL WINS, with its own flag, as SetProject does: +// the listener sets it with SetAttribution, before any plugin runs, so a plugin that rewrites +// the session-id header cannot change which session the request named (#1923). +func (c *Context) SetNamedSession(id string) { + if c.namedSet { + return + } + c.namedSet, c.namedSession = true, id +} + +// NamedSession is the pinned session id the request named, "" when none was pinned. +func (c *Context) NamedSession() string { return c.namedSession } + // SetProject pins the project label (project.Label of the agent process's repo root) for // every row recorded from this request. FIRST CALL WINS, as SetAttribution does, with its // own flag: a path that pins "" keeps "", and no later call can write a label. diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 566458742..db2be5123 100644 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -184,6 +184,9 @@ type Context struct { clientProc string namingHeader string attributed bool + // namedSession and namedSet hold SetNamedSession's pin. + namedSession string + namedSet bool // project and projectSet hold SetProject's pin. project string diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index b573c017a..4e63bf493 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -94,6 +94,10 @@ type Stats struct { DroppedCaptures uint64 // Paused is true after ENOSPC, until the next retention pass. Paused bool + // FailedClosed is true once more sessions asked not to be recorded, while the queue was + // full, than the archive keeps track of (#1923). Until a restart it records no event of a + // conversation. + FailedClosed bool } // Archive is the session archive: a session.Recorder that writes every event the store appends @@ -115,11 +119,9 @@ type Archive struct { maxOpen int tickEvery time.Duration - // Test seams: gate holds the writer before each op; segHook sees each new segment writer; - // deliverWait is called while deliver waits for room. - gate chan struct{} - segHook func(*segmentWriter) - deliverWait func() + // Test seams: gate holds the writer before each op; segHook sees each new segment writer. + gate chan struct{} + segHook func(*segmentWriter) ops chan op stop chan struct{} @@ -133,10 +135,21 @@ type Archive struct { // recent clear, for AwaitClear; see clear.go. mu sync.Mutex lastSeq map[string]uint64 - // stopped is set by the writer once it has drained for Close: deliver then refuses an - // opt-out op rather than queue one nothing will read. Under mu. + // stopped is set by the writer once it has drained for Close: sendLocked and optOutLocked + // then refuse rather than queue what nothing will read. Under mu. stopped bool - pending *pendingClear + // sent is the position of the last op sendLocked queued, and marks the position of the last + // queued op that can move or remove an opt-out's mark (a rename, a delete, a clear). optOuts + // holds, per session id, the positions after which its opt-out applies, for opt-outs that + // found no room in the queue; see norecord.go. optOutN is len(optOuts), for the writer's fast + // path. All under mu. + sent, marks uint64 + optOuts map[string][]uint64 + optOutN atomic.Int64 + // failClosed is set when optOuts reached its bound: from then on until a restart, no event + // of a conversation is recorded (#1923). + failClosed atomic.Bool + pending *pendingClear // deletes is the most recent delete per session id, for AwaitDelete; see delete.go. nil // until the first. deletes map[string]*pendingClear @@ -164,7 +177,8 @@ type Archive struct { idxMu sync.RWMutex index map[string]*indexEntry - // Writer goroutine only. + // Writer goroutine only. at is the position of the last positioned op the writer took. + at uint64 sessions map[string]*sessionState lastMeta time.Time lastPrune time.Time @@ -220,6 +234,7 @@ type op struct { ev model.SessionEvent cap *model.RequestCapture // ev's request capture, nil for most events after uint64 // where a store entry began: on its first queued event, and on its rename + n uint64 // its position, from sendLocked; 0 for an op sent without one clear *pendingClear f func() done chan struct{} @@ -342,21 +357,31 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if a.closed.Load() { return } - // A flagged event of a conversation is never dropped (#1923): the writer's mark is the - // opt-out's only state, and a DELETE, CLEAR or prune still queued may be about to remove it, - // so no event may assume the session is already marked. See deliver. - mustDeliver := e.NoRecord && session.IsConversation(sessionID) + // A flagged event of a conversation is an opt-out (#1923), never lost: when the queue has no + // room for it, the opt-out is kept at its place in the queue instead (optOutLocked), and the + // event itself, withheld anyway, goes. Nothing here waits. + optOut := e.NoRecord && session.IsConversation(sessionID) a.mu.Lock() if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq } - // Every send happens under the store's write lock, so this check and the send below cannot - // race another producer; the writer only ever makes room. - if !mustDeliver && len(a.ops) >= cap(a.ops)-reserve { + if a.failClosed.Load() && session.IsConversation(sessionID) { a.mu.Unlock() a.dropped.Add(1) return } + // The reserve is for renames, deletes and clears. Every send happens under the store's + // write lock, so this check and the send below cannot race another producer; the writer + // only ever makes room. + if len(a.ops) >= cap(a.ops)-reserve { + if optOut { + a.optOutLocked(sessionID) + } else { + a.dropped.Add(1) + } + a.mu.Unlock() + return + } o := op{kind: opEvent, id: sessionID, ev: *e} // A request that asked not to be recorded queues no bodies: the writer withholds its // session anyway (withhold), so the copy would only take capture budget from others. @@ -372,26 +397,39 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if starting && !st.queued { o.after = st.after } - if mustDeliver { + if a.sendLocked(o) { if starting && !st.queued { st.queued = true a.started[sessionID] = st } a.mu.Unlock() - a.deliver(o) // outside a.mu: it may wait for the writer, which takes a.mu return } + if optOut { + a.optOutLocked(sessionID) + } else { + a.dropped.Add(1) + } + a.mu.Unlock() + a.releaseCapture(o.cap) +} + +// sendLocked queues o without blocking and gives it its position in the queue, which is how the +// writer applies a kept opt-out at its place (optOutLocked). It reports whether o was queued: not +// when the queue is full, and not once the writer has stopped for Close. The caller holds a.mu. +// Every producer that holds the store's write lock sends through here, so positions follow the +// queue's order. +func (a *Archive) sendLocked(o op) bool { + if a.stopped { + return false + } + o.n = a.sent + 1 select { case a.ops <- o: - if starting && !st.queued { - st.queued = true - a.started[sessionID] = st - } - a.mu.Unlock() + a.sent = o.n + return true default: - a.mu.Unlock() - a.dropped.Add(1) - a.releaseCapture(o.cap) + return false } } @@ -442,10 +480,12 @@ func (a *Archive) Rekeyed(oldID, newID string) { } a.lastSeq[newID] = max(a.lastSeq[newID], n) } + ok := a.sendLocked(op{kind: opRename, id: oldID, to: newID, after: st.after}) + if ok { + a.marks = a.sent // a rename moves a mark: see optOutLocked + } a.mu.Unlock() - select { - case a.ops <- op{kind: opRename, id: oldID, to: newID, after: st.after}: - default: + if !ok { a.droppedRenames.Add(1) // Session ids are attribution keys the operator needs to find the // history this rename left behind. @@ -473,6 +513,7 @@ func (a *Archive) Stats() Stats { DroppedRenames: a.droppedRenames.Load(), DroppedCaptures: a.droppedCaptures.Load(), Paused: a.paused.Load(), + FailedClosed: a.failClosed.Load(), } } @@ -510,15 +551,18 @@ func (a *Archive) run() { case o := <-a.ops: a.take(o) case <-t.C: + a.applyOptOuts(false) a.tick() case <-a.stop: a.drain() - // From here no opt-out op is accepted (deliver checks stopped under a.mu), so one - // more drain takes every op that was (#1923). + // From here nothing more is queued or kept (sendLocked and optOutLocked check + // stopped under a.mu), so one more drain takes every op that was, and every kept + // opt-out is due (#1923). shutdown then writes their marks. a.mu.Lock() a.stopped = true a.mu.Unlock() a.drain() + a.applyOptOuts(true) a.closeErr = a.shutdown() return } @@ -541,6 +585,11 @@ func (a *Archive) take(o op) { if a.gate != nil { <-a.gate } + // A kept opt-out applies before the first op queued after it (#1923). + a.applyOptOuts(false) + if o.n > 0 { + a.at = o.n + } switch o.kind { case opEvent: if o.after > 0 { diff --git a/core/session/archive/clear.go b/core/session/archive/clear.go index e0c24b1c3..77c08387b 100644 --- a/core/session/archive/clear.go +++ b/core/session/archive/clear.go @@ -53,16 +53,19 @@ func (a *Archive) Cleared() { a.mu.Lock() p.seqs = maps.Clone(a.lastSeq) a.pending = p - a.mu.Unlock() if a.closed.Load() { + a.mu.Unlock() p.finish(ClearResult{}, errors.New("the session archive is closed")) return } // Into the reserve, like a rename: Record stops short of it, so only renames and clears can // fill it. - select { - case a.ops <- op{kind: opClear, clear: p}: - default: + ok := a.sendLocked(op{kind: opClear, clear: p}) + if ok { + a.marks = a.sent // a clear removes every mark: see optOutLocked + } + a.mu.Unlock() + if !ok { slog.Error("session archive: a clear was lost to a full queue; history on disk is unchanged") p.finish(ClearResult{}, errors.New("the session archive's queue is full; nothing on disk was cleared")) } diff --git a/core/session/archive/delete.go b/core/session/archive/delete.go index ed53b0a6f..89d375aa6 100644 --- a/core/session/archive/delete.go +++ b/core/session/archive/delete.go @@ -26,15 +26,18 @@ func (a *Archive) Deleted(sessionID string) { a.deletes = map[string]*pendingClear{} } a.deletes[sessionID] = p - a.mu.Unlock() if a.closed.Load() { + a.mu.Unlock() p.finish(ClearResult{}, errors.New("the session archive is closed")) return } // Into the reserve, like a rename and a clear: Record stops short of it. - select { - case a.ops <- op{kind: opDelete, id: sessionID, clear: p}: - default: + ok := a.sendLocked(op{kind: opDelete, id: sessionID, clear: p}) + if ok { + a.marks = a.sent // a delete removes a mark: see optOutLocked + } + a.mu.Unlock() + if !ok { slog.Error("session archive: a delete was lost to a full queue; the session's history on disk is unchanged", "session", sessionID) p.finish(ClearResult{}, errors.New("the session archive's queue is full; nothing on disk was erased")) diff --git a/core/session/archive/import.go b/core/session/archive/import.go index 0690ea6ce..1fc4ba8aa 100644 --- a/core/session/archive/import.go +++ b/core/session/archive/import.go @@ -66,9 +66,12 @@ func (a *Archive) writeSession(id string, events []model.SessionEvent, replace b a.mu.Lock() old := a.lastSeq[id] switch { - case s != nil && s.meta.Summary.NoRecord(): - // The session asked not to be recorded (#1923). Its marker is the user's own choice, - // so no import adds to it or replaces it, also one whose events were imported first. + case s != nil && s.meta.Summary.NoRecord(), a.failClosed.Load(): + // The session asked not to be recorded (#1923), or the archive failed closed and no + // longer knows which sessions asked. A kept opt-out needs no check here: this op is + // queued after its place, so the writer has applied it. The marker is the user's own + // choice, so no import adds to it or replaces it, also one whose events were imported + // first. a.mu.Unlock() return ErrLive case s == nil && old > 0: diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index c7905ab49..f2b11293f 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -10,16 +10,30 @@ import ( ) // THE WRITER OWNS THE OPT-OUT (#1923). A session is withheld when its fold is marked noRecord, -// and only the writer goroutine marks a fold, in the same FIFO as renames, deletes, clears and -// prune, so they cannot reorder against it. There is no other copy of the state: RecordCapture -// never drops a flagged event of a conversation, and Withhold never drops its op (deliver), so -// nothing has to be remembered elsewhere until the writer gets to it. +// and only the writer goroutine marks a fold, in the same FIFO order as renames, deletes, clears +// and prune, so they cannot reorder against it. +// +// AN OPT-OUT IS NEVER LOST, AND NO PRODUCER WAITS FOR IT. A flagged event of a conversation, or a +// Withhold, is queued like any op when there is room below the reserve. When there is not (the +// writer is a full queue behind, e.g. on a stalled disk), it is kept instead in optOuts, tagged +// with the position of the last op queued before it, and the writer applies it at that place: after +// every op queued before it, before every op queued after it (applyOptOuts). So it lands where it +// would have been in the queue: after a DELETE, CLEAR or rename queued earlier, and before the +// session's later rows, which the mark then withholds. The producer, which holds the session +// store's write lock, never waits, so a stalled writer never stalls other sessions' traffic. +// +// optOuts holds one entry per session id, removed when the writer applies it. It grows only while +// the queue is full, and is bounded at as many ids as the queue holds ops (a.depth). Past that the +// archive FAILS CLOSED: until a restart it records no event of a conversation, because it no longer +// knows which ones asked not to be recorded. On a single-user laptop that many opted-out sessions +// during one stall means forged session ids. // withhold keeps session id out of the archive from now on. It runs on the writer goroutine for an // event that asked not to be recorded (SessionEvent.NoRecord, which listeners set only for a -// request that names its own session), for an opNoRecord (Withhold), and for every later event of -// a session already withheld, flagged or not: a session's header-less calls (telemetry, say) are -// filed to it by process or client affinity and carry no flag. +// request that names its own session), for an opNoRecord (Withhold), for a kept opt-out +// (applyOptOuts), and for every later event of a session already withheld, flagged or not: a +// session's header-less calls (telemetry, say) are filed to it by process or client affinity and +// carry no flag. // // FROM NOW ON, NEVER BACKWARDS. What the archive already holds for id stays: it was recorded // before the opt-out, and the session-id header that names id is client-asserted, so a request @@ -58,67 +72,89 @@ func (a *Archive) withhold(id string, s *sessionState, now time.Time) { } // Withhold implements session.Withholder: the store tells the archive that sessionID is withheld -// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). The op it -// queues is never dropped (deliver), and it precedes those rows in the FIFO, because the store -// calls this before the request is hopped. It runs under the store's write lock. +// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). Its opt-out +// is queued, or kept at its place when the queue is full, so it precedes those rows, because the +// store calls this before the request is hopped. It runs under the store's write lock and never +// waits. +// +// Every time, also for a session the writer has marked already: a DELETE, CLEAR or prune still +// queued may be about to remove that mark. Withholding a marked session is cheap. func (a *Archive) Withhold(sessionID string) { if a.closed.Load() || !session.IsConversation(sessionID) { return } - // Every time, also for a session the writer has marked already: a DELETE, CLEAR or prune - // still queued may be about to remove that mark. Withholding a marked session is cheap. - a.deliver(op{kind: opNoRecord, id: sessionID}) + a.mu.Lock() + defer a.mu.Unlock() + if len(a.ops) >= cap(a.ops)-reserve || !a.sendLocked(op{kind: opNoRecord, id: sessionID}) { + a.optOutLocked(sessionID) + } } -// deliver queues o, an op an opt-out depends on, and never drops it: when the queue has no room -// below the reserve it waits for the writer. The reserve stays for renames, deletes and clears, -// which must not be crowded out by flagged events. -// -// PRIVACY OVER LATENCY, a deliberate trade: while the archive is saturated (its writer a full -// queue behind, e.g. on a stalled disk) a request of an opted-out session waits here, without -// bound if the disk never recovers. An unsaturated archive never waits. +// optOutLocked keeps the opt-out of id, which found no room in the queue, at its place: after the +// last op queued so far (a.sent). The caller holds a.mu. // -// No deadlock: the caller holds the store's write lock and must not hold a.mu. The writer takes -// a.mu and never the store's lock, so it keeps draining, also while paused. Only producers that -// hold the store's lock send, so once there is room below the reserve the send cannot block, and -// the FIFO order of the store's appends holds. It gives up only when the archive closes, and -// says so. -func (a *Archive) deliver(o op) { - for len(a.ops) >= cap(a.ops)-reserve { - if a.deliverWait != nil { - a.deliverWait() // tests: the producer is waiting for room - } - select { - case <-a.done: - a.undelivered(o) - return - case <-time.After(time.Millisecond): - } - } - // The send and the writer's stop are ordered by a.mu: an op is either queued before the - // writer's last drain, and taken by it, or refused here and counted. Non-blocking: there is - // room below the reserve, and no other producer sends while the store's lock is held. - a.mu.Lock() +// One entry per id. A later opt-out of the same id adds a position only when a rename, delete or +// clear was queued after the entry's last one (a.marks): that op may move or remove the mark, so +// the opt-out must apply again after it, as it would from the queue. Otherwise the mark the entry +// already applies covers it. So an entry holds at most one position per such op still queued. +func (a *Archive) optOutLocked(id string) { if a.stopped { - a.mu.Unlock() - a.undelivered(o) + a.dropped.Add(1) + slog.Warn("session archive: closed before a request that asked not to be recorded reached the writer; nothing of it was written") return } - select { - case a.ops <- o: - a.mu.Unlock() - default: - a.mu.Unlock() - a.undelivered(o) + tags, ok := a.optOuts[id] + switch { + case !ok: + if len(a.optOuts) >= a.depth { + if !a.failClosed.Swap(true) { + slog.Error("session archive: more sessions asked not to be recorded, while the writer was behind, than it keeps track of; " + + "recording no conversation until a restart") + } + a.dropped.Add(1) + return + } + if a.optOuts == nil { + a.optOuts = map[string][]uint64{} + } + a.optOuts[id] = []uint64{a.sent} + a.optOutN.Add(1) + case tags[len(tags)-1] < a.marks: + a.optOuts[id] = append(tags, a.sent) } } -// undelivered counts and reports an opt-out op lost to the archive closing. No session id: it -// comes from a client's request header. -func (a *Archive) undelivered(o op) { - a.releaseCapture(o.cap) - a.dropped.Add(1) - slog.Warn("session archive: closed before a request that asked not to be recorded reached the writer; nothing of it was written") +// applyOptOuts withholds every kept opt-out that is due: its position is at or before a.at, the +// last positioned op the writer took, or, with all, every one (Close, once nothing more is queued). +// It runs on the writer goroutine before each op, so an opt-out applies before the first op queued +// after it, and on each tick, for one with no op behind it. +func (a *Archive) applyOptOuts(all bool) { + if a.optOutN.Load() == 0 { + return + } + var due []string + a.mu.Lock() + for id, tags := range a.optOuts { + i := 0 + for i < len(tags) && (all || tags[i] <= a.at) { + i++ + } + if i == 0 { + continue + } + due = append(due, id) + if i == len(tags) { + delete(a.optOuts, id) + a.optOutN.Add(-1) + } else { + a.optOuts[id] = tags[i:] + } + } + a.mu.Unlock() + now := a.now() + for _, id := range due { + a.withhold(id, a.sessions[id], now) + } } // withheld reports whether e, an event of id, is to be withheld: it asks it, or the session is @@ -141,18 +177,28 @@ func later(a, b time.Time) time.Time { // markerOnly reduces s, a withheld session that has nothing left on disk, to its marker: the // fold drops what the expired segments contributed (title, counts, projects), so nothing -// derived from the prompt outlives them, and keeps only the mark. prune calls it for a marker it -// keeps. +// derived from the prompt outlives them, and keeps only the mark. The summary of what the session +// held when the store's entry began (before) goes too, for the same reason: the store's list row +// merges it in. prune calls it for a marker it keeps. func (a *Archive) markerOnly(s *sessionState) { - if sum := s.meta.Summary.Summary(s.meta.ID); sum.EventCount == 0 && sum.Title == "" { + if onlyMark(s.meta.Summary, s.meta.ID) && (s.before == nil || onlyMark(s.before, s.meta.ID)) { return // already only a marker: no rewrite on every prune } fold := session.NewSummaryFold() fold.MarkNoRecord() s.meta.Summary, s.metaDirty = fold, true + if s.before != nil { + s.before = fold.Clone() + } a.publish(s) } +// onlyMark reports whether f, a fold of session id, holds nothing derived from content. +func onlyMark(f *session.SummaryFold, id string) bool { + sum := f.Summary(id) + return sum.EventCount == 0 && sum.Title == "" +} + // keepMarker reports whether prune keeps s, a session with nothing on disk, because it marks a // withheld session. A marker lives retainDays after the session's last event, like a segment: // after that, the agent's own transcript has usually expired too. Open makes retainDays positive. diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index 7ab1425e4..788ae5891 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -2,6 +2,7 @@ package archive import ( "errors" + "fmt" "path/filepath" "sync" "testing" @@ -74,19 +75,61 @@ func TestNoRecord_HopMarkSurvivesEviction(t *testing.T) { } } +// gateWriter holds a's writer before its next op and returns an idempotent release, also +// registered as cleanup with a's Close, so a failing test never leaves the writer gated. +func gateWriter(t *testing.T, a *Archive) (release func()) { + t.Helper() + gate := make(chan struct{}) + a.gate = gate + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(func() { + release() + _ = a.Close() + }) + return release +} + +// fillToReserve queues events of id until the queue is at the reserve line, with the gated +// writer holding the first one, and returns how many it recorded. +func fillToReserve(t *testing.T, a *Archive, r *recorder, id string) int { + t.Helper() + evs := synthSession(390, 1, 512)[:1] + r.record(id, evs...) + deadline := time.Now().Add(10 * time.Second) + for len(a.ops) > 0 { // the writer has taken it and waits at the gate + if time.Now().After(deadline) { + t.Fatal("the writer never took the first op") + } + time.Sleep(time.Millisecond) + } + n := 1 + for len(a.ops) < cap(a.ops)-reserve { + r.record(id, plain(n)...) + n++ + } + return n +} + +// plain is one unflagged event, distinct by i. +func plain(i int) []pipeline.SessionEvent { + e := synthSession(391, 1, 512)[0] + e.At = e.At.Add(time.Duration(i) * time.Millisecond) + return []pipeline.SessionEvent{e} +} + // The writer's mark is the opt-out's only state, so a flagged event of a session not marked yet is -// never dropped: with the queue full, reserve included, it waits for the writer instead. -func TestNoRecord_AFlaggedEventIsNeverDroppedWhileUnmarked(t *testing.T) { +// never lost: with the queue full, reserve included, it is kept at its place and returns at once. +func TestNoRecord_AFlaggedEventIsNeverLostWhileUnmarked(t *testing.T) { root := t.TempDir() a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) - a.gate = make(chan struct{}) + release := gateWriter(t, a) r := newRecorder(a) evs := synthSession(332, 3, 512) // 9 events r.record("s1", evs[:6]...) // to the reserve for len(a.ops) < cap(a.ops) { // and the reserve too a.Rekeyed("filler", "filler2") } - dropped := a.Stats().DroppedEvents done := make(chan struct{}) go func() { r.record("s1", flagged(evs[6:7])...) @@ -94,14 +137,10 @@ func TestNoRecord_AFlaggedEventIsNeverDroppedWhileUnmarked(t *testing.T) { }() select { case <-done: - t.Fatal("a flagged event of an unmarked session returned with the queue full: it was dropped") - case <-time.After(200 * time.Millisecond): - } - close(a.gate) - <-done - if got := a.Stats().DroppedEvents; got != dropped { - t.Fatalf("dropped events %d → %d, want the flagged one delivered", dropped, got) + case <-time.After(2 * time.Second): + t.Fatal("a flagged event waited for the writer") } + release() settle(a) r.record("s1", evs[7:]...) a.Close() @@ -110,6 +149,190 @@ func TestNoRecord_AFlaggedEventIsNeverDroppedWhileUnmarked(t *testing.T) { } } +// clawgenti's probe on #1959, through the real store: with the writer stuck (a hung fsync, a +// stalled mount) and the queue at its reserve line, one opted-out request must not block any other +// session's traffic, nor itself. Its opt-out still lands, and no flagged event took the reserve. +func TestNoRecord_AStuckWriterNeverBlocksTheStore(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + st := session.New(time.Hour, 0, 0) + t.Cleanup(st.Close) // registered first, so it runs after gateWriter's release + st.AddRecorder(a) + release := gateWriter(t, a) + evs := synthSession(392, 4, 512) // 12 events + st.Append("s1", evs[0]) + for deadline := time.Now().Add(10 * time.Second); len(a.ops) > 0; { + if time.Now().After(deadline) { + t.Fatal("the writer never took the first op") + } + time.Sleep(time.Millisecond) + } + n := 1 + for ; len(a.ops) < cap(a.ops)-reserve; n++ { + st.Append("s1", evs[n]) + } + fl := evs[n] + fl.NoRecord = true + other := synthSession(393, 1, 512) + done := make(chan string, 3) + go func() { st.Append("s1", fl); done <- "the opted-out Append" }() + go func() { st.Append("s2", other[0]); done <- "another session's Append" }() + go func() { st.ListSessions(); done <- "ListSessions" }() + for range 3 { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the store blocked behind a stuck archive writer") + } + } + if q := len(a.ops); q > cap(a.ops)-reserve { + t.Fatalf("queue at %d with the writer gated, past the reserve line %d: a flagged event took the reserve", q, cap(a.ops)-reserve) + } + a.Rekeyed("x1", "x2") // the reserve is still free for a rename + if got := a.Stats().DroppedRenames; got != 0 { + t.Fatalf("dropped %d renames: the reserve was taken", got) + } + release() + settle(a) + for _, e := range evs[n+1:] { + st.Append("s1", e) // unflagged, after the opt-out + } + st.Append("s2", other[1]) + settle(a) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != n { + t.Fatalf("s1 holds %d events, want the %d recorded before the opt-out", len(got), n) + } + if got := onDisk(t, root, "s2"); len(got) != 1 { + t.Fatalf("s2 holds %d events, want its row after the writer recovered", len(got)) + } +} + +// A kept opt-out applies at its place in the queue: after a DELETE queued before it, which would +// otherwise remove its mark, and again after a DELETE queued between two opt-outs of one session +// (the entry adds a position). In both cases the unflagged rows that follow are withheld. +func TestNoRecord_AKeptOptOutKeepsItsPlaceAfterADelete(t *testing.T) { + for _, tc := range []struct { + name string + ops func(a *Archive) + }{ + {"delete, then opt-out", func(a *Archive) { + a.Deleted("s1") + a.Withhold("s1") + }}, + {"opt-out, delete, opt-out", func(a *Archive) { + a.Withhold("s1") + a.Deleted("s1") + a.Withhold("s1") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + r.record("s1", flagged(synthSession(394, 1, 512))...) + settle(a) // s1 is marked + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + tc.ops(a) // with the queue at the line: every Withhold is kept, not queued + if a.optOutN.Load() != 1 { + t.Fatalf("%d kept opt-outs, want 1 for s1", a.optOutN.Load()) + } + release() + if _, err := a.AwaitDelete("s1", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + settle(a) + r.seq["s1"] = 0 + r.record("s1", synthSession(395, 1, 512)...) // unflagged, after the opt-out + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the opt-out was applied before the delete", segs) + } + }) + } +} + +// One entry per session, however often it opts out while the queue is full, and none left once +// the writer has applied it. +func TestNoRecord_KeptOptOutsAreOnePerSessionAndCleared(t *testing.T) { + a := openTest(t, t.TempDir(), newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for range 1000 { + a.Withhold("s1") + r.record("s1", flagged(plain(1))...) + } + a.mu.Lock() + n, tags := len(a.optOuts), len(a.optOuts["s1"]) + a.mu.Unlock() + if n != 1 || tags != 1 { + t.Fatalf("%d entries, s1 with %d positions; want 1 and 1", n, tags) + } + release() + settle(a) + if got := a.optOutN.Load(); got != 0 { + t.Fatalf("%d kept opt-outs after the writer caught up, want 0", got) + } +} + +// A kept opt-out still pending at Close is written as a marker, so the session stays withheld +// after a restart. A praxis Withhold is kept the same way. +func TestNoRecord_AKeptOptOutSurvivesCloseAndRestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + r.record("s1", flagged(plain(1))...) + a.Withhold("s2") + release() + a.Close() + + a2 := openTest(t, root, newClock()) + r2 := newRecorder(a2) + r2.seq["s1"], r2.seq["s2"] = 1, 0 + r2.record("s1", synthSession(396, 1, 512)...) + r2.record("s2", synthSession(397, 1, 512)...) + a2.Close() + for _, id := range []string{"s1", "s2"} { + if segs := segmentsIn(t, root, id); len(segs) != 0 { + t.Fatalf("%s: segments %v after a restart, want none", id, segs) + } + } +} + +// Past its bound on kept opt-outs the archive fails closed: no conversation is recorded until a +// restart, and Stats says so. A shared bucket still records. +func TestNoRecord_TooManyKeptOptOutsFailClosed(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for i := range a.depth + 1 { + a.Withhold(fmt.Sprintf("forged-%d", i)) + } + if !a.Stats().FailedClosed { + t.Fatal("past the bound, the archive did not fail closed") + } + if got := a.optOutN.Load(); got != int64(a.depth) { + t.Fatalf("%d kept opt-outs, want the bound %d", got, a.depth) + } + release() + settle(a) + r.record("s9", synthSession(398, 1, 512)...) + r.record(session.DefaultSessionID, synthSession(399, 1, 512)...) + a.Close() + if segs := segmentsIn(t, root, "s9"); len(segs) != 0 { + t.Fatalf("segments %v for a conversation after failing closed, want none", segs) + } + if got := onDisk(t, root, session.DefaultSessionID); len(got) != 3 { + t.Fatalf("default holds %d events, want its 3", len(got)) + } +} + // A DELETE still queued when a praxis Withhold arrives: the session was marked, but the delete is // about to remove the mark, so Withhold must not take "already marked" for granted. The rows // ingested after it are withheld. @@ -156,44 +379,62 @@ func TestNoRecord_FlaggedEventWhileAClearIsQueued(t *testing.T) { } } -// Flagged events wait below the reserve, never in it: the reserve stays for renames, deletes and -// clears, which a burst of flagged events must not crowd out. +// Flagged events never go into the reserve: it stays for renames, deletes and clears, which a +// burst of flagged events must not crowd out. Past the reserve line they are kept, and the +// producer returns. func TestNoRecord_FlaggedEventsLeaveTheReserveAlone(t *testing.T) { root := t.TempDir() a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) - a.gate = make(chan struct{}) + release := gateWriter(t, a) r := newRecorder(a) - r.record("s1", synthSession(344, 2, 512)[:6]...) // up to the reserve - // More than the reserve holds, counting the slot the gated writer has already taken off - // the queue: if flagged events could use the reserve, they would fill it. + fillToReserve(t, a, r, "s1") + // More than the reserve holds: if flagged events could use the reserve, they would fill it. evs := make([]pipeline.SessionEvent, reserve+2) for i := range evs { evs[i].NoRecord = true } - waiting := make(chan struct{}) - var once sync.Once - a.deliverWait = func() { once.Do(func() { close(waiting) }) } done := make(chan struct{}) go func() { r.record("s2", evs...) close(done) }() select { - case <-waiting: // a flagged event reached the wait for room case <-done: - // Every flagged event was queued without waiting: they took the reserve. - case <-time.After(10 * time.Second): - t.Fatal("the flagged producer never reached deliver's wait") + case <-time.After(2 * time.Second): + t.Fatal("the flagged producer waited for the writer") + } + if q := len(a.ops); q > cap(a.ops)-reserve { + t.Fatalf("queue at %d with the writer gated, past the reserve line %d", q, cap(a.ops)-reserve) } a.Rekeyed("s1", "s3") // with the writer still gated: must still find a reserve slot if got := a.Stats().DroppedRenames; got != 0 { t.Fatalf("dropped %d renames: flagged events took the reserve", got) } - close(a.gate) - <-done + release() a.Close() } +// Review round 3: when retention leaves a withheld session with nothing, the summary of what it +// held when the store's entry began (before) goes too, so the store's list row shows no expired +// title or counts. +func TestNoRecord_MarkerOnlyDropsThePriorSummary(t *testing.T) { + a := openTest(t, t.TempDir(), newClock()) + defer a.Close() + r := newRecorder(a) + r.record("s1", synthSession(346, 1, 512)...) + settle(a) + a.inWriter(func() { + s := a.sessions["s1"] + s.before = s.meta.Summary.Clone() // as begin leaves it for a new store entry + s.meta.Summary.MarkNoRecord() + a.markerOnly(s) + if !onlyMark(s.before, "s1") || !onlyMark(s.meta.Summary, "s1") { + t.Errorf("after markerOnly: before %+v, fold %+v; want only the mark", + s.before.Summary("s1"), s.meta.Summary.Summary("s1")) + } + }) +} + // Review: when retention expires the last segment of a withheld session, the marker it keeps // holds only the mark, not the expired history's title and counts. func TestNoRecord_AnExpiredHistoryLeavesOnlyTheMark(t *testing.T) { @@ -218,9 +459,9 @@ func TestNoRecord_AnExpiredHistoryLeavesOnlyTheMark(t *testing.T) { } } -// Review: an op that reaches deliver after the writer stopped reading the queue for Close is -// refused and counted, never queued where nothing reads it. -func TestNoRecord_DeliverRefusesAfterTheWriterStopped(t *testing.T) { +// Review: an opt-out that reaches the archive after the writer stopped reading the queue for Close +// is refused and counted, never queued or kept where nothing reads it. +func TestNoRecord_AnOptOutIsRefusedAfterTheWriterStopped(t *testing.T) { a := openTest(t, t.TempDir(), newClock()) defer a.Close() a.mu.Lock() @@ -231,8 +472,8 @@ func TestNoRecord_DeliverRefusesAfterTheWriterStopped(t *testing.T) { if got := a.Stats().DroppedEvents; got != before+1 { t.Fatalf("dropped %d → %d, want the op counted as undelivered", before, got) } - if n := len(a.ops); n != 0 { - t.Fatalf("%d ops queued after the writer stopped, want none", n) + if n := len(a.ops); n != 0 || a.optOutN.Load() != 0 { + t.Fatalf("%d ops queued and %d opt-outs kept after the writer stopped, want none", n, a.optOutN.Load()) } a.mu.Lock() a.stopped = false From 89d558c06145d1f38313968786390ee209122311 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Thu, 19 Nov 2026 10:28:52 +0200 Subject: [PATCH 08/10] test: Cover a row queued while an opt-out is kept, and state the list's limits From an independent review of the kept opt-outs: - Test: an unflagged row that gets a queue slot while its session's opt-out is still kept is withheld. With the opt-out applied only after the row, the test fails. - Docs: the kept list bounds its ids, but each id's positions can grow by one per rename, delete or clear queued after its last one (about depth x depth in the worst case), and the writer scans the list before each op while any entry is kept. Failing closed is not durable: after a restart, a session whose opt-out did not fit is recorded again until it next asks. The error logged when it trips says so. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/session/archive/norecord.go | 17 +++++++++----- core/session/archive/norecord2_test.go | 31 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go index f2b11293f..e03e6dc9d 100644 --- a/core/session/archive/norecord.go +++ b/core/session/archive/norecord.go @@ -23,10 +23,16 @@ import ( // store's write lock, never waits, so a stalled writer never stalls other sessions' traffic. // // optOuts holds one entry per session id, removed when the writer applies it. It grows only while -// the queue is full, and is bounded at as many ids as the queue holds ops (a.depth). Past that the -// archive FAILS CLOSED: until a restart it records no event of a conversation, because it no longer -// knows which ones asked not to be recorded. On a single-user laptop that many opted-out sessions -// during one stall means forged session ids. +// the queue is full. Its ids are bounded at as many as the queue holds ops (a.depth). Each id's +// list of positions grows by at most one per rename, delete or clear queued after its last one, +// so in the worst case the list holds about depth × depth positions. While any entry is kept, the +// writer scans the list under a.mu before each op (applyOptOuts). +// +// Past the bound on ids the archive FAILS CLOSED: until a restart it records no event of a +// conversation, because it no longer knows which ones asked not to be recorded. The opt-outs that +// did not fit are not kept anywhere durable, so after a restart such a session's unflagged rows are +// recorded again until it sends another flagged request. On a single-user laptop that many +// opted-out sessions during one stall means forged session ids. // withhold keeps session id out of the archive from now on. It runs on the writer goroutine for an // event that asked not to be recorded (SessionEvent.NoRecord, which listeners set only for a @@ -109,7 +115,8 @@ func (a *Archive) optOutLocked(id string) { if len(a.optOuts) >= a.depth { if !a.failClosed.Swap(true) { slog.Error("session archive: more sessions asked not to be recorded, while the writer was behind, than it keeps track of; " + - "recording no conversation until a restart") + "recording no conversation until a restart. The opt-outs that did not fit are not remembered across that restart: " + + "those sessions' unflagged rows are recorded again until they next ask") } a.dropped.Add(1) return diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index 788ae5891..fcff2436e 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -253,6 +253,37 @@ func TestNoRecord_AKeptOptOutKeepsItsPlaceAfterADelete(t *testing.T) { } } +// An unflagged row of a session that gets a queue slot while its opt-out is still kept (the +// writer has made room, but not reached the opt-out's place) is withheld: the opt-out applies +// before it. +func TestNoRecord_ARowQueuedWhileTheOptOutIsKeptIsWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + r.record("s1", flagged(plain(1))...) // no room: kept + if a.optOutN.Load() != 1 { + t.Fatal("precondition: the opt-out is kept") + } + a.gate <- struct{}{} // one op through: the writer takes the next and waits again + for deadline := time.Now().Add(10 * time.Second); len(a.ops) >= cap(a.ops)-reserve; { + if time.Now().After(deadline) { + t.Fatal("the writer made no room") + } + time.Sleep(time.Millisecond) + } + r.record("s1", plain(2)...) // unflagged, queued behind the opt-out's place + if a.optOutN.Load() != 1 { + t.Fatal("the opt-out was applied before the row was queued") + } + release() + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: a row queued while the opt-out was kept was recorded", segs) + } +} + // One entry per session, however often it opts out while the queue is full, and none left once // the writer has applied it. func TestNoRecord_KeptOptOutsAreOnePerSessionAndCleared(t *testing.T) { From 5bb0e5e84072c2395f94df052fa3f87328be1002 Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Thu, 19 Nov 2026 16:50:19 +0200 Subject: [PATCH 09/10] docs: Document X-Cortex-Record and noRecord, and report failedClosed From clawgenti's fourth review of #1959: - docs/session-api.md: noRecord in GET /v1/sessions and in the event schema, failedClosed in the archive object, and a section on keeping a session out of the archive (only "off" counts; the request must name its own session; from then on, never backwards, and sticky; only DELETE erases; what it does not cover). - docs/session-export.md: a withheld session exports nothing from its opt-out on. - A flagged event goes on to normal admission while the archive is failed closed, as Withhold's does, so its mark is still written. Test: an opt-out after failing closed is withheld after a restart; without the change it fails. - ArchiveUsage reports failedClosed, copied from Stats. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- core/model/session.go | 4 ++ core/session/archive/archive.go | 4 +- core/session/archive/norecord2_test.go | 30 ++++++++++++++ core/sessionapi/archive.go | 1 + docs/session-api.md | 55 +++++++++++++++++++++++++- docs/session-export.md | 8 ++++ 6 files changed, 99 insertions(+), 3 deletions(-) diff --git a/core/model/session.go b/core/model/session.go index 4ea89d0a0..5775dc232 100644 --- a/core/model/session.go +++ b/core/model/session.go @@ -1079,6 +1079,10 @@ type ArchiveUsage struct { DroppedCaptures uint64 `json:"droppedCaptures,omitempty"` // Paused is true after the disk filled, until the archive's next retention pass. Paused bool `json:"paused,omitempty"` + // FailedClosed is true once more sessions asked not to be recorded, while the archive's + // writer was behind, than it keeps track of (#1923): until a restart it records no + // conversation. + FailedClosed bool `json:"failedClosed,omitempty"` } // ImportedSourcePrefix starts the Source of every event that an import wrote. diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 16af5b8e6..e5da5024c 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -365,7 +365,9 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq } - if a.failClosed.Load() && session.IsConversation(sessionID) { + // Failed closed, no conversation event is recorded. An opt-out still goes on to the + // admission below, as Withhold's does, so its mark is written when there is room. + if a.failClosed.Load() && session.IsConversation(sessionID) && !optOut { a.mu.Unlock() a.dropped.Add(1) return diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go index fcff2436e..66cb6f747 100644 --- a/core/session/archive/norecord2_test.go +++ b/core/session/archive/norecord2_test.go @@ -364,6 +364,36 @@ func TestNoRecord_TooManyKeptOptOutsFailClosed(t *testing.T) { } } +// Failed closed, the archive records no conversation, but a flagged event that finds room still +// writes its mark: the fail-closed state is lost at a restart, and the mark is not. +func TestNoRecord_AnOptOutAfterFailingClosedIsStillMarked(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for i := range a.depth + 1 { + a.Withhold(fmt.Sprintf("forged-%d", i)) + } + if !a.Stats().FailedClosed { + t.Fatal("precondition: the archive failed closed") + } + release() + settle(a) + r.record("s7", flagged(plain(1))...) // room again: the opt-out is admitted + settle(a) + a.Close() + + a2 := openTest(t, root, newClock()) + r2 := newRecorder(a2) + r2.seq["s7"] = 1 + r2.record("s7", synthSession(400, 1, 512)...) // unflagged, after the restart + a2.Close() + if segs := segmentsIn(t, root, "s7"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none: the opt-out sent while failed closed was lost", segs) + } +} + // A DELETE still queued when a praxis Withhold arrives: the session was marked, but the delete is // about to remove the mark, so Withhold must not take "already marked" for granted. The rows // ingested after it are withheld. diff --git a/core/sessionapi/archive.go b/core/sessionapi/archive.go index 70271075e..946d5953f 100644 --- a/core/sessionapi/archive.go +++ b/core/sessionapi/archive.go @@ -146,5 +146,6 @@ func (s *Server) withArchivedRows(resident []session.SessionSummary) ([]session. DroppedRenames: st.DroppedRenames, DroppedCaptures: st.DroppedCaptures, Paused: st.Paused, + FailedClosed: st.FailedClosed, } } diff --git a/docs/session-api.md b/docs/session-api.md index 904be7d3a..276c195a7 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -90,6 +90,10 @@ knowing by name: imported (see [`POST /v1/import`](#post-v1import--transcript-import-laptop-only)). It is absent when the session has one or more live events. So a live request for an imported session gives it a live event, and the mark goes. +- **`noRecord`** — `true` once the session asked not to be recorded, with the + `X-Cortex-Record` header: the archive keeps none of its events from then on. Absent + otherwise. It is sticky, and the archive's mark carries it across a restart. See + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive). - **Pending buckets.** A coding agent's calls before its process names a session are recorded under `pending:@.` (or `pending:` when the process could not be looked up), carrying their `proc`. Nothing renames a @@ -105,8 +109,11 @@ knowing by name: (`session.archive`, laptop only): every session the archive holds and memory does not, marked `resident: false`, plus a top-level `archive` object — `{bytes, maxBytes, retentionDays}` and, only when nonzero, `droppedEvents`, - `writeErrors`, `droppedRenames`, `droppedCaptures`, `paused`, any of which means the history has - gaps. A session in both is listed once, as its resident row, and that row + `writeErrors`, `droppedRenames`, `droppedCaptures`, `paused`, `failedClosed`, any of which means the history has + gaps. `failedClosed` is permanent until a restart: more sessions asked not to be + recorded, while the archive's writer was behind, than it keeps track of, so it + records no conversation at all (see + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive)). A session in both is listed once, as its resident row, and that row covers the session's whole history: `eventCount`, `totalTokens`, `costMicros`, `title`, `agent`, `createdAt` and the rest continue from the archive's fold of what came before this process's entry. **This applies to the default list @@ -397,6 +404,45 @@ The next `agentop import` brings an erased session back from its transcript, as an imported session. Its cost is not counted again: the erase kept the ledger rows, and a forget record tells the import which turns those rows hold. +## Keeping a session out of the archive + +A client keeps its own session out of the session archive by sending +`X-Cortex-Record: off` on its requests. For Claude Code that is +`ANTHROPIC_CUSTOM_HEADERS="X-Cortex-Record: off"`. The rules: + +- **Only `off` counts**, matched without case or padding. Any other value, or + none, records as before. The header is removed before any plugin runs, and it + is never forwarded upstream. +- **The request must name its own session** with a session-id header from + `session.id_headers`. A request that names none is filed by fallback (the + active session or `default`), so its flag is ignored: it must not stop another + session's recording. So the header does nothing with `id_headers` empty, on a + vmcontrol session (attribution is pinned empty there), and through ext_proc, + which pins no naming header (#1960). `default` and pending buckets are never + withheld. +- **From then on, never backwards.** From the first flagged request, the archive + keeps none of the session's events, also the header-less ones filed to it by + process or client affinity. What it already held stays: the session-id header + is client-asserted, so a request must not be able to erase another session's + history by naming it. +- **Sticky.** The archive keeps a content-free marker for the session, also + across a restart, for `retention_days` after its last event. A resumed session + without the header stays withheld, and sending `on` later does not resume + recording. An import refuses a withheld session. +- **Only [`DELETE /v1/sessions/{id}`](#delete-v1sessionsid--erase-one-session) + erases**, and `DELETE /v1/sessions` (which also clears the markers). After a + delete, the session records again unless it still sends the header, and the + next `agentop import` can bring it back from its transcript. +- **What it does not cover.** The store holds the rows in memory as usual, and + the live streams carry them, flagged `noRecord`. The cost ledger and usage + totals count the session: their rows carry no content. Plugins with their own + per-session state, such as context-guru's database, are not covered (#1962). + +The archive never makes a request wait for the opt-out. If more sessions opt +out while its writer is behind than it keeps track of (as many as its queue +holds, 4096 by default), it records no conversation until a restart and reports +`failedClosed` (see [`GET /v1/sessions`](#get-v1sessions)). + ## `POST /v1/import` — transcript import (laptop only) Reads the transcripts that an agent keeps on disk, and writes each session into @@ -945,6 +991,11 @@ Every event on `/v1/sessions/{id}` and `/v1/events` carries: [`POST /v1/import`](#post-v1import--transcript-import-laptop-only) wrote. Absent on a live event. The proxy did not see an imported exchange, so such an event has no `host` and no capture. +- `noRecord` — `true` on the request and response rows of a request that sent + `X-Cortex-Record: off` and named its own session (see + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive)). + Absent otherwise, and on every row of `default` and of a pending bucket. Such + rows are in the store and on the live streams as usual; the archive keeps none. - `a2a` / `mcp` / `inference` — protocol parser payloads (one at most). - `invocations` — per-plugin invocation records for every plugin that ran on the pipeline pass, structured `{inbound: [...], outbound: [...]}`. Each entry diff --git a/docs/session-export.md b/docs/session-export.md index 0a55cd438..dd86e2b2b 100644 --- a/docs/session-export.md +++ b/docs/session-export.md @@ -115,6 +115,14 @@ secret reaches the encoded bytes, which fails CI. The count above is pinned. `TestDocSecretCountMatchesAllSecrets` fails if this page and the test disagree. +### A session that asked not to be recorded + +The archive is the export's only source, so a session that sent `X-Cortex-Record: off` +exports none of its events from then on. What the archive held from before the opt-out +stays, and exports as usual. A session that asked from its first request exports no +records at all. See +[Keeping a session out of the archive](session-api.md#keeping-a-session-out-of-the-archive). + ### Questions an export answers - Token spend per model From 6ce74bedd453b54d3c77929d2a69c97c454c350e Mon Sep 17 00:00:00 2001 From: Michal Jacovi Date: Thu, 19 Nov 2026 21:34:31 +0200 Subject: [PATCH 10/10] fix: List noRecord as a field the gateway never sets praxis's every_session_event_field_is_set_or_listed reads the Go event golden, which now has noRecord (#1923). Cortex marks the session before the hop (Store.MarkNoRecord), so the gateway never sets it. Part of #1923. Assisted-By: Claude (Anthropic AI) Signed-off-by: Michal Jacovi --- praxis/filters/src/record/line.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/praxis/filters/src/record/line.rs b/praxis/filters/src/record/line.rs index f377be9f8..c4df10431 100644 --- a/praxis/filters/src/record/line.rs +++ b/praxis/filters/src/record/line.rs @@ -46,6 +46,8 @@ pub const NOT_SET_IN_PRAXIS_MODE: &[&str] = &[ "parentProject", // An import sets it; every gateway row is live. "source", + // Cortex marks the session before the hop (Store.MarkNoRecord), so the gateway never sets it. + "noRecord", ]; fn is_zero(v: &i64) -> bool {