diff --git a/core/cost/ledger/norecord_test.go b/core/cost/ledger/norecord_test.go new file mode 100644 index 000000000..39eef2477 --- /dev/null +++ b/core/cost/ledger/norecord_test.go @@ -0,0 +1,33 @@ +package ledger + +import ( + "testing" + "time" +) + +// A session that asked not to be recorded (SessionEvent.NoRecord, #1923) is kept out of the +// session archive only. Its spend still counts here: ledger rows carry no content, and the cost +// views must stay complete. +func TestWriter_CountsAWithheldSession(t *testing.T) { + dir := t.TempDir() + now := at + w := newTestWriter(t, dir, func() time.Time { return now }) + + for i := 0; i < 2; i++ { + ev := costedEvent(t, "gw", "m", 0.25, 100, 50) + ev.NoRecord = true + w.Record("s1", ev) + } + now = at.Add(time.Minute) + later := costedEvent(t, "gw", "m", 0.10, 10, 5) + later.At, later.NoRecord = now, true + w.Record("s1", later) + if err := w.sync(); err != nil { + t.Fatalf("sync: %v", err) + } + + rows := readAllRows(t, dir) + if len(rows) != 1 || rows[0].Requests != 2 || rows[0].CostMicros != 500_000 { + t.Fatalf("rows = %+v, want one closed minute of 2 requests and 500000 micros", rows) + } +} diff --git a/core/listener/extproc/norecord_test.go b/core/listener/extproc/norecord_test.go new file mode 100644 index 000000000..6880e2356 --- /dev/null +++ b/core/listener/extproc/norecord_test.go @@ -0,0 +1,89 @@ +package extproc + +import ( + "context" + "net/http" + "slices" + "testing" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + + "github.com/rossoctl/cortex/core/listener/skiphost" + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// removed returns the RemoveHeaders of a request-headers or request-body response. +func removed(t *testing.T, resp *extprocv3.ProcessingResponse) []string { + t.Helper() + var cr *extprocv3.CommonResponse + switch r := resp.Response.(type) { + case *extprocv3.ProcessingResponse_RequestHeaders: + cr = r.RequestHeaders.Response + case *extprocv3.ProcessingResponse_RequestBody: + cr = r.RequestBody.Response + } + if cr == nil || cr.HeaderMutation == nil { + return nil + } + return cr.HeaderMutation.RemoveHeaders +} + +// A skip_hosts destination runs no pipeline, but the header is still for Cortex only: both the +// headers phase and the body phase remove it. +func TestNoRecord_ExtProcSkipHostRemovesTheHeader(t *testing.T) { + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + skip, err := skiphost.New([]string{"otel-collector.*.svc.cluster.local"}) + if err != nil { + t.Fatal(err) + } + srv, _ := newSkipServer(t, store, skip) + hdrs := func() *corev3.HeaderMap { + return makeHeaders(":authority", "otel-collector.ns.svc.cluster.local:8335", ":path", "/v1/traces", + "x-cortex-record", "off") + } + + stream := &mockStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{outboundRequest(hdrs())}} + _ = srv.Process(stream) + if len(stream.responses) != 1 { + t.Fatalf("headers phase: %d responses, want 1", len(stream.responses)) + } + if got := removed(t, stream.responses[0]); !slices.Contains(got, "x-cortex-record") { + t.Fatalf("headers phase: RemoveHeaders %v, want x-cortex-record removed", got) + } + + resp, pctx := srv.handleOutboundBody(&mockStream{ctx: context.Background()}, hdrs(), []byte(`{}`)) + if pctx != nil { + t.Fatal("body phase ran the pipeline for a skipped host") + } + if got := removed(t, resp); !slices.Contains(got, "x-cortex-record") { + t.Fatalf("body phase: RemoveHeaders %v, want x-cortex-record removed", got) + } +} + +// ext_proc forwards the wire headers and only the changes it emits. ResolveClient removed +// X-Cortex-Record from pctx.Headers before the listener cloned them, so the diff alone cannot +// see it: withHeaderMutation must remove it explicitly, whatever its value. +func TestNoRecord_ExtProcRemovesTheHeader(t *testing.T) { + for _, v := range []string{"off", "on"} { + h := http.Header{} + h.Set(model.RecordHeader, v) + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + orig := pctx.Headers.Clone() // as handleInbound and handleOutbound do + got := removed(t, withHeaderMutation(allowResponse(), pctx, orig)) + if !slices.Contains(got, "x-cortex-record") { + t.Fatalf("value %q: RemoveHeaders = %v, want x-cortex-record", v, got) + } + } + // No header: nothing extra is removed. + pctx := &pipeline.Context{Headers: http.Header{"User-Agent": {"curl/8"}}} + pctx.ResolveClient() + if got := removed(t, withHeaderMutation(allowResponse(), pctx, pctx.Headers.Clone())); slices.Contains(got, "x-cortex-record") { + t.Fatalf("RemoveHeaders = %v with no header sent", got) + } +} diff --git a/core/listener/extproc/server.go b/core/listener/extproc/server.go index 3ea7c6de8..4db084c00 100644 --- a/core/listener/extproc/server.go +++ b/core/listener/extproc/server.go @@ -26,6 +26,7 @@ import ( "github.com/rossoctl/cortex/core/listener/internal/sessionevent" "github.com/rossoctl/cortex/core/listener/internal/sseframe" "github.com/rossoctl/cortex/core/listener/skiphost" + "github.com/rossoctl/cortex/core/model" "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/session" ) @@ -504,7 +505,9 @@ func (s *Server) handleOutbound(stream extprocv3.ExternalProcessor_ProcessServer // handleResponseBody) and the deferred RunFinish to no-op as well — // all four phases are skipped consistently. See ListenerConfig.SkipHosts. if s.SkipHosts.Match(pctx.Host) { - return passResponse(), nil + // No pipeline ran, so pctx.Headers is its own original and only the X-Cortex-Record + // removal is emitted: the header is for Cortex only, also toward a skipped host (#1923). + return withHeaderMutation(passResponse(), pctx, pctx.Headers), nil } if s.Sessions != nil { @@ -555,7 +558,7 @@ func (s *Server) handleOutboundBody(stream extprocv3.ExternalProcessor_ProcessSe if pat, matched := s.SkipHosts.MatchPattern(pctx.Host); matched { slog.Info("ext_proc: skip_hosts match (body phase) — bypassing pipeline + session recording", "host", pipeline.SanitizeLogValue(pctx.Host), "pattern", pat, "path", pipeline.SanitizeLogValue(pctx.Path)) - return allowBodyResponse(), nil + return withHeaderMutation(allowBodyResponse(), pctx, pctx.Headers), nil // as in handleOutbound } if s.Sessions != nil { @@ -968,7 +971,9 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte var set []*corev3.HeaderValueOption var del []string for k, vv := range pctx.Headers { - if skip(k) || slices.Equal(orig[k], vv) { + // Never forward X-Cortex-Record (#1923), also when a plugin wrote it back: it is removed + // below, and a set beside the remove would leave the outcome to Envoy's ordering. + if skip(k) || slices.Equal(orig[k], vv) || strings.EqualFold(k, model.RecordHeader) { continue } if len(vv) == 0 { @@ -1008,6 +1013,11 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte del = append(del, strings.ToLower(k)) // plugin removed it } } + // X-Cortex-Record is for Cortex only (#1923). ResolveClient removed it from pctx.Headers + // before orig was cloned, so the loop above cannot see it: remove it from the wire here. + if pctx.RecordHeaderSeen() { + del = append(del, strings.ToLower(model.RecordHeader)) + } if len(set) == 0 && len(del) == 0 { return resp } diff --git a/core/listener/forwardproxy/identity.go b/core/listener/forwardproxy/identity.go index f7c98108d..e8dd43f48 100644 --- a/core/listener/forwardproxy/identity.go +++ b/core/listener/forwardproxy/identity.go @@ -80,12 +80,21 @@ func (s *Server) attributeForRecording(pctx *pipeline.Context, trusted string, c at = s.attribute(clientHeaders, chain) } pctx.SetAttribution(at.Proc, at.ClientProc, at.Header) + pctx.SetNamedSession(namedSession(at)) pctx.SetProject(s.projectFor(trusted, m, chain, at)) sid := at.RecordSession(trusted, s.activeSession) pctx.SetParent(s.parentFor(trusted, m, chain, at, sid)) return sid } +// namedSession is the session id a session-id header named in at, "" when none did. +func namedSession(at attribution.Attribution) string { + if at.Header == "" { + return "" + } + return at.Session +} + // tunnelSessionID is recordingSessionID for a tunnel's own row. Under client affinity // resolved is "" only for affinity's ambiguous answer or for no identity at all, and // neither sends a tunnel row to default: a CONNECT rarely carries a User-Agent, so the @@ -179,6 +188,7 @@ func (s *Server) hydrateSession(pctx *pipeline.Context, trusted string, clientHe at = s.attribute(clientHeaders, chain) } pctx.SetAttribution(at.Proc, at.ClientProc, at.Header) + pctx.SetNamedSession(namedSession(at)) pctx.SetProject(s.projectFor(trusted, m, chain, at)) sid, source := at.PluginSession(trusted, s.activeSession) pctx.SetParent(s.parentFor(trusted, m, chain, at, sid)) diff --git a/core/listener/forwardproxy/norecord_test.go b/core/listener/forwardproxy/norecord_test.go new file mode 100644 index 000000000..fa6ed23a4 --- /dev/null +++ b/core/listener/forwardproxy/norecord_test.go @@ -0,0 +1,289 @@ +package forwardproxy + +import ( + "bufio" + "fmt" + "io/fs" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" + "github.com/rossoctl/cortex/core/session/archive" +) + +// row is what a recorder saw of one event. +type row struct { + session string + phase pipeline.SessionPhase + noRecord bool +} + +// rows records every event's session, phase and NoRecord. +type rows struct { + mu sync.Mutex + got []row +} + +func (r *rows) Record(sid string, e *pipeline.SessionEvent) { + r.mu.Lock() + defer r.mu.Unlock() + r.got = append(r.got, row{sid, e.Phase, e.NoRecord}) +} + +func (r *rows) take() []row { + r.mu.Lock() + defer r.mu.Unlock() + out := r.got + r.got = nil + return out +} + +func noRecordProxy(t *testing.T) (proxy *httptest.Server, backendURL string, rec *rows, seen *[]string) { + t.Helper() + var got []string + var mu sync.Mutex + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + // "" when the header is not there at all, not just empty. + v := "" + if vs := r.Header.Values(model.RecordHeader); len(vs) > 0 { + v = strings.Join(vs, ",") + } + got = append(got, v) + mu.Unlock() + })) + t.Cleanup(backend.Close) + p, err := pipeline.New(nil) + if err != nil { + t.Fatal(err) + } + store := session.New(5*time.Minute, 100, 0) + t.Cleanup(store.Close) + rec = &rows{} + store.AddRecorder(rec) + srv := &Server{OutboundPipeline: pipeline.NewHolder(p), Sessions: store, Client: http.DefaultClient, + SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}} + proxy = httptest.NewServer(srv.Handler()) + t.Cleanup(proxy.Close) + return proxy, backend.URL, rec, &got +} + +// A request that names its own session and carries X-Cortex-Record: off is forwarded without the +// header, and both of its rows carry NoRecord. +func TestNoRecord_OwnSessionIsWithheldAndHeaderStripped(t *testing.T) { + proxy, backend, rec, seen := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{ + model.RecordHeader: {"off"}, + session.ClaudeCodeSessionHeader: {"11111111-2222-3333-4444-555555555555"}, + }) + if len(*seen) != 1 || (*seen)[0] != "" { + t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) + } + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if !r.noRecord || r.session != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("row %+v, want NoRecord under the request's own session", r) + } + } +} + +// A request that names no session is filed by fallback (the active session or the default +// bucket), so its flag must not withhold that bucket: it is recorded as usual. The header is +// still stripped. +func TestNoRecord_FallbackFiledRequestIsRecorded(t *testing.T) { + proxy, backend, rec, seen := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{model.RecordHeader: {"off"}}) + if len(*seen) != 1 || (*seen)[0] != "" { + t.Fatalf("upstream saw %s = %q, want it stripped", model.RecordHeader, *seen) + } + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if r.noRecord { + t.Fatalf("row %+v under a fallback bucket carries NoRecord", r) + } + } +} + +// The normal laptop path for an inference host is the TLS bridge: the CONNECT names nothing, and +// the bridged tunnel's open row is filed with the first decrypted request. When that request +// withholds its session, the open row shares its fate, so the archive holds no segment at all +// for the session (#1923), not even the CONNECT row with its project. +func TestNoRecord_BridgedRequestLeavesNoSegment(t *testing.T) { + root := t.TempDir() + arch, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + // Closed explicitly below before the scan; this covers an early exit. A second Close is a + // no-op. + t.Cleanup(func() { _ = arch.Close() }) + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + store.AddRecorder(arch) + proxyAddr, target, ca, done := bridgingProxyWith(t, store, func(s *Server) { + s.SessionIDHeaders = []string{session.ClaudeCodeSessionHeader} + }) + + raw, br, _ := sendConnect(t, proxyAddr, target) + tc := bridgedTLS(t, raw, br, target, ca) + req, _ := http.NewRequest(http.MethodGet, "https://"+target+"/v1/messages", nil) + req.Header.Set(session.ClaudeCodeSessionHeader, "11111111-2222-3333-4444-555555555555") + req.Header.Set(model.RecordHeader, "off") + if resp := bridgedRoundTrip(t, tc, req); resp.StatusCode != http.StatusOK { + t.Fatalf("bridged request status %d, want 200", resp.StatusCode) + } + _ = tc.Close() + waitDone(t, done) + + open, next := openThen(t, store, "11111111-2222-3333-4444-555555555555") + if !open.NoRecord || !next.NoRecord { + t.Fatalf("open row NoRecord=%v, request row NoRecord=%v, want both", open.NoRecord, next.NoRecord) + } + if err := arch.Close(); err != nil { + t.Fatal(err) + } + var segs []string + if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err // an unread subtree must fail the test, not pass it + } + if strings.HasSuffix(p, ".jsonl.zst") { + segs = append(segs, p) + } + return nil + }); err != nil { + t.Fatalf("scan %s: %v", root, err) + } + if len(segs) != 0 { + t.Fatalf("segments on disk %v, want none", segs) + } +} + +// sendConnectWith is sendConnect with extra request headers on the CONNECT. +func sendConnectWith(t *testing.T, proxyAddr, target string, h http.Header) (net.Conn, *bufio.Reader) { + t.Helper() + raw, err := net.Dial("tcp", proxyAddr) + if err != nil { + t.Fatalf("dial proxy: %v", err) + } + t.Cleanup(func() { _ = raw.Close() }) + _ = raw.SetDeadline(time.Now().Add(10 * time.Second)) + var b strings.Builder + fmt.Fprintf(&b, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n", target, target) + for k, vs := range h { + for _, v := range vs { + fmt.Fprintf(&b, "%s: %s\r\n", k, v) + } + } + b.WriteString("\r\n") + if _, err := raw.Write([]byte(b.String())); err != nil { + t.Fatalf("write CONNECT: %v", err) + } + br := bufio.NewReader(raw) + resp, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("read CONNECT response: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("CONNECT status = %d, want 200", resp.StatusCode) + } + return raw, br +} + +// An opaque (not bridged) CONNECT that names its own session and sends off withholds that +// session: its open row files under the named session at once, so it carries the flag. When the +// row files under another session (the active one, with no pin), it carries none, so no other +// session is stopped. +func TestNoRecord_OpaqueConnectWithholdsOnlyItsOwnSession(t *testing.T) { + const own = "11111111-2222-3333-4444-555555555555" + hdr := http.Header{session.ClaudeCodeSessionHeader: {own}, model.RecordHeader: {"off"}} + for _, tc := range []struct { + name string + affinity bool // pins the row to the session the CONNECT names + active string // a session active before the CONNECT, or "" + filed string // where the open row files + want bool + }{ + {"files under its own session", true, "", own, true}, + {"files under another session", false, "sess-A", "sess-A", false}, + } { + t.Run(tc.name, func(t *testing.T) { + store := session.New(5*time.Minute, 100, 0) + defer store.Close() + if tc.active != "" { + store.Append(tc.active, pipeline.SessionEvent{At: time.Now(), Direction: pipeline.Outbound, Phase: pipeline.SessionRequest}) + } + s := connectServer(t, store, nil) // no bridge: an opaque tunnel + s.SessionIDHeaders = []string{session.ClaudeCodeSessionHeader} + s.ClientAffinity = tc.affinity + proxyAddr, done := connectProxy(t, s) + raw, br := sendConnectWith(t, proxyAddr, pingPongOrigin(t), hdr) + pingPong(t, raw, br) + waitDone(t, done) + open, _ := onePair(t, store, tc.filed) + if open.NoRecord != tc.want { + t.Fatalf("open row under %s: NoRecord=%v, want %v", tc.filed, open.NoRecord, tc.want) + } + }) + } +} + +// The CONNECT's opt-out compares the filing session with the id pinned before the gate plugins +// ran, not with a re-read of pctx.Headers: a plugin that removes or rewrites the session-id header +// must not make an own-session opt-out record its open row. +func TestNoRecord_OpaqueConnectIgnoresARewrittenHeader(t *testing.T) { + const own = "11111111-2222-3333-4444-555555555555" + for _, rewritten := range []string{"", "99999999-2222-3333-4444-555555555555"} { + store := session.New(5*time.Minute, 100, 0) + h := http.Header{session.ClaudeCodeSessionHeader: {own}, model.RecordHeader: {"off"}} + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + pctx.SetAttribution("", "", session.ClaudeCodeSessionHeader) + pctx.SetNamedSession(own) + pctx.OutboundSessionID = own + // A gate plugin rewrites the header after the pin. + if rewritten == "" { + h.Del(session.ClaudeCodeSessionHeader) + } else { + h.Set(session.ClaudeCodeSessionHeader, rewritten) + } + s := &Server{Sessions: store, ClientAffinity: true, SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}} + s.appendTunnelOpen(pctx, pipeline.SessionEvent{At: time.Now(), Direction: pipeline.Outbound, Phase: pipeline.SessionRequest}) + v := store.View(own) + if v == nil || len(v.Events) != 1 || !v.Events[0].NoRecord { + t.Fatalf("header rewritten to %q: rows under %s %+v, want one flagged open row", rewritten, own, v) + } + store.Close() + } +} + +// A request without the header is unchanged. +func TestNoRecord_UnflaggedRequestIsRecorded(t *testing.T) { + proxy, backend, rec, _ := noRecordProxy(t) + postThrough(t, proxy, backend+"/v1/messages", `{}`, http.Header{ + session.ClaudeCodeSessionHeader: {"11111111-2222-3333-4444-555555555555"}, + }) + got := rec.take() + if len(got) != 2 { + t.Fatalf("rows %+v, want a request and a response", got) + } + for _, r := range got { + if r.noRecord { + t.Fatalf("row %+v carries NoRecord with no header", r) + } + } +} diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index 98e19d674..d3b2f00a5 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -748,6 +748,12 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // carries StatusCode). skip_hosts traffic never reaches here (the // !skipped guard above), so it stays suppressed by design. if hopped { + // The gateway records this request and its rows come back through /v1/ingest + // without the flag (#1923), so the store stamps them. Only the session the request + // named, as appendTunnelOpen checks: a hop never marks another one. + if pctx.WithholdsSession() && sid == pctx.NamedSession() { + s.Sessions.MarkNoRecord(sid) + } s.Hop.own(pctx, hopWire(pctx, r.Header, sid)) defer s.Hop.release(pctx) } else { diff --git a/core/listener/forwardproxy/transparent.go b/core/listener/forwardproxy/transparent.go index 416776a23..ad3a33fe8 100644 --- a/core/listener/forwardproxy/transparent.go +++ b/core/listener/forwardproxy/transparent.go @@ -336,7 +336,16 @@ func (s *Server) appendTunnelOpen(pctx *pipeline.Context, ev pipeline.SessionEve if s.ClientAffinity || s.processesOn() || pctx.SessionSource == pipeline.SessionSourceConnection { pinned = pctx.OutboundSessionID } - return s.Sessions.AppendBucket(attribution.FiledSession(pinned, s.Sessions.ActiveSession), ev) + sid := attribution.FiledSession(pinned, s.Sessions.ActiveSession) + // A CONNECT that names its own session and asks not to be recorded withholds that session + // (#1923), but only when its row files under the session it named: by these rules the row can + // go to the active session instead, and a flag there would stop another session's recording. + // The named id is the one pinned before the gate plugins ran (NamedSession), not a re-read + // of pctx.Headers, which a plugin can rewrite. + if pctx.WithholdsSession() && sid == pctx.NamedSession() { + ev.NoRecord = true + } + return s.Sessions.AppendBucket(sid, ev) } // recordTunnelClosed emits the SessionResponse row for a tunnel that ended, or that diff --git a/core/listener/forwardproxy/tunnel.go b/core/listener/forwardproxy/tunnel.go index 04e97cdac..9f693e254 100644 --- a/core/listener/forwardproxy/tunnel.go +++ b/core/listener/forwardproxy/tunnel.go @@ -887,6 +887,10 @@ func (t *tunnelLog) recordWithLocked(sid string, ev pipeline.SessionEvent, c *pi t.deferred, t.opened = false, true open := t.openEv open.At = ev.At + // The open row shares the request's fate (#1923): it goes into the same bucket, only to be + // folded into ev, and it comes first. Without this, a withheld session's archive would write + // the open row (with its project) before it ever saw the request's flag. + open.NoRecord = ev.NoRecord t.bucket = t.s.Sessions.AppendPairCapture(sid, open, ev, c) return true } diff --git a/core/listener/forwardproxy/websocket.go b/core/listener/forwardproxy/websocket.go index 5ef5087e1..881c937d9 100644 --- a/core/listener/forwardproxy/websocket.go +++ b/core/listener/forwardproxy/websocket.go @@ -838,6 +838,7 @@ func (ws *wsSocket) newTurnContext(body []byte) *pipeline.Context { // process on the same session, and without it no turn row names the header that named // the session — the rows #1535's fold counts procs and owner from. pctx.SetAttribution(hs.AttributedProc(), hs.AttributedClientProc(), hs.NamingHeader()) + pctx.SetNamedSession(hs.NamedSession()) pctx.SetProject(hs.Project()) pctx.SetParent(hs.ParentSession(), hs.ParentProject()) return pctx diff --git a/core/listener/internal/sessionevent/norecord_test.go b/core/listener/internal/sessionevent/norecord_test.go new file mode 100644 index 000000000..388bc83bb --- /dev/null +++ b/core/listener/internal/sessionevent/norecord_test.go @@ -0,0 +1,29 @@ +package sessionevent + +import ( + "net/http" + "testing" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/pipeline" +) + +// A request that names its own session and asks not to be recorded flags its request and +// response rows, but never a tunnel row: a bridged tunnel's open row can be filed later under the +// session its first decrypted request names, which need not be the one the CONNECT named (#1923). +func TestNoRecord_TunnelRowsCarryNoFlag(t *testing.T) { + h := http.Header{} + h.Set(model.RecordHeader, "off") + pctx := &pipeline.Context{Headers: h} + pctx.ResolveClient() + pctx.SetAttribution("", "", "X-Claude-Code-Session-Id") + if !pctx.WithholdsSession() { + t.Fatal("precondition: the request withholds its own session") + } + if !Request(pctx, pipeline.Outbound).NoRecord || !Response(pctx, pipeline.Outbound, 200, nil).NoRecord { + t.Fatal("request and response rows must carry NoRecord") + } + if TunnelOpen(pctx, "").NoRecord || TunnelClose(pctx, "", 200, nil, 0, 0).NoRecord { + t.Fatal("a tunnel row carries NoRecord") + } +} diff --git a/core/listener/internal/sessionevent/sessionevent.go b/core/listener/internal/sessionevent/sessionevent.go index c0a3b89ef..5cd3eba27 100644 --- a/core/listener/internal/sessionevent/sessionevent.go +++ b/core/listener/internal/sessionevent/sessionevent.go @@ -77,6 +77,7 @@ func Deny(pctx *pipeline.Context, action pipeline.Action, dir pipeline.Direction SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), ParentSession: pctx.ParentSession(), ParentProject: pctx.ParentProject(), TLS: TLS(pctx), @@ -99,6 +100,13 @@ func Deny(pctx *pipeline.Context, action pipeline.Action, dir pipeline.Direction // // dir is a parameter for the reason Deny gives. The caller still decides // whether to record and in which session bucket. +// +// NoRecord (#1923), here and on the Deny and Response rows, needs no check of that bucket +// against the named session, unlike a tunnel row: WithholdsSession requires a naming header, +// attribution resolves a request's valid session-id header before anything else +// (Attributor.resolveLocked), and the bucket a listener files these rows under is that answer +// (recordingSessionID, attributeForRecording). A trusted session pins attribution empty, so +// its rows carry no flag. func Request(pctx *pipeline.Context, dir pipeline.Direction) pipeline.SessionEvent { return pipeline.SessionEvent{ At: time.Now(), @@ -123,6 +131,7 @@ func Request(pctx *pipeline.Context, dir pipeline.Direction) pipeline.SessionEve SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), ParentSession: pctx.ParentSession(), ParentProject: pctx.ParentProject(), ClientProc: pctx.AttributedClientProc(), @@ -166,6 +175,7 @@ func Response(pctx *pipeline.Context, dir pipeline.Direction, status int, fail * SubagentKind: pctx.SubagentKind(), Proc: pctx.AttributedProc(), Project: pctx.Project(), + NoRecord: pctx.WithholdsSession(), ParentSession: pctx.ParentSession(), ParentProject: pctx.ParentProject(), TLS: TLS(pctx), @@ -183,6 +193,11 @@ func DenyResponse(pctx *pipeline.Context, action pipeline.Action, dir pipeline.D // TunnelOpen builds a tunnel's open row. reason says why the bytes stayed // opaque. The caller knows it, so it is a parameter and not derived here from // the host shape. +// +// No NoRecord on either tunnel row (#1923): a bridged tunnel's open row can be +// filed later under the session its first decrypted request names, which need +// not be the one the CONNECT named. The rows carry no content, and a withheld +// session's archive withholds them anyway. func TunnelOpen(pctx *pipeline.Context, reason pipeline.TunnelReason) pipeline.SessionEvent { return pipeline.SessionEvent{ At: time.Now(), diff --git a/core/model/norecord.go b/core/model/norecord.go new file mode 100644 index 000000000..21ccb42e9 --- /dev/null +++ b/core/model/norecord.go @@ -0,0 +1,38 @@ +package model + +import ( + "net/http" + "strings" +) + +// RecordHeader is the request header a client sends to keep its session out of the +// session archive (#1923). RecordOff is the one value that does so. +// +// For Cortex only: a listener reads it before any plugin runs and removes it from the +// request it forwards, so the upstream never sees it. Named like the other X-Cortex-* +// headers. It is not a session-id header and is never in session.id_headers. +// +// Claude Code sends it from ANTHROPIC_CUSTOM_HEADERS, e.g. +// ANTHROPIC_CUSTOM_HEADERS="X-Cortex-Record: off". +const RecordHeader = "X-Cortex-Record" + +// RecordOff is the value of RecordHeader that withholds a session from the archive. +const RecordOff = "off" + +// RecordsOff reports whether h asks not to be recorded: RecordHeader is present with the +// value RecordOff, compared without case and padding. Any other value, or none, records as +// today, so a typo can never hide a session by accident. Several values: any one "off" counts, +// because the header can only withhold. +func RecordsOff(h http.Header) bool { + if h == nil { + return false + } + for _, v := range h.Values(RecordHeader) { + for _, part := range strings.Split(v, ",") { + if strings.EqualFold(strings.TrimSpace(part), RecordOff) { + return true + } + } + } + return false +} diff --git a/core/model/norecord_test.go b/core/model/norecord_test.go new file mode 100644 index 000000000..84c2bd517 --- /dev/null +++ b/core/model/norecord_test.go @@ -0,0 +1,72 @@ +package model + +import ( + "net/http" + "strings" + "testing" +) + +func TestRecordsOff(t *testing.T) { + cases := []struct { + name string + vals []string + want bool + }{ + {"absent", nil, false}, + {"off", []string{"off"}, true}, + {"any case and padding", []string{" OFF "}, true}, + {"one of a list", []string{"on, off"}, true}, + {"one of several values", []string{"on", "off"}, true}, + {"on", []string{"on"}, false}, + {"a typo records", []string{"of"}, false}, + {"empty", []string{""}, false}, + {"false is not off", []string{"false"}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := http.Header{} + for _, v := range tc.vals { + h.Add(RecordHeader, v) + } + if got := RecordsOff(h); got != tc.want { + t.Fatalf("RecordsOff(%q) = %v, want %v", tc.vals, got, tc.want) + } + }) + } + if RecordsOff(nil) { + t.Fatal("RecordsOff(nil) = true") + } +} + +// A producer that sends noRecord with the wrong type loses the flag, never the row: it decodes as +// false, the safe direction for a field that can only withhold. +func TestSessionEvent_NoRecordIsLenient(t *testing.T) { + for _, v := range []string{`"yes"`, `1`, `null`, `{}`} { + var e SessionEvent + if err := e.UnmarshalJSON([]byte(`{"at":"2026-11-18T09:00:00Z","direction":"outbound","phase":"request","host":"h","noRecord":` + v + `}`)); err != nil { + t.Fatalf("noRecord=%s: %v, want the row decoded", v, err) + } + if e.NoRecord || e.Host != "h" { + t.Fatalf("noRecord=%s: NoRecord=%v Host=%q, want false and the row's other fields", v, e.NoRecord, e.Host) + } + } +} + +// NoRecord round-trips through the wire form and is absent from a recorded row. +func TestSessionEvent_NoRecordWire(t *testing.T) { + b, err := SessionEvent{NoRecord: true}.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var back SessionEvent + if err := back.UnmarshalJSON(b); err != nil { + t.Fatal(err) + } + if !back.NoRecord { + t.Fatalf("NoRecord lost in %s", b) + } + plain, _ := SessionEvent{}.MarshalJSON() + if got := string(plain); strings.Contains(got, "noRecord") { + t.Fatalf("a recorded row carries the key: %s", got) + } +} diff --git a/core/model/session.go b/core/model/session.go index 2d88d7168..0dca66b7e 100644 --- a/core/model/session.go +++ b/core/model/session.go @@ -447,6 +447,16 @@ type SessionEvent struct { // Use ImportedSource to make the value and Imported to test it. Source string + // NoRecord is true when the request withholds its own session from the archive: it + // carried RecordHeader with the value RecordOff and named its session with a session-id + // header (#1923; Context.WithholdsSession, pinned before any plugin runs). The proxy + // still forwards the request and runs its plugins. The archive keeps none of the + // session's events from then on; what it held before the opt-out stays. + // + // Client-asserted, with the same caveat as Client above. It can stop a session's + // recording, never put one there or erase what is recorded. + NoRecord bool + // SubagentKind is the kind of a Bob child task, from Context.SubagentKind: // SubagentKindSubtask or SubagentKindSubagent. Set only on a Bob child's rows, and // empty everywhere else, Claude Code rows included (read Inference.AgentRole there). @@ -722,6 +732,9 @@ type sessionEventWire struct { ParentSubagent string `json:"parentSubagent,omitempty"` // omitempty: absent on every live event. Source LenientString `json:"source,omitempty"` + // omitempty: absent on every recorded row, which is every row the archive keeps. Lenient, as + // Source is: a producer (cortex-gateway, say) that sends a wrong type must not cost the row. + NoRecord LenientBool `json:"noRecord,omitempty"` // omitempty: absent on every row but a Bob child task's. SubagentKind string `json:"subagentKind,omitempty"` // omitempty: absent wherever the listener could not name the client's process, and on @@ -770,6 +783,7 @@ func (e SessionEvent) MarshalJSON() ([]byte, error) { Subagent: e.Subagent, ParentSubagent: e.ParentSubagent, Source: LenientString(e.Source), + NoRecord: LenientBool(e.NoRecord), SubagentKind: e.SubagentKind, Proc: e.Proc, ClientProc: e.ClientProc, @@ -820,6 +834,7 @@ func (e *SessionEvent) UnmarshalJSON(data []byte) error { Subagent: w.Subagent, ParentSubagent: w.ParentSubagent, Source: string(w.Source), + NoRecord: bool(w.NoRecord), SubagentKind: w.SubagentKind, Proc: w.Proc, ClientProc: w.ClientProc, @@ -1077,6 +1092,10 @@ type ArchiveUsage struct { DroppedCaptures uint64 `json:"droppedCaptures,omitempty"` // Paused is true after the disk filled, until the archive's next retention pass. Paused bool `json:"paused,omitempty"` + // FailedClosed is true once more sessions asked not to be recorded, while the archive's + // writer was behind, than it keeps track of (#1923): until a restart it records no + // conversation. + FailedClosed bool `json:"failedClosed,omitempty"` } // ImportedSourcePrefix starts the Source of every event that an import wrote. @@ -1098,6 +1117,19 @@ func (s *LenientString) UnmarshalJSON(b []byte) error { return nil } +// LenientBool decodes JSON true as true and any other JSON value as false, for the reason +// LenientString gives. Used where false is the safe default: noRecord can only withhold. +type LenientBool bool + +func (b *LenientBool) UnmarshalJSON(raw []byte) error { + var v bool + if json.Unmarshal(raw, &v) != nil { + v = false + } + *b = LenientBool(v) + return nil +} + // Imported tells if an import wrote e. A live event gives false. func (e *SessionEvent) Imported() bool { return strings.HasPrefix(e.Source, ImportedSourcePrefix) diff --git a/core/pipeline/attributed.go b/core/pipeline/attributed.go index ab381644d..07fbadfbe 100644 --- a/core/pipeline/attributed.go +++ b/core/pipeline/attributed.go @@ -27,6 +27,20 @@ func (c *Context) AttributedProc() string { return c.attributedProc } // none did. Recorded as SessionEvent.SessionHeader on the request's own row. func (c *Context) NamingHeader() string { return c.namingHeader } +// SetNamedSession pins the session id the request named with its session-id header (the one +// NamingHeader names), "" when none did. FIRST CALL WINS, with its own flag, as SetProject does: +// the listener sets it with SetAttribution, before any plugin runs, so a plugin that rewrites +// the session-id header cannot change which session the request named (#1923). +func (c *Context) SetNamedSession(id string) { + if c.namedSet { + return + } + c.namedSet, c.namedSession = true, id +} + +// NamedSession is the pinned session id the request named, "" when none was pinned. +func (c *Context) NamedSession() string { return c.namedSession } + // SetProject pins the project label (project.Label of the agent process's repo root) for // every row recorded from this request. FIRST CALL WINS, as SetAttribution does, with its // own flag: a path that pins "" keeps "", and no later call can write a label. diff --git a/core/pipeline/context.go b/core/pipeline/context.go index d8fe1a351..4f3964b35 100644 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -171,11 +171,22 @@ type Context struct { // bobChild is true when subagent came from Bob's X-Task-Id, not Claude Code's header. bobChild bool + // noRecord and noRecordParsed memoize NoRecord's answer, on the same terms as client + // and clientParsed: filled by ResolveClient before any plugin runs. + noRecord bool + noRecordParsed bool + // recordHeaderSeen is true when the client sent model.RecordHeader with any value, so a + // listener that forwards by explicit mutation (ext_proc) knows to remove it. + recordHeaderSeen bool + // attributedProc, namingHeader and attributed hold SetAttribution's pin. attributedProc string clientProc string namingHeader string attributed bool + // namedSession and namedSet hold SetNamedSession's pin. + namedSession string + namedSet bool // project and projectSet hold SetProject's pin. project string @@ -557,10 +568,14 @@ func (c *Context) ClientInfo() *EventClient { // X-Claude-Code-Agent-Id a plugin rewrote must not re-file a row under another subagent, // and recorders reading the memo must not be the ones filling it. // +// It pins NoRecord too, which also removes model.RecordHeader from Headers, so no plugin +// sees it and the forwarded request does not carry it. +// // Idempotent: the second call is the memo's own no-op. func (c *Context) ResolveClient() { _ = c.ClientInfo() _ = c.Subagent() + _ = c.NoRecord() } // SetSessionFacts is for listeners: pass what session.Store.ViewFacts returned. A zero diff --git a/core/pipeline/norecord.go b/core/pipeline/norecord.go new file mode 100644 index 000000000..546af2938 --- /dev/null +++ b/core/pipeline/norecord.go @@ -0,0 +1,54 @@ +package pipeline + +import "github.com/rossoctl/cortex/core/model" + +// NoRecord reports whether the request asked not to be recorded (#1923): it carried +// model.RecordHeader set to model.RecordOff. With an own session id (WithholdsSession), it is +// recorded as SessionEvent.NoRecord on the request's rows, and the archive keeps none of that +// session's events from then on; what it held before stays. +// +// The first call also deletes model.RecordHeader from Headers. The header is for Cortex +// only: plugins never see it, and the listeners that send the request on from Headers +// (httpx.SyncRequestHeaders) do not forward it. ResolveClient makes that first call at +// construction, before any plugin runs, so a plugin cannot set or clear it either. +// +// Memoized and pinned by ResolveClient on exactly the terms ClientInfo is; see there. +// +// CLIENT-ASSERTED AND UNAUTHENTICATED, like Subagent. It can only withhold the sender's own +// session from the archive. +func (c *Context) NoRecord() bool { + if c.noRecordParsed { + return c.noRecord + } + c.noRecordParsed = true + if c.Headers == nil { + return false + } + c.noRecord = model.RecordsOff(c.Headers) + c.recordHeaderSeen = len(c.Headers.Values(model.RecordHeader)) > 0 + c.Headers.Del(model.RecordHeader) + return c.noRecord +} + +// RecordHeaderSeen reports whether the client sent model.RecordHeader, with any value. NoRecord +// removed it from Headers, which is enough for a listener that forwards from Headers. One that +// forwards the wire headers and emits changes (ext_proc) must remove it explicitly. +func (c *Context) RecordHeaderSeen() bool { + _ = c.NoRecord() + return c.recordHeaderSeen +} + +// WithholdsSession reports whether this request's session is to be withheld from the archive: +// it asked not to be recorded (NoRecord) AND it names its own session with a session-id header +// (NamingHeader, pinned by SetAttribution). This is what listeners record as +// SessionEvent.NoRecord. +// +// THE OWN-ID RULE IS THE SAFETY. A request that names no session is filed by fallback, to the +// active session or the shared default bucket (attribution.FiledSession), so honouring its flag +// would stop recording a session that is not the sender's. Such a request is recorded as if it +// had not asked; its header is still stripped. A request whose header names another session can +// still stop that session's recording from then on, but never erase what is already archived: +// the session-id header is client-asserted, with session.IDFromHeaders' TRUST caveat. +func (c *Context) WithholdsSession() bool { + return c.NoRecord() && c.namingHeader != "" +} diff --git a/core/pipeline/norecord_test.go b/core/pipeline/norecord_test.go new file mode 100644 index 000000000..66bf7eb50 --- /dev/null +++ b/core/pipeline/norecord_test.go @@ -0,0 +1,42 @@ +package pipeline + +import ( + "net/http" + "testing" + + "github.com/rossoctl/cortex/core/model" +) + +// ResolveClient pins NoRecord from the headers as the client sent them, and removes the header +// so no plugin sees it and the forwarded request does not carry it. +func TestNoRecord_PinnedAndStripped(t *testing.T) { + h := http.Header{} + h.Set(model.RecordHeader, "off") + h.Set("User-Agent", "claude-cli/2.1.266 (external, cli)") + c := &Context{Headers: h} + c.ResolveClient() + // Before calling NoRecord here: its first call would remove the header itself, so this + // checks that ResolveClient made it. + if _, ok := c.Headers[http.CanonicalHeaderKey(model.RecordHeader)]; ok { + t.Fatal("the header is still in Headers after ResolveClient") + } + if !c.NoRecord() { + t.Fatal("NoRecord = false for X-Cortex-Record: off") + } + // A plugin that adds it back later changes nothing: the answer was pinned. + c.Headers.Set(model.RecordHeader, "on") + if !c.NoRecord() { + t.Fatal("a later header change unpinned NoRecord") + } +} + +func TestNoRecord_AbsentRecords(t *testing.T) { + c := &Context{Headers: http.Header{"User-Agent": {"curl/8"}}} + c.ResolveClient() + if c.NoRecord() { + t.Fatal("NoRecord = true with no header") + } + if (&Context{}).NoRecord() { + t.Fatal("NoRecord = true with nil Headers") + } +} diff --git a/core/pipeline/session_golden_test.go b/core/pipeline/session_golden_test.go index 94ff03155..fb8916cc4 100644 --- a/core/pipeline/session_golden_test.go +++ b/core/pipeline/session_golden_test.go @@ -86,6 +86,7 @@ func goldenEvent() SessionEvent { ParentSession: "s-parent", ParentProject: "~/src/parent", Source: "imported:claude-code", + NoRecord: true, SessionHeader: "x-session-id", DialAddr: "203.0.113.7:443", } diff --git a/core/pipeline/testdata/session-event.golden.json b/core/pipeline/testdata/session-event.golden.json index 4655d997c..b76360acd 100644 --- a/core/pipeline/testdata/session-event.golden.json +++ b/core/pipeline/testdata/session-event.golden.json @@ -197,6 +197,7 @@ "subagent": "a1a2e36cb5946f31c", "parentSubagent": "a0974b0b8591dc76b", "source": "imported:claude-code", + "noRecord": true, "subagentKind": "subtask", "proc": "/bin/claude", "clientProc": "/bin/zsh", diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 15ea3130f..e5da5024c 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -94,6 +94,10 @@ type Stats struct { DroppedCaptures uint64 // Paused is true after ENOSPC, until the next retention pass. Paused bool + // FailedClosed is true once more sessions asked not to be recorded, while the queue was + // full, than the archive keeps track of (#1923). Until a restart it records no event of a + // conversation. + FailedClosed bool } // Archive is the session archive: a session.Recorder that writes every event the store appends @@ -131,7 +135,21 @@ type Archive struct { // recent clear, for AwaitClear; see clear.go. mu sync.Mutex lastSeq map[string]uint64 - pending *pendingClear + // stopped is set by the writer once it has drained for Close: sendLocked and optOutLocked + // then refuse rather than queue what nothing will read. Under mu. + stopped bool + // sent is the position of the last op sendLocked queued, and marks the position of the last + // queued op that can move or remove an opt-out's mark (a rename, a delete, a clear). optOuts + // holds, per session id, the positions after which its opt-out applies, for opt-outs that + // found no room in the queue; see norecord.go. optOutN is len(optOuts), for the writer's fast + // path. All under mu. + sent, marks uint64 + optOuts map[string][]uint64 + optOutN atomic.Int64 + // failClosed is set when optOuts reached its bound: from then on until a restart, no event + // of a conversation is recorded (#1923). + failClosed atomic.Bool + pending *pendingClear // deletes is the most recent delete per session id, for AwaitDelete; see delete.go. nil // until the first. deletes map[string]*pendingClear @@ -159,7 +177,8 @@ type Archive struct { idxMu sync.RWMutex index map[string]*indexEntry - // Writer goroutine only. + // Writer goroutine only. at is the position of the last positioned op the writer took. + at uint64 sessions map[string]*sessionState lastMeta time.Time lastPrune time.Time @@ -203,6 +222,8 @@ const ( opClear opDelete opFunc + // opNoRecord marks a session as withheld (#1923), queued with its first flagged event. + opNoRecord ) // op is one unit of work for the writer goroutine. Events and renames share one FIFO, so a @@ -213,6 +234,7 @@ type op struct { ev model.SessionEvent cap *model.RequestCapture // ev's request capture, nil for most events after uint64 // where a store entry began: on its first queued event, and on its rename + n uint64 // its position, from sendLocked; 0 for an op sent without one clear *pendingClear f func() done chan struct{} @@ -335,19 +357,37 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if a.closed.Load() { return } + // A flagged event of a conversation is an opt-out (#1923), never lost: when the queue has no + // room for it, the opt-out is kept at its place in the queue instead (optOutLocked), and the + // event itself, withheld anyway, goes. Nothing here waits. + optOut := e.NoRecord && session.IsConversation(sessionID) a.mu.Lock() - defer a.mu.Unlock() if e.Seq > a.lastSeq[sessionID] { a.lastSeq[sessionID] = e.Seq } - // Every send happens under the store's write lock, so this check and the send below cannot - // race another producer; the writer only ever makes room. - if len(a.ops) >= cap(a.ops)-reserve { + // Failed closed, no conversation event is recorded. An opt-out still goes on to the + // admission below, as Withhold's does, so its mark is written when there is room. + if a.failClosed.Load() && session.IsConversation(sessionID) && !optOut { + a.mu.Unlock() a.dropped.Add(1) return } + // The reserve is for renames, deletes and clears. Every send happens under the store's + // write lock, so this check and the send below cannot race another producer; the writer + // only ever makes room. + if len(a.ops) >= cap(a.ops)-reserve { + if optOut { + a.optOutLocked(sessionID) + } else { + a.dropped.Add(1) + } + a.mu.Unlock() + return + } o := op{kind: opEvent, id: sessionID, ev: *e} - if c != nil { + // A request that asked not to be recorded queues no bodies: the writer withholds its + // session anyway (withhold), so the copy would only take capture budget from others. + if c != nil && !e.NoRecord { if n := captureSize(c); a.captureBytes.Load()+n <= a.captureBudget { a.captureBytes.Add(n) o.cap = c @@ -359,15 +399,39 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode if starting && !st.queued { o.after = st.after } - select { - case a.ops <- o: + if a.sendLocked(o) { if starting && !st.queued { st.queued = true a.started[sessionID] = st } - default: + a.mu.Unlock() + return + } + if optOut { + a.optOutLocked(sessionID) + } else { a.dropped.Add(1) - a.releaseCapture(o.cap) + } + a.mu.Unlock() + a.releaseCapture(o.cap) +} + +// sendLocked queues o without blocking and gives it its position in the queue, which is how the +// writer applies a kept opt-out at its place (optOutLocked). It reports whether o was queued: not +// when the queue is full, and not once the writer has stopped for Close. The caller holds a.mu. +// Every producer that holds the store's write lock sends through here, so positions follow the +// queue's order. +func (a *Archive) sendLocked(o op) bool { + if a.stopped { + return false + } + o.n = a.sent + 1 + select { + case a.ops <- o: + a.sent = o.n + return true + default: + return false } } @@ -418,10 +482,12 @@ func (a *Archive) Rekeyed(oldID, newID string) { } a.lastSeq[newID] = max(a.lastSeq[newID], n) } + ok := a.sendLocked(op{kind: opRename, id: oldID, to: newID, after: st.after}) + if ok { + a.marks = a.sent // a rename moves a mark: see optOutLocked + } a.mu.Unlock() - select { - case a.ops <- op{kind: opRename, id: oldID, to: newID, after: st.after}: - default: + if !ok { a.droppedRenames.Add(1) // Session ids are attribution keys the operator needs to find the // history this rename left behind. @@ -449,6 +515,7 @@ func (a *Archive) Stats() Stats { DroppedRenames: a.droppedRenames.Load(), DroppedCaptures: a.droppedCaptures.Load(), Paused: a.paused.Load(), + FailedClosed: a.failClosed.Load(), } } @@ -486,26 +553,45 @@ func (a *Archive) run() { case o := <-a.ops: a.take(o) case <-t.C: + a.applyOptOuts(false) a.tick() case <-a.stop: - for drained := false; !drained; { - select { - case o := <-a.ops: - a.take(o) - default: - drained = true - } - } + a.drain() + // From here nothing more is queued or kept (sendLocked and optOutLocked check + // stopped under a.mu), so one more drain takes every op that was, and every kept + // opt-out is due (#1923). shutdown then writes their marks. + a.mu.Lock() + a.stopped = true + a.mu.Unlock() + a.drain() + a.applyOptOuts(true) a.closeErr = a.shutdown() return } } } +// drain takes every op queued now, without waiting for more. +func (a *Archive) drain() { + for { + select { + case o := <-a.ops: + a.take(o) + default: + return + } + } +} + func (a *Archive) take(o op) { if a.gate != nil { <-a.gate } + // A kept opt-out applies before the first op queued after it (#1923). + a.applyOptOuts(false) + if o.n > 0 { + a.at = o.n + } switch o.kind { case opEvent: if o.after > 0 { @@ -525,6 +611,10 @@ func (a *Archive) take(o op) { case opFunc: o.f() close(o.done) + case opNoRecord: + // Also while paused: the mark is one small session.json, and it is what keeps the + // session out after a restart. + a.withhold(o.id, a.sessions[o.id], a.now()) } } @@ -549,12 +639,18 @@ func (a *Archive) begin(id string, after uint64) { // write appends one event, with its request's capture c when it has one, to its session's open // segment, opening one if needed. func (a *Archive) write(id string, e *model.SessionEvent, c *model.RequestCapture) { + now := a.now() + s := a.sessions[id] + // Before the pause check: the marker is one small session.json, and it is the opt-out's only + // state, so it is written while paused too (#1923). + if a.withheld(id, s, e) { + a.withhold(id, s, now) + return + } if a.paused.Load() { a.dropped.Add(1) return } - now := a.now() - s := a.sessions[id] fresh := s == nil if fresh { s = &sessionState{ @@ -673,22 +769,42 @@ func (a *Archive) rename(oldID, newID string, after uint64) { if s == nil { return } + // A withheld session stays withheld under its new id (#1923): the store's entry moves to newID + // whatever happens to the files here. On the paths that move the fold, renameEntry and the + // move below keep its mark; where the move is refused, the new id is marked in place. All on + // the writer, in FIFO order with any later delete or clear of newID. + marked := s.meta.Summary.NoRecord() newDir := filepath.Join(a.data, dirName(newID)) if _, taken := a.sessions[newID]; taken { slog.Warn("session archive: rename target already has history; keeping it under the old id", "from", oldID, "to", newID) + if marked { + a.withhold(newID, a.sessions[newID], a.now()) + } return } if _, err := os.Stat(newDir); err == nil { slog.Warn("session archive: rename target directory exists; keeping history under the old id", "from", oldID, "to", newID) + // newID's next event would build its state on that directory anyway (write), so mark it + // there now, as for a taken target: unflagged events under newID must not be recorded. + if marked { + a.withhold(newID, nil, a.now()) + } return } if after > 0 && a.renameEntry(s, oldID, newID, newDir, after) { + // The history stayed under oldID, whole or in part. If no part moved, newID has no state + // yet, and gets the mark as a marker. + if marked && a.sessions[newID] == nil { + a.withhold(newID, nil, a.now()) + } return } a.closeWriterBeforeRename(s) // a no-op except on Windows: see rename_windows.go s.meta.ID = newID + // Dirty before the write: a failure is retried, and Close reports a withheld session's (#1923). + s.metaDirty = true a.writeMeta(s) if err := os.Rename(s.dir, newDir); err != nil { slog.Warn("session archive: could not move a renamed session's directory; it will be moved at the next start", @@ -719,10 +835,15 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte } startAfter, before := s.startAfter, s.before s.startAfter, s.before = 0, nil + // Both folds below are rebuilt from segments, which do not hold the mark (#1923). + noRec := s.meta.Summary.NoRecord() if entry == len(segs) { if startAfter == after { a.closeWriter(s) s.meta.Summary, _ = foldSegments(s.dir, s.meta.Segments, oldID) + if noRec { + s.meta.Summary.MarkNoRecord() + } } else { // The fold also holds the history's events, which stay under oldID. The new id must // not take the history's hosts as its own. @@ -758,6 +879,13 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte meta: &sessionMeta{ID: newID, CreatedAt: createdAt, UpdatedAt: latestWrite(move), Summary: since, Segments: move}, lastWrite: s.lastWrite, } + renewed := s.meta.UpdatedAt // a withheld session's last event, which wrote no segment + if noRec { + ns.meta.Summary.MarkNoRecord() + // The marker lives retainDays after the session's last event (keepMarker), not after its + // last segment. + ns.meta.UpdatedAt = later(ns.meta.UpdatedAt, renewed) + } // session.json first: a crash before a segment moves leaves it listed where it is not, which // reconcile drops, rather than a directory loadMetas cannot read. if err := writeMeta(newDir, ns.meta); err != nil { @@ -781,6 +909,12 @@ func (a *Archive) renameEntry(s *sessionState, oldID, newID, newDir string, afte } // A CLONE: before was published, and the old id's live fold is Added to from here on. s.meta.Segments, s.meta.Summary, s.meta.UpdatedAt = keep, before.Clone(), latestWrite(keep) + if noRec { + // before predates the opt-out, so it does not hold the mark; the old id keeps it (#1923), + // and the renewal time with it. + s.meta.Summary.MarkNoRecord() + s.meta.UpdatedAt = later(s.meta.UpdatedAt, renewed) + } a.writeMeta(s) if ns.metaDirty { a.writeMeta(ns) @@ -900,9 +1034,14 @@ func (a *Archive) prune(now time.Time) { } var gone []*sessionState for _, s := range a.sessions { - if s.w == nil && len(s.meta.Segments) == 0 { - gone = append(gone, s) + if s.w != nil || len(s.meta.Segments) > 0 { + continue } + if keepMarker(s, now, a.retainDays) { + a.markerOnly(s) // its history expired: keep the mark, not the history's fold + continue + } + gone = append(gone, s) } a.forget(gone) for _, s := range gone { @@ -976,6 +1115,11 @@ func (a *Archive) shutdown() error { for _, s := range a.sessions { if s.metaDirty { a.writeMeta(s) + // A withheld session's marker is its opt-out: report one that could not be written, + // rather than restart without it (#1923). No session id: it comes from a client header. + if s.metaDirty && s.meta.Summary.NoRecord() { + errs = append(errs, errors.New("a withheld session's marker could not be written")) + } } } return errors.Join(errs...) diff --git a/core/session/archive/clear.go b/core/session/archive/clear.go index e0c24b1c3..77c08387b 100644 --- a/core/session/archive/clear.go +++ b/core/session/archive/clear.go @@ -53,16 +53,19 @@ func (a *Archive) Cleared() { a.mu.Lock() p.seqs = maps.Clone(a.lastSeq) a.pending = p - a.mu.Unlock() if a.closed.Load() { + a.mu.Unlock() p.finish(ClearResult{}, errors.New("the session archive is closed")) return } // Into the reserve, like a rename: Record stops short of it, so only renames and clears can // fill it. - select { - case a.ops <- op{kind: opClear, clear: p}: - default: + ok := a.sendLocked(op{kind: opClear, clear: p}) + if ok { + a.marks = a.sent // a clear removes every mark: see optOutLocked + } + a.mu.Unlock() + if !ok { slog.Error("session archive: a clear was lost to a full queue; history on disk is unchanged") p.finish(ClearResult{}, errors.New("the session archive's queue is full; nothing on disk was cleared")) } diff --git a/core/session/archive/delete.go b/core/session/archive/delete.go index ed53b0a6f..89d375aa6 100644 --- a/core/session/archive/delete.go +++ b/core/session/archive/delete.go @@ -26,15 +26,18 @@ func (a *Archive) Deleted(sessionID string) { a.deletes = map[string]*pendingClear{} } a.deletes[sessionID] = p - a.mu.Unlock() if a.closed.Load() { + a.mu.Unlock() p.finish(ClearResult{}, errors.New("the session archive is closed")) return } // Into the reserve, like a rename and a clear: Record stops short of it. - select { - case a.ops <- op{kind: opDelete, id: sessionID, clear: p}: - default: + ok := a.sendLocked(op{kind: opDelete, id: sessionID, clear: p}) + if ok { + a.marks = a.sent // a delete removes a mark: see optOutLocked + } + a.mu.Unlock() + if !ok { slog.Error("session archive: a delete was lost to a full queue; the session's history on disk is unchanged", "session", sessionID) p.finish(ClearResult{}, errors.New("the session archive's queue is full; nothing on disk was erased")) diff --git a/core/session/archive/import.go b/core/session/archive/import.go index 2269f8173..1fc4ba8aa 100644 --- a/core/session/archive/import.go +++ b/core/session/archive/import.go @@ -66,6 +66,14 @@ func (a *Archive) writeSession(id string, events []model.SessionEvent, replace b a.mu.Lock() old := a.lastSeq[id] switch { + case s != nil && s.meta.Summary.NoRecord(), a.failClosed.Load(): + // The session asked not to be recorded (#1923), or the archive failed closed and no + // longer knows which sessions asked. A kept opt-out needs no check here: this op is + // queued after its place, so the writer has applied it. The marker is the user's own + // choice, so no import adds to it or replaces it, also one whose events were imported + // first. + a.mu.Unlock() + return ErrLive case s == nil && old > 0: a.mu.Unlock() return ErrLive diff --git a/core/session/archive/norecord.go b/core/session/archive/norecord.go new file mode 100644 index 000000000..e03e6dc9d --- /dev/null +++ b/core/session/archive/norecord.go @@ -0,0 +1,214 @@ +package archive + +import ( + "log/slog" + "path/filepath" + "time" + + "github.com/rossoctl/cortex/core/model" + "github.com/rossoctl/cortex/core/session" +) + +// THE WRITER OWNS THE OPT-OUT (#1923). A session is withheld when its fold is marked noRecord, +// and only the writer goroutine marks a fold, in the same FIFO order as renames, deletes, clears +// and prune, so they cannot reorder against it. +// +// AN OPT-OUT IS NEVER LOST, AND NO PRODUCER WAITS FOR IT. A flagged event of a conversation, or a +// Withhold, is queued like any op when there is room below the reserve. When there is not (the +// writer is a full queue behind, e.g. on a stalled disk), it is kept instead in optOuts, tagged +// with the position of the last op queued before it, and the writer applies it at that place: after +// every op queued before it, before every op queued after it (applyOptOuts). So it lands where it +// would have been in the queue: after a DELETE, CLEAR or rename queued earlier, and before the +// session's later rows, which the mark then withholds. The producer, which holds the session +// store's write lock, never waits, so a stalled writer never stalls other sessions' traffic. +// +// optOuts holds one entry per session id, removed when the writer applies it. It grows only while +// the queue is full. Its ids are bounded at as many as the queue holds ops (a.depth). Each id's +// list of positions grows by at most one per rename, delete or clear queued after its last one, +// so in the worst case the list holds about depth × depth positions. While any entry is kept, the +// writer scans the list under a.mu before each op (applyOptOuts). +// +// Past the bound on ids the archive FAILS CLOSED: until a restart it records no event of a +// conversation, because it no longer knows which ones asked not to be recorded. The opt-outs that +// did not fit are not kept anywhere durable, so after a restart such a session's unflagged rows are +// recorded again until it sends another flagged request. On a single-user laptop that many +// opted-out sessions during one stall means forged session ids. + +// withhold keeps session id out of the archive from now on. It runs on the writer goroutine for an +// event that asked not to be recorded (SessionEvent.NoRecord, which listeners set only for a +// request that names its own session), for an opNoRecord (Withhold), for a kept opt-out +// (applyOptOuts), and for every later event of a session already withheld, flagged or not: a +// session's header-less calls (telemetry, say) are filed to it by process or client affinity and +// carry no flag. +// +// FROM NOW ON, NEVER BACKWARDS. What the archive already holds for id stays: it was recorded +// before the opt-out, and the session-id header that names id is client-asserted, so a request +// must not be able to erase another session's history by naming it. The open segment is closed, +// so nothing more is written to it, and the session is marked. Erasing is the user's explicit +// DELETE /v1/sessions/{id}. +// +// For an id the archive does not hold, withhold writes a content-free marker: a session.json with +// no segments whose summary says noRecord and nothing else. The event itself is never folded, +// because the fold reads content (the title comes from the prompt). The mark is what keeps the +// opt-out after a restart, and what makes an import refuse id (WriteSession returns ErrLive). +// +// metaDirty is set before the write, so a failed write is retried at the next metaEvery and at +// shutdown, and Close reports a marker it could not write. +func (a *Archive) withhold(id string, s *sessionState, now time.Time) { + if s != nil && s.meta.Summary.NoRecord() { + // Already marked. A marker's age is measured from the session's last event + // (keepMarker), so it is renewed, and written with the next metaEvery. + s.meta.UpdatedAt, s.metaDirty = now, true + return + } + if s == nil { + fold := session.NewSummaryFold() + s = &sessionState{ + dir: filepath.Join(a.data, dirName(id)), + meta: &sessionMeta{ID: id, CreatedAt: now, Summary: fold}, + } + a.sessions[id] = s + } else { + a.closeWriter(s) + } + s.meta.Summary.MarkNoRecord() + s.meta.UpdatedAt, s.lastWrite, s.metaDirty = now, now, true + a.writeMeta(s) + a.publish(s) +} + +// Withhold implements session.Withholder: the store tells the archive that sessionID is withheld +// from its next event on, for rows that arrive unflagged (praxis mode's /v1/ingest). Its opt-out +// is queued, or kept at its place when the queue is full, so it precedes those rows, because the +// store calls this before the request is hopped. It runs under the store's write lock and never +// waits. +// +// Every time, also for a session the writer has marked already: a DELETE, CLEAR or prune still +// queued may be about to remove that mark. Withholding a marked session is cheap. +func (a *Archive) Withhold(sessionID string) { + if a.closed.Load() || !session.IsConversation(sessionID) { + return + } + a.mu.Lock() + defer a.mu.Unlock() + if len(a.ops) >= cap(a.ops)-reserve || !a.sendLocked(op{kind: opNoRecord, id: sessionID}) { + a.optOutLocked(sessionID) + } +} + +// optOutLocked keeps the opt-out of id, which found no room in the queue, at its place: after the +// last op queued so far (a.sent). The caller holds a.mu. +// +// One entry per id. A later opt-out of the same id adds a position only when a rename, delete or +// clear was queued after the entry's last one (a.marks): that op may move or remove the mark, so +// the opt-out must apply again after it, as it would from the queue. Otherwise the mark the entry +// already applies covers it. So an entry holds at most one position per such op still queued. +func (a *Archive) optOutLocked(id string) { + if a.stopped { + a.dropped.Add(1) + slog.Warn("session archive: closed before a request that asked not to be recorded reached the writer; nothing of it was written") + return + } + tags, ok := a.optOuts[id] + switch { + case !ok: + if len(a.optOuts) >= a.depth { + if !a.failClosed.Swap(true) { + slog.Error("session archive: more sessions asked not to be recorded, while the writer was behind, than it keeps track of; " + + "recording no conversation until a restart. The opt-outs that did not fit are not remembered across that restart: " + + "those sessions' unflagged rows are recorded again until they next ask") + } + a.dropped.Add(1) + return + } + if a.optOuts == nil { + a.optOuts = map[string][]uint64{} + } + a.optOuts[id] = []uint64{a.sent} + a.optOutN.Add(1) + case tags[len(tags)-1] < a.marks: + a.optOuts[id] = append(tags, a.sent) + } +} + +// applyOptOuts withholds every kept opt-out that is due: its position is at or before a.at, the +// last positioned op the writer took, or, with all, every one (Close, once nothing more is queued). +// It runs on the writer goroutine before each op, so an opt-out applies before the first op queued +// after it, and on each tick, for one with no op behind it. +func (a *Archive) applyOptOuts(all bool) { + if a.optOutN.Load() == 0 { + return + } + var due []string + a.mu.Lock() + for id, tags := range a.optOuts { + i := 0 + for i < len(tags) && (all || tags[i] <= a.at) { + i++ + } + if i == 0 { + continue + } + due = append(due, id) + if i == len(tags) { + delete(a.optOuts, id) + a.optOutN.Add(-1) + } else { + a.optOuts[id] = tags[i:] + } + } + a.mu.Unlock() + now := a.now() + for _, id := range due { + a.withhold(id, a.sessions[id], now) + } +} + +// withheld reports whether e, an event of id, is to be withheld: it asks it, or the session is +// marked. Never for a shared bucket: the store already clears the flag there, and the archive +// does not trust that alone (session.IsConversation). +func (a *Archive) withheld(id string, s *sessionState, e *model.SessionEvent) bool { + if !session.IsConversation(id) { + return false + } + return e.NoRecord || (s != nil && s.meta.Summary.NoRecord()) +} + +// later returns the later of a and b. +func later(a, b time.Time) time.Time { + if b.After(a) { + return b + } + return a +} + +// markerOnly reduces s, a withheld session that has nothing left on disk, to its marker: the +// fold drops what the expired segments contributed (title, counts, projects), so nothing +// derived from the prompt outlives them, and keeps only the mark. The summary of what the session +// held when the store's entry began (before) goes too, for the same reason: the store's list row +// merges it in. prune calls it for a marker it keeps. +func (a *Archive) markerOnly(s *sessionState) { + if onlyMark(s.meta.Summary, s.meta.ID) && (s.before == nil || onlyMark(s.before, s.meta.ID)) { + return // already only a marker: no rewrite on every prune + } + fold := session.NewSummaryFold() + fold.MarkNoRecord() + s.meta.Summary, s.metaDirty = fold, true + if s.before != nil { + s.before = fold.Clone() + } + a.publish(s) +} + +// onlyMark reports whether f, a fold of session id, holds nothing derived from content. +func onlyMark(f *session.SummaryFold, id string) bool { + sum := f.Summary(id) + return sum.EventCount == 0 && sum.Title == "" +} + +// keepMarker reports whether prune keeps s, a session with nothing on disk, because it marks a +// withheld session. A marker lives retainDays after the session's last event, like a segment: +// after that, the agent's own transcript has usually expired too. Open makes retainDays positive. +func keepMarker(s *sessionState, now time.Time, retainDays int) bool { + return s.meta.Summary.NoRecord() && now.Sub(s.meta.UpdatedAt) < time.Duration(retainDays)*24*time.Hour +} diff --git a/core/session/archive/norecord2_test.go b/core/session/archive/norecord2_test.go new file mode 100644 index 000000000..66cb6f747 --- /dev/null +++ b/core/session/archive/norecord2_test.go @@ -0,0 +1,604 @@ +package archive + +import ( + "errors" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/session" +) + +// Review round 2, item 1: an imported session that a live request later withholds is never +// replaced by an import, also after a restart, when its Source still says imported and its +// numbering matches the disk. +func TestNoRecord_AnImportNeverReplacesAWithheldImportedSession(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + if err := a.WriteSession("s1", importedEvents(4), false); err != nil { + t.Fatalf("import: %v", err) + } + r := newRecorder(a) + r.seq["s1"] = 4 + r.record("s1", flagged(synthSession(320, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + defer a2.Close() + if err := a2.WriteSession("s1", importedEvents(6), true); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(replace) on a withheld imported session = %v, want ErrLive", err) + } +} + +// Review round 2, item 3: the default bucket and a pending bucket are never withheld, even when a +// flagged event reaches the archive under their id. They hold many clients' calls. +func TestNoRecord_SharedBucketsAreNeverWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + for _, id := range []string{session.DefaultSessionID, session.PendingSessionID("claude-code")} { + r.record(id, flagged(synthSession(321, 1, 512))...) + } + a.Close() + for _, id := range []string{session.DefaultSessionID, session.PendingSessionID("claude-code")} { + if got := onDisk(t, root, id); len(got) != 3 { + t.Fatalf("%s holds %d events, want all 3 recorded", id, len(got)) + } + } +} + +// Review round 3, item 1: praxis mode. The store mark for a hopped request (MarkNoRecord) is +// dropped when the store evicts the entry, and the gateway's ingested rows carry no flag. The +// archive must already hold its own mark by then, so those rows are withheld. +func TestNoRecord_HopMarkSurvivesEviction(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + st := session.New(time.Hour, 0, 1) // one session: the next one evicts the first + defer st.Close() + st.AddRecorder(a) + evs := synthSession(330, 2, 512) + for _, e := range evs[:3] { + st.Append("s1", e) // recorded before the opt-out + } + st.MarkNoRecord("s1") // a hopped request with X-Cortex-Record: off + st.Append("s2", synthSession(331, 1, 512)[0]) + for _, e := range evs[3:] { + st.Append("s1", e) // the gateway's rows, ingested without the flag + } + settle(a) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != 3 { + t.Fatalf("s1 holds %d events, want the 3 recorded before the mark", len(got)) + } +} + +// gateWriter holds a's writer before its next op and returns an idempotent release, also +// registered as cleanup with a's Close, so a failing test never leaves the writer gated. +func gateWriter(t *testing.T, a *Archive) (release func()) { + t.Helper() + gate := make(chan struct{}) + a.gate = gate + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(func() { + release() + _ = a.Close() + }) + return release +} + +// fillToReserve queues events of id until the queue is at the reserve line, with the gated +// writer holding the first one, and returns how many it recorded. +func fillToReserve(t *testing.T, a *Archive, r *recorder, id string) int { + t.Helper() + evs := synthSession(390, 1, 512)[:1] + r.record(id, evs...) + deadline := time.Now().Add(10 * time.Second) + for len(a.ops) > 0 { // the writer has taken it and waits at the gate + if time.Now().After(deadline) { + t.Fatal("the writer never took the first op") + } + time.Sleep(time.Millisecond) + } + n := 1 + for len(a.ops) < cap(a.ops)-reserve { + r.record(id, plain(n)...) + n++ + } + return n +} + +// plain is one unflagged event, distinct by i. +func plain(i int) []pipeline.SessionEvent { + e := synthSession(391, 1, 512)[0] + e.At = e.At.Add(time.Duration(i) * time.Millisecond) + return []pipeline.SessionEvent{e} +} + +// The writer's mark is the opt-out's only state, so a flagged event of a session not marked yet is +// never lost: with the queue full, reserve included, it is kept at its place and returns at once. +func TestNoRecord_AFlaggedEventIsNeverLostWhileUnmarked(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + release := gateWriter(t, a) + r := newRecorder(a) + evs := synthSession(332, 3, 512) // 9 events + r.record("s1", evs[:6]...) // to the reserve + for len(a.ops) < cap(a.ops) { // and the reserve too + a.Rekeyed("filler", "filler2") + } + done := make(chan struct{}) + go func() { + r.record("s1", flagged(evs[6:7])...) + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a flagged event waited for the writer") + } + release() + settle(a) + r.record("s1", evs[7:]...) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != 6 { + t.Fatalf("s1 holds %d events, want the 6 recorded before the flag", len(got)) + } +} + +// clawgenti's probe on #1959, through the real store: with the writer stuck (a hung fsync, a +// stalled mount) and the queue at its reserve line, one opted-out request must not block any other +// session's traffic, nor itself. Its opt-out still lands, and no flagged event took the reserve. +func TestNoRecord_AStuckWriterNeverBlocksTheStore(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + st := session.New(time.Hour, 0, 0) + t.Cleanup(st.Close) // registered first, so it runs after gateWriter's release + st.AddRecorder(a) + release := gateWriter(t, a) + evs := synthSession(392, 4, 512) // 12 events + st.Append("s1", evs[0]) + for deadline := time.Now().Add(10 * time.Second); len(a.ops) > 0; { + if time.Now().After(deadline) { + t.Fatal("the writer never took the first op") + } + time.Sleep(time.Millisecond) + } + n := 1 + for ; len(a.ops) < cap(a.ops)-reserve; n++ { + st.Append("s1", evs[n]) + } + fl := evs[n] + fl.NoRecord = true + other := synthSession(393, 1, 512) + done := make(chan string, 3) + go func() { st.Append("s1", fl); done <- "the opted-out Append" }() + go func() { st.Append("s2", other[0]); done <- "another session's Append" }() + go func() { st.ListSessions(); done <- "ListSessions" }() + for range 3 { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the store blocked behind a stuck archive writer") + } + } + if q := len(a.ops); q > cap(a.ops)-reserve { + t.Fatalf("queue at %d with the writer gated, past the reserve line %d: a flagged event took the reserve", q, cap(a.ops)-reserve) + } + a.Rekeyed("x1", "x2") // the reserve is still free for a rename + if got := a.Stats().DroppedRenames; got != 0 { + t.Fatalf("dropped %d renames: the reserve was taken", got) + } + release() + settle(a) + for _, e := range evs[n+1:] { + st.Append("s1", e) // unflagged, after the opt-out + } + st.Append("s2", other[1]) + settle(a) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != n { + t.Fatalf("s1 holds %d events, want the %d recorded before the opt-out", len(got), n) + } + if got := onDisk(t, root, "s2"); len(got) != 1 { + t.Fatalf("s2 holds %d events, want its row after the writer recovered", len(got)) + } +} + +// A kept opt-out applies at its place in the queue: after a DELETE queued before it, which would +// otherwise remove its mark, and again after a DELETE queued between two opt-outs of one session +// (the entry adds a position). In both cases the unflagged rows that follow are withheld. +func TestNoRecord_AKeptOptOutKeepsItsPlaceAfterADelete(t *testing.T) { + for _, tc := range []struct { + name string + ops func(a *Archive) + }{ + {"delete, then opt-out", func(a *Archive) { + a.Deleted("s1") + a.Withhold("s1") + }}, + {"opt-out, delete, opt-out", func(a *Archive) { + a.Withhold("s1") + a.Deleted("s1") + a.Withhold("s1") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + r.record("s1", flagged(synthSession(394, 1, 512))...) + settle(a) // s1 is marked + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + tc.ops(a) // with the queue at the line: every Withhold is kept, not queued + if a.optOutN.Load() != 1 { + t.Fatalf("%d kept opt-outs, want 1 for s1", a.optOutN.Load()) + } + release() + if _, err := a.AwaitDelete("s1", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + settle(a) + r.seq["s1"] = 0 + r.record("s1", synthSession(395, 1, 512)...) // unflagged, after the opt-out + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the opt-out was applied before the delete", segs) + } + }) + } +} + +// An unflagged row of a session that gets a queue slot while its opt-out is still kept (the +// writer has made room, but not reached the opt-out's place) is withheld: the opt-out applies +// before it. +func TestNoRecord_ARowQueuedWhileTheOptOutIsKeptIsWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + r.record("s1", flagged(plain(1))...) // no room: kept + if a.optOutN.Load() != 1 { + t.Fatal("precondition: the opt-out is kept") + } + a.gate <- struct{}{} // one op through: the writer takes the next and waits again + for deadline := time.Now().Add(10 * time.Second); len(a.ops) >= cap(a.ops)-reserve; { + if time.Now().After(deadline) { + t.Fatal("the writer made no room") + } + time.Sleep(time.Millisecond) + } + r.record("s1", plain(2)...) // unflagged, queued behind the opt-out's place + if a.optOutN.Load() != 1 { + t.Fatal("the opt-out was applied before the row was queued") + } + release() + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: a row queued while the opt-out was kept was recorded", segs) + } +} + +// One entry per session, however often it opts out while the queue is full, and none left once +// the writer has applied it. +func TestNoRecord_KeptOptOutsAreOnePerSessionAndCleared(t *testing.T) { + a := openTest(t, t.TempDir(), newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for range 1000 { + a.Withhold("s1") + r.record("s1", flagged(plain(1))...) + } + a.mu.Lock() + n, tags := len(a.optOuts), len(a.optOuts["s1"]) + a.mu.Unlock() + if n != 1 || tags != 1 { + t.Fatalf("%d entries, s1 with %d positions; want 1 and 1", n, tags) + } + release() + settle(a) + if got := a.optOutN.Load(); got != 0 { + t.Fatalf("%d kept opt-outs after the writer caught up, want 0", got) + } +} + +// A kept opt-out still pending at Close is written as a marker, so the session stays withheld +// after a restart. A praxis Withhold is kept the same way. +func TestNoRecord_AKeptOptOutSurvivesCloseAndRestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + r.record("s1", flagged(plain(1))...) + a.Withhold("s2") + release() + a.Close() + + a2 := openTest(t, root, newClock()) + r2 := newRecorder(a2) + r2.seq["s1"], r2.seq["s2"] = 1, 0 + r2.record("s1", synthSession(396, 1, 512)...) + r2.record("s2", synthSession(397, 1, 512)...) + a2.Close() + for _, id := range []string{"s1", "s2"} { + if segs := segmentsIn(t, root, id); len(segs) != 0 { + t.Fatalf("%s: segments %v after a restart, want none", id, segs) + } + } +} + +// Past its bound on kept opt-outs the archive fails closed: no conversation is recorded until a +// restart, and Stats says so. A shared bucket still records. +func TestNoRecord_TooManyKeptOptOutsFailClosed(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for i := range a.depth + 1 { + a.Withhold(fmt.Sprintf("forged-%d", i)) + } + if !a.Stats().FailedClosed { + t.Fatal("past the bound, the archive did not fail closed") + } + if got := a.optOutN.Load(); got != int64(a.depth) { + t.Fatalf("%d kept opt-outs, want the bound %d", got, a.depth) + } + release() + settle(a) + r.record("s9", synthSession(398, 1, 512)...) + r.record(session.DefaultSessionID, synthSession(399, 1, 512)...) + a.Close() + if segs := segmentsIn(t, root, "s9"); len(segs) != 0 { + t.Fatalf("segments %v for a conversation after failing closed, want none", segs) + } + if got := onDisk(t, root, session.DefaultSessionID); len(got) != 3 { + t.Fatalf("default holds %d events, want its 3", len(got)) + } +} + +// Failed closed, the archive records no conversation, but a flagged event that finds room still +// writes its mark: the fail-closed state is lost at a restart, and the mark is not. +func TestNoRecord_AnOptOutAfterFailingClosedIsStillMarked(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + r := newRecorder(a) + release := gateWriter(t, a) + fillToReserve(t, a, r, "s0") + for i := range a.depth + 1 { + a.Withhold(fmt.Sprintf("forged-%d", i)) + } + if !a.Stats().FailedClosed { + t.Fatal("precondition: the archive failed closed") + } + release() + settle(a) + r.record("s7", flagged(plain(1))...) // room again: the opt-out is admitted + settle(a) + a.Close() + + a2 := openTest(t, root, newClock()) + r2 := newRecorder(a2) + r2.seq["s7"] = 1 + r2.record("s7", synthSession(400, 1, 512)...) // unflagged, after the restart + a2.Close() + if segs := segmentsIn(t, root, "s7"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none: the opt-out sent while failed closed was lost", segs) + } +} + +// A DELETE still queued when a praxis Withhold arrives: the session was marked, but the delete is +// about to remove the mark, so Withhold must not take "already marked" for granted. The rows +// ingested after it are withheld. +func TestNoRecord_WithholdWhileADeleteIsQueued(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(340, 1, 512))...) + settle(a) // marked and published + a.gate = make(chan struct{}) + a.Deleted("s1") // queued, not applied yet + a.Withhold("s1") // a hopped request withholds s1 again + r.record("s1", synthSession(341, 1, 512)...) // the gateway's rows, unflagged + close(a.gate) + if _, err := a.AwaitDelete("s1", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the rows after the delete were archived", segs) + } +} + +// The same for a flagged event behind a queued CLEAR: it is delivered, and the session is marked +// again after the clear. +func TestNoRecord_FlaggedEventWhileAClearIsQueued(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(342, 1, 512))...) + settle(a) + a.gate = make(chan struct{}) + a.Cleared() + evs := synthSession(343, 1, 512) + r.record("s1", flagged(evs[:1])...) + r.record("s1", evs[1:]...) + close(a.gate) + if _, err := a.AwaitClear(5 * time.Second); err != nil { + t.Fatalf("clear: %v", err) + } + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none: the opt-out after the clear was lost", segs) + } +} + +// Flagged events never go into the reserve: it stays for renames, deletes and clears, which a +// burst of flagged events must not crowd out. Past the reserve line they are kept, and the +// producer returns. +func TestNoRecord_FlaggedEventsLeaveTheReserveAlone(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock(), WithQueueDepth(reserve+6)) + release := gateWriter(t, a) + r := newRecorder(a) + fillToReserve(t, a, r, "s1") + // More than the reserve holds: if flagged events could use the reserve, they would fill it. + evs := make([]pipeline.SessionEvent, reserve+2) + for i := range evs { + evs[i].NoRecord = true + } + done := make(chan struct{}) + go func() { + r.record("s2", evs...) + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the flagged producer waited for the writer") + } + if q := len(a.ops); q > cap(a.ops)-reserve { + t.Fatalf("queue at %d with the writer gated, past the reserve line %d", q, cap(a.ops)-reserve) + } + a.Rekeyed("s1", "s3") // with the writer still gated: must still find a reserve slot + if got := a.Stats().DroppedRenames; got != 0 { + t.Fatalf("dropped %d renames: flagged events took the reserve", got) + } + release() + a.Close() +} + +// Review round 3: when retention leaves a withheld session with nothing, the summary of what it +// held when the store's entry began (before) goes too, so the store's list row shows no expired +// title or counts. +func TestNoRecord_MarkerOnlyDropsThePriorSummary(t *testing.T) { + a := openTest(t, t.TempDir(), newClock()) + defer a.Close() + r := newRecorder(a) + r.record("s1", synthSession(346, 1, 512)...) + settle(a) + a.inWriter(func() { + s := a.sessions["s1"] + s.before = s.meta.Summary.Clone() // as begin leaves it for a new store entry + s.meta.Summary.MarkNoRecord() + a.markerOnly(s) + if !onlyMark(s.before, "s1") || !onlyMark(s.meta.Summary, "s1") { + t.Errorf("after markerOnly: before %+v, fold %+v; want only the mark", + s.before.Summary("s1"), s.meta.Summary.Summary("s1")) + } + }) +} + +// Review: when retention expires the last segment of a withheld session, the marker it keeps +// holds only the mark, not the expired history's title and counts. +func TestNoRecord_AnExpiredHistoryLeavesOnlyTheMark(t *testing.T) { + root := t.TempDir() + clk := newClock() + a := openTest(t, root, clk, WithRetentionDays(1)) + r := newRecorder(a) + evs := synthSession(345, 1, 512) + r.record("s1", evs[:2]...) // history before the opt-out, with a title + settle(a) + clk.advance(30 * time.Hour) // past retention for the segment + r.record("s1", flagged(evs[2:])...) // the opt-out, today + settle(a) + a.inWriter(func() { a.prune(clk.now()) }) + a.Close() + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatalf("the marker is gone: %v", err) + } + if sum := m.Summary.Summary("s1"); !sum.NoRecord || sum.EventCount != 0 || sum.Title != "" || len(m.Segments) != 0 { + t.Fatalf("after retention: %+v with %d segments, want only the mark", sum, len(m.Segments)) + } +} + +// Review: an opt-out that reaches the archive after the writer stopped reading the queue for Close +// is refused and counted, never queued or kept where nothing reads it. +func TestNoRecord_AnOptOutIsRefusedAfterTheWriterStopped(t *testing.T) { + a := openTest(t, t.TempDir(), newClock()) + defer a.Close() + a.mu.Lock() + a.stopped = true // as the writer sets it after its drain for Close + a.mu.Unlock() + before := a.Stats().DroppedEvents + a.Withhold("s1") + if got := a.Stats().DroppedEvents; got != before+1 { + t.Fatalf("dropped %d → %d, want the op counted as undelivered", before, got) + } + if n := len(a.ops); n != 0 || a.optOutN.Load() != 0 { + t.Fatalf("%d ops queued and %d opt-outs kept after the writer stopped, want none", n, a.optOutN.Load()) + } + a.mu.Lock() + a.stopped = false + a.mu.Unlock() +} + +// A rename moves a marked session's mark inside the rename op, so a DELETE queued after it +// removes the new id for good: no marker comes back behind the delete. +func TestNoRecord_RenameThenDeleteLeavesNothing(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(333, 1, 512))...) + settle(a) + a.Rekeyed("s1", "s2") + a.Deleted("s2") + if _, err := a.AwaitDelete("s2", 5*time.Second); err != nil { + t.Fatalf("delete: %v", err) + } + settle(a) + r.seq["s2"] = 0 + r.record("s2", synthSession(334, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s2"); len(got) != 3 { + t.Fatalf("s2 holds %d events after DELETE, want the 3 recorded since: the mark came back", len(got)) + } +} + +// A clear removes every mark with the history: the user's explicit erase. Later unflagged events +// record again. +func TestNoRecord_ClearRemovesTheMark(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(335, 1, 512))...) + settle(a) + a.Cleared() + if _, err := a.AwaitClear(5 * time.Second); err != nil { + t.Fatalf("clear: %v", err) + } + r.seq["s1"] = 0 + r.record("s1", synthSession(336, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s1"); len(got) != 3 { + t.Fatalf("s1 holds %d events after CLEAR, want the 3 recorded since", len(got)) + } +} + +// Review round 2, item 4: a flagged event dropped while the archive is paused still persists the +// mark, so a restart before the session's next event keeps it withheld. +func TestNoRecord_ADroppedFlagIsPersistedBeforeARestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + a.paused.Store(true) + newRecorder(a).record("s1", flagged(synthSession(322, 1, 512)[:1])...) + settle(a) + a.Close() + + a2 := openTest(t, root, newClock()) + r := newRecorder(a2) + r.seq["s1"] = 1 + r.record("s1", synthSession(323, 1, 512)...) + a2.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none", segs) + } +} diff --git a/core/session/archive/norecord_test.go b/core/session/archive/norecord_test.go new file mode 100644 index 000000000..224c76395 --- /dev/null +++ b/core/session/archive/norecord_test.go @@ -0,0 +1,200 @@ +package archive + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/pipeline" +) + +// flagged marks every event as asking not to be recorded, as the listeners do when a request +// carries X-Cortex-Record: off. +func flagged(evs []pipeline.SessionEvent) []pipeline.SessionEvent { + for i := range evs { + evs[i].NoRecord = true + } + return evs +} + +// segmentsIn lists the segment files under id's directory. +func segmentsIn(t *testing.T, root, id string) []string { + t.Helper() + entries, err := os.ReadDir(filepath.Join(root, dataDirName, dirName(id))) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + var out []string + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".jsonl.zst") { + out = append(out, e.Name()) + } + } + return out +} + +// A flagged session writes no segment, only a session.json that marks it as not recorded and +// holds nothing else: no events, no title, no tokens. +func TestNoRecord_WritesOnlyAContentFreeMarker(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(301, 2, 512))...) + a.Close() + + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatalf("no marker: %v", err) + } + sum := m.Summary.Summary("s1") + if !sum.NoRecord || sum.EventCount != 0 || sum.Title != "" || sum.TotalTokens != 0 || len(m.Segments) != 0 { + t.Fatalf("marker = %+v with %d segments, want noRecord and nothing else", sum, len(m.Segments)) + } +} + +// Header-less events filed to a withheld session carry no flag. They are withheld too. +func TestNoRecord_LaterUnflaggedEventsAreWithheld(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(302, 2, 512) + r.record("s1", flagged(evs[:2])...) + r.record("s1", evs[2:]...) + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } +} + +// The opt-out applies from the first flagged event on and never backwards: what the archive +// already held stays (a session-id header is client-asserted, so a request naming another +// session must not erase it), and nothing after the flag is added. +func TestNoRecord_KeepsWhatCameBeforeTheFlagAndAddsNothing(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(303, 2, 512) + r.record("s1", evs[:2]...) + r.record("s1", flagged(evs[2:3])...) + r.record("s1", evs[3:]...) + a.Close() + got := onDisk(t, root, "s1") + if len(got) != 2 || got[0].Seq != 1 || got[1].Seq != 2 { + t.Fatalf("s1 holds %d events, want the 2 recorded before the flag", len(got)) + } + m, err := readMeta(filepath.Join(root, dataDirName, dirName("s1"))) + if err != nil { + t.Fatal(err) + } + if !m.Summary.NoRecord() { + t.Fatal("the session is not marked after the flag") + } +} + +// A withheld session stays withheld when the store renames it, on the path that rebuilds the +// fold from segments (an entry re-created after the marker, then renamed whole). +func TestNoRecord_RenameKeepsTheMark(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(309, 1, 512))...) + settle(a) + a.inWriter(func() { a.begin("s1", 3) }) + a.inWriter(func() { a.rename("s1", "s2", 3) }) + // Right after the rename and before any event of s2, so the in-memory mark cannot stand in + // for a fold that lost it: the summary under the new id, and session.json on disk. + var marked bool + a.inWriter(func() { s := a.sessions["s2"]; marked = s != nil && s.meta.Summary.NoRecord() }) + if !marked { + t.Fatal("the renamed session's summary is not marked") + } + if m, err := readMeta(filepath.Join(root, dataDirName, dirName("s2"))); err != nil || !m.Summary.NoRecord() { + t.Fatalf("session.json under the new id: %v, marked %v", err, err == nil && m.Summary.NoRecord()) + } + r.seq["s2"] = 3 + r.record("s2", synthSession(310, 1, 512)...) + a.Close() + if segs := segmentsIn(t, root, "s2"); len(segs) != 0 { + t.Fatalf("segments %v under the new id, want none", segs) + } + a2 := openTest(t, root, newClock()) + defer a2.Close() + if err := a2.WriteSession("s2", importedEvents(4), false); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(s2) after the rename = %v, want ErrLive", err) + } +} + +// A flagged event the writer never sees (here, while the archive is paused) still marks the +// session, so the header-less events after it are withheld. +func TestNoRecord_ADroppedFlaggedEventStillMarks(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + evs := synthSession(311, 1, 512) + a.paused.Store(true) + r.record("s1", flagged(evs[:1])...) + settle(a) + a.paused.Store(false) + r.record("s1", evs[1:]...) + a.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v, want none", segs) + } +} + +// The marker keeps the opt-out across a restart, when the store's own mark is gone: an +// unflagged event for the id is still withheld. +func TestNoRecord_SurvivesARestart(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(304, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + r := newRecorder(a2) + r.seq["s1"] = a2.LastSeq("s1") + r.record("s1", synthSession(305, 1, 512)...) + a2.Close() + if segs := segmentsIn(t, root, "s1"); len(segs) != 0 { + t.Fatalf("segments %v after a restart, want none", segs) + } +} + +// An import refuses a withheld session, also after a restart: otherwise agentop import would +// bring it in from the agent's own transcript. +func TestNoRecord_AnImportRefusesTheSession(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + newRecorder(a).record("s1", flagged(synthSession(306, 1, 512))...) + a.Close() + + a2 := openTest(t, root, newClock()) + defer a2.Close() + for _, replace := range []bool{false, true} { + if err := a2.WriteSession("s1", importedEvents(4), replace); !errors.Is(err, ErrLive) { + t.Fatalf("WriteSession(replace=%v) = %v, want ErrLive", replace, err) + } + } +} + +// Another session is untouched by one that opted out. +func TestNoRecord_OtherSessionsAreRecorded(t *testing.T) { + root := t.TempDir() + a := openTest(t, root, newClock()) + r := newRecorder(a) + r.record("s1", flagged(synthSession(307, 1, 512))...) + r.record("s2", synthSession(308, 1, 512)...) + a.Close() + if got := onDisk(t, root, "s2"); len(got) != 3 { + t.Fatalf("s2 holds %d events, want 3", len(got)) + } + for _, e := range onDisk(t, root, "s2") { + if e.NoRecord { + t.Fatalf("a recorded event carries NoRecord: seq %d", e.Seq) + } + } +} diff --git a/core/session/fold.go b/core/session/fold.go index 888273fd4..bc486d488 100644 --- a/core/session/fold.go +++ b/core/session/fold.go @@ -301,8 +301,18 @@ type SummaryFold struct { // (Source ""). Summary marks the session as imported only while live is false. imported string live bool + // noRecord is true once any event asked not to be recorded (SessionEvent.NoRecord, #1923), + // or once MarkNoRecord was called. Sticky: a later event without the flag does not clear it. + noRecord bool } +// MarkNoRecord marks the session as not recorded without folding an event, for the archive's +// content-free marker of an opted-out session (#1923). +func (f *SummaryFold) MarkNoRecord() { f.noRecord = true } + +// NoRecord reports whether the session asked not to be recorded; see SummaryFold.noRecord. +func (f *SummaryFold) NoRecord() bool { return f.noRecord } + // InferenceHosts is a copy of the fold's table, and false when the fold holds history it has no // table for. func (f *SummaryFold) InferenceHosts() (model.InferenceHosts, bool) { @@ -365,6 +375,9 @@ func (f *SummaryFold) Add(sessionID string, e *model.SessionEvent) { } else if f.imported == "" { f.imported = e.Source } + if e.NoRecord { + f.noRecord = true + } } // promptContext is the fold's CTX figure, nil when nothing can be said. @@ -394,6 +407,7 @@ func (f *SummaryFold) Summary(sessionID string) SessionSummary { LastRequestAt: f.lastRequestAt, Agent: pickAgent(sessionID, f.owner, nil, f.agents), Projects: slices.Clone(f.projects), + NoRecord: f.noRecord, ParentSession: model.LenientString(f.parentSession), } if !f.live { @@ -443,6 +457,9 @@ type foldJSON struct { // before they existed holds no imported events, so it decodes as not imported. Imported string `json:"imported,omitempty"` Live bool `json:"live,omitempty"` + // NoRecord is SummaryFold's field of the same name. Absent from every recorded session's + // session.json, which decodes to recorded. + NoRecord bool `json:"noRecord,omitempty"` } // hostsDoc is SummaryFold's InferenceHosts table in session.json. V is the hostRuleVersion that @@ -499,6 +516,7 @@ func (f *SummaryFold) MarshalJSON() ([]byte, error) { ParentSession: model.LenientString(f.parentSession), Imported: f.imported, Live: f.live, + NoRecord: f.noRecord, } if !f.hostsUnknown { j.InferenceHosts = &hostsDoc{V: hostRuleVersion, Hosts: f.hosts} @@ -537,6 +555,7 @@ func (f *SummaryFold) UnmarshalJSON(b []byte) error { // session.json written before Live existed. live: j.Live || (j.EventCount > 0 && j.Imported == ""), hostsUnknown: j.InferenceHosts == nil || j.InferenceHosts.V != hostRuleVersion, + noRecord: j.NoRecord, } if j.TitleRank != nil { f.titleRank = *j.TitleRank diff --git a/core/session/history.go b/core/session/history.go index c1e2ec95e..a38de8733 100644 --- a/core/session/history.go +++ b/core/session/history.go @@ -179,6 +179,9 @@ func (s *Store) withPriorLocked(sum *SessionSummary, id string, sess *entry) { f := p.Fold sum.EventCount += f.events sum.TotalTokens += f.tokens + // The archive's mark is the opt-out (#1923): a withheld session shows it after a restart, + // before its next flagged event. + sum.NoRecord = sum.NoRecord || f.noRecord // CostSum's addition for the sums, OR for the flags: a total that clamped on either side is a // bound, whichever side it was. cost, avoided := sess.cost, sess.avoided diff --git a/core/session/norecord_test.go b/core/session/norecord_test.go new file mode 100644 index 000000000..972a7aac8 --- /dev/null +++ b/core/session/norecord_test.go @@ -0,0 +1,121 @@ +package session + +import ( + "slices" + "testing" + "time" + + "github.com/rossoctl/cortex/core/model" +) + +// A session shows NoRecord from its first flagged event on, and a later event without the flag +// (a header-less call filed to it) does not clear it. Another session is not marked. +func TestNoRecord_SummaryIsSticky(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest}) + s.Append("s2", model.SessionEvent{Phase: model.SessionRequest}) + + got := map[string]bool{} + for _, sum := range s.ListSessions() { + got[sum.ID] = sum.NoRecord + } + if !got["s1"] || got["s2"] { + t.Fatalf("NoRecord by session = %v, want s1 only", got) + } +} + +// MarkNoRecord tells every Withholder (the archive, which owns the opt-out) and marks the live +// entry's summary, for rows the listener does not build (praxis mode's /v1/ingest). It never +// names a shared bucket. +func TestNoRecord_MarkNoRecordTellsTheArchive(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + w := &withholder{} + s.AddRecorder(w) + s.Append("s1", model.SessionEvent{Phase: model.SessionRequest}) + s.MarkNoRecord("s1") + s.MarkNoRecord("s2") // no live entry yet: the archive still hears it + s.MarkNoRecord(DefaultSessionID) + if want := []string{"s1", "s2"}; !slices.Equal(w.ids, want) { + t.Fatalf("Withhold calls %v, want %v", w.ids, want) + } + found := false + for _, sum := range s.ListSessions() { + if sum.ID == "s1" { + found = true + if !sum.NoRecord { + t.Fatal("s1's live summary is not marked") + } + } + } + if !found { + t.Fatal("s1 is not in the session list") + } +} + +// withholder is a Recorder that also implements Withholder, as the archive does. +type withholder struct{ ids []string } + +func (w *withholder) Record(string, *model.SessionEvent) {} +func (w *withholder) Withhold(id string) { w.ids = append(w.ids, id) } + +// A header may name the default bucket or a pending bucket. Neither is one conversation, so +// the store clears the flag there and MarkNoRecord ignores them: no recorder sees a withheld +// shared bucket. +func TestNoRecord_SharedBucketsAreNeverWithheld(t *testing.T) { + s := New(time.Hour, 0, 0) + defer s.Close() + rec := &flagRecorder{} + s.AddRecorder(rec) + pending := PendingSessionID("claude-code") + s.Append(DefaultSessionID, model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.Append(pending, model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + s.MarkNoRecord(DefaultSessionID) + s.MarkNoRecord(pending) + s.Append(DefaultSessionID, model.SessionEvent{Phase: model.SessionRequest}) + s.Append(pending, model.SessionEvent{Phase: model.SessionRequest}) + if want := []bool{false, false, false, false}; !slices.Equal(rec.flags, want) { + t.Fatalf("NoRecord by event = %v, want %v", rec.flags, want) + } + for _, sum := range s.ListSessions() { + if sum.NoRecord { + t.Fatalf("%s is marked NoRecord", sum.ID) + } + } +} + +// flagRecorder keeps each event's NoRecord in order. The store calls it under its write lock. +type flagRecorder struct{ flags []bool } + +func (r *flagRecorder) Record(_ string, e *model.SessionEvent) { r.flags = append(r.flags, e.NoRecord) } + +// The archive's fold keeps the mark through session.json. +func TestNoRecord_FoldRoundTrips(t *testing.T) { + f := NewSummaryFold() + f.Add("s1", &model.SessionEvent{Phase: model.SessionRequest, NoRecord: true}) + f.Add("s1", &model.SessionEvent{Phase: model.SessionRequest}) + b, err := f.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var back SummaryFold + if err := back.UnmarshalJSON(b); err != nil { + t.Fatal(err) + } + if !back.NoRecord() || !back.Summary("s1").NoRecord { + t.Fatalf("NoRecord lost through %s", b) + } + plain, err := NewSummaryFold().MarshalJSON() + if err != nil { + t.Fatal(err) + } + var p SummaryFold + if err := p.UnmarshalJSON(plain); err != nil { + t.Fatal(err) + } + if p.NoRecord() { + t.Fatal("an unmarked fold decodes as NoRecord") + } +} diff --git a/core/session/store.go b/core/session/store.go index 498999756..5c151b2ea 100644 --- a/core/session/store.go +++ b/core/session/store.go @@ -85,6 +85,11 @@ type entry struct { // count rather than a set because a trim has to know when the LAST event in a unit leaves. units map[string]int + // noRecord is true once an event of this entry asked not to be recorded + // (SessionEvent.NoRecord, #1923). Sticky, and read by ListSessions as + // SessionSummary.NoRecord. The archive keeps its own sticky mark, which survives a restart. + noRecord bool + // money is each event's decoded contribution, PARALLEL TO Events: same length, same // order, same trim. // @@ -315,6 +320,38 @@ func (s *Store) backgroundCleanup() { } } +// MarkNoRecord withholds sessionID from the archive from now on (#1923), for a request whose rows +// the listener does not build itself: in praxis mode the gateway records a hopped request, and +// its rows come back through /v1/ingest without the flag, the header having been stripped before +// the hop. It tells every Withholder (the archive), whose mark is the opt-out's only state, and +// marks the live entry's summary when there is one. Call it before the request is hopped: the +// archive's op then precedes every row that comes back. +// +// The caller must have checked pipeline.Context.WithholdsSession: the request named sessionID +// with its own session-id header. Never for a fallback bucket. +func (s *Store) MarkNoRecord(sessionID string) { + if !IsConversation(sessionID) { // see appendLocked + return + } + s.mu.Lock() + defer s.mu.Unlock() + if sess := s.sessions[sessionID]; sess != nil { + sess.noRecord = true + } + for _, r := range s.recorders { + if w, ok := r.(Withholder); ok { + w.Withhold(sessionID) + } + } +} + +// Withholder is optionally implemented by a Recorder that keeps a session's events (the +// session archive): Withhold tells it, from now on, to keep none of sessionID's (#1923). Called +// under the store's write lock. It may wait for its own writer, never for the store. +type Withholder interface { + Withhold(sessionID string) +} + // Append adds an event to the named session. Creates the session if it // doesn't exist. Updates activeID to this session. Evicts the oldest event if the // session exceeds maxEvents — which by default it cannot, maxEvents being unset. @@ -497,6 +534,16 @@ func (s *Store) appendLocked(sessionID string, b *Bucket, event model.SessionEve // without needing to know which session it was appended to — critical for // outbound events that have no protocol-native session field. event.SessionID = sessionID + // Only one conversation's own session can opt out (#1923). The default bucket and a pending + // bucket hold many clients' calls, and a header can name them ("default" passes the + // session-id rules): a flag there would withhold everyone filed to it, and a pending mark + // would follow an Adopt onto a session that never asked. Such an event records as usual. + if event.NoRecord && !IsConversation(sessionID) { + event.NoRecord = false + } + if event.NoRecord { + sess.noRecord = true + } // Stamp the cursor before publishLocked below, so a subscriber streaming events // live sees the same Seq a paging client would get for that event. If this were diff --git a/core/session/summary.go b/core/session/summary.go index 1f15e89de..6f5a76cab 100644 --- a/core/session/summary.go +++ b/core/session/summary.go @@ -228,6 +228,11 @@ type SessionSummary struct { // Source is "imported:" while every event of the session is imported, and "" // otherwise. Only the archive's fold sets it: store entries are live. Source string `json:"source,omitempty"` + // NoRecord is true when the session asked not to be recorded (#1923): the archive keeps none + // of its events from then on, and what it held before stays. A session with nothing before + // is only a marker that it opted out. A live session shows it from its first flagged request, + // and after a restart from the archive's mark. + NoRecord bool `json:"noRecord,omitempty"` // LinkedTo, on a pending bucket, is the sessions its process named that OwnerAt gives a // call of the bucket, each with the span of its naming requests (Link says how close that // is); LinkedAll says every request in the bucket falls before one of them. Set by Link when @@ -288,6 +293,7 @@ func (s *Store) ListSessions() []SessionSummary { Title: sess.Title, Agent: pickAgent(id, sess.owner, nil, sess.agents), Projects: slices.Clone(sess.projects), + NoRecord: sess.noRecord, // The first non-empty parent, read like Projects. ParentSession: model.LenientString(sess.parentSession), // Still a walk, and deliberately left as one: it is a pointer deref per event diff --git a/core/sessionapi/archive.go b/core/sessionapi/archive.go index 70271075e..946d5953f 100644 --- a/core/sessionapi/archive.go +++ b/core/sessionapi/archive.go @@ -146,5 +146,6 @@ func (s *Server) withArchivedRows(resident []session.SessionSummary) ([]session. DroppedRenames: st.DroppedRenames, DroppedCaptures: st.DroppedCaptures, Paused: st.Paused, + FailedClosed: st.FailedClosed, } } diff --git a/core/sessionapi/summary_test.go b/core/sessionapi/summary_test.go index b0bc06325..36f9e04e0 100644 --- a/core/sessionapi/summary_test.go +++ b/core/sessionapi/summary_test.go @@ -101,6 +101,7 @@ func fullEvent() pipeline.SessionEvent { ParentSession: "s-parent", ParentProject: "~/src/parent", Source: "imported:claude-code", + NoRecord: true, } } @@ -241,6 +242,11 @@ func TestSummarizeEvent_DropsPayloadsKeepsTimelineFields(t *testing.T) { if got.Source != full.Source { t.Errorf("source = %q, want %q", got.Source, full.Source) } + // noRecord is kept for the same reason: a bool that marks a row of a session the archive + // withholds (#1923). + if got.NoRecord != full.NoRecord { + t.Errorf("noRecord = %v, want %v", got.NoRecord, full.NoRecord) + } if got.Direction != full.Direction || got.Phase != full.Phase { t.Error("direction/phase did not survive") } @@ -452,7 +458,9 @@ func TestSummarizeEvent_ShapeIsGuarded(t *testing.T) { // scalar, so the struct copy keeps it; asserted above. // 38 since ParentSession and ParentProject, TIMELINE data: a reader files the row // under RollupProject (#1991). Scalars, so the struct copy keeps them; asserted above. - {"SessionEvent", reflect.TypeOf(pipeline.SessionEvent{}), 38}, + // 39 since NoRecord, TIMELINE data: it marks a row of a withheld session (#1923). A + // scalar, so the struct copy keeps it; asserted above. + {"SessionEvent", reflect.TypeOf(pipeline.SessionEvent{}), 39}, // 25 since AgentRole, which is TIMELINE data: agentop's CONTEXT gauge reads it on every // row the timeline serves. It needs no assertion of its own in the projection test // beyond the equality one there — a scalar survives the struct copy, unlike the two diff --git a/docs/session-api.md b/docs/session-api.md index 6093d500d..8874134ce 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -100,6 +100,10 @@ knowing by name: imported (see [`POST /v1/import`](#post-v1import--transcript-import-laptop-only)). It is absent when the session has one or more live events. So a live request for an imported session gives it a live event, and the mark goes. +- **`noRecord`** — `true` once the session asked not to be recorded, with the + `X-Cortex-Record` header: the archive keeps none of its events from then on. Absent + otherwise. It is sticky, and the archive's mark carries it across a restart. See + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive). - **Pending buckets.** A coding agent's calls before its process names a session are recorded under `pending:@.` (or `pending:` when the process could not be looked up), carrying their `proc`. Nothing renames a @@ -115,8 +119,11 @@ knowing by name: (`session.archive`, laptop only): every session the archive holds and memory does not, marked `resident: false`, plus a top-level `archive` object — `{bytes, maxBytes, retentionDays}` and, only when nonzero, `droppedEvents`, - `writeErrors`, `droppedRenames`, `droppedCaptures`, `paused`, any of which means the history has - gaps. A session in both is listed once, as its resident row, and that row + `writeErrors`, `droppedRenames`, `droppedCaptures`, `paused`, `failedClosed`, any of which means the history has + gaps. `failedClosed` is permanent until a restart: more sessions asked not to be + recorded, while the archive's writer was behind, than it keeps track of, so it + records no conversation at all (see + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive)). A session in both is listed once, as its resident row, and that row covers the session's whole history: `eventCount`, `totalTokens`, `costMicros`, `title`, `agent`, `createdAt` and the rest continue from the archive's fold of what came before this process's entry. **This applies to the default list @@ -407,6 +414,47 @@ The next `agentop import` brings an erased session back from its transcript, as an imported session. Its cost is not counted again: the erase kept the ledger rows, and a forget record tells the import which turns those rows hold. +## Keeping a session out of the archive + +A client keeps its own session out of the session archive by sending +`X-Cortex-Record: off` on its requests. For Claude Code that is +`ANTHROPIC_CUSTOM_HEADERS="X-Cortex-Record: off"`. The rules: + +- **Only `off` counts**, matched without case or padding. Any other value, or + none, records as before. The header is removed before any plugin runs, and it + is never forwarded upstream. +- **The request must name its own session** with a session-id header from + `session.id_headers`. A request that names none is filed by fallback (the + active session or `default`), so its flag is ignored: it must not stop another + session's recording. So the header does nothing with `id_headers` empty, on a + vmcontrol session (attribution is pinned empty there), and through ext_proc, + which pins no naming header (#1960). `default` and pending buckets are never + withheld. +- **From then on, never backwards.** From the first flagged request, the archive + keeps none of the session's events, also the header-less ones filed to it by + process or client affinity. What it already held stays: the session-id header + is client-asserted, so a request must not be able to erase another session's + history by naming it. +- **Sticky.** The archive keeps a content-free marker for the session, also + across a restart, for `retention_days` after its last event. A resumed session + without the header stays withheld, and sending `on` later does not resume + recording. An import refuses a withheld session. An opt-out that did not fit + while the archive was failed closed is not remembered across a restart: the + session records again until it next sends the header. +- **Only [`DELETE /v1/sessions/{id}`](#delete-v1sessionsid--erase-one-session) + erases**, and `DELETE /v1/sessions` (which also clears the markers). After a + delete, the session records again unless it still sends the header, and the + next `agentop import` can bring it back from its transcript. +- **What it does not cover.** The store holds the rows in memory as usual, and + the live streams carry them, flagged `noRecord`. The cost ledger and usage + totals count the session: their rows carry no content. Plugins with their own + per-session state, such as context-guru's database, are not covered (#1962). + +The archive never makes a request wait for the opt-out. If more sessions opt +out while its writer is behind than it keeps track of (as many as its queue +holds, 4096 by default), it records no conversation until a restart and reports +`failedClosed` (see [`GET /v1/sessions`](#get-v1sessions)). + ## `POST /v1/import` — transcript import (laptop only) Reads the transcripts that an agent keeps on disk, and writes each session into @@ -955,6 +1003,11 @@ Every event on `/v1/sessions/{id}` and `/v1/events` carries: [`POST /v1/import`](#post-v1import--transcript-import-laptop-only) wrote. Absent on a live event. The proxy did not see an imported exchange, so such an event has no `host` and no capture. +- `noRecord` — `true` on the request and response rows of a request that sent + `X-Cortex-Record: off` and named its own session (see + [Keeping a session out of the archive](#keeping-a-session-out-of-the-archive)). + Absent otherwise, and on every row of `default` and of a pending bucket. Such + rows are in the store and on the live streams as usual; the archive keeps none. - `a2a` / `mcp` / `inference` — protocol parser payloads (one at most). - `invocations` — per-plugin invocation records for every plugin that ran on the pipeline pass, structured `{inbound: [...], outbound: [...]}`. Each entry diff --git a/docs/session-export.md b/docs/session-export.md index 0a55cd438..dd86e2b2b 100644 --- a/docs/session-export.md +++ b/docs/session-export.md @@ -115,6 +115,14 @@ secret reaches the encoded bytes, which fails CI. The count above is pinned. `TestDocSecretCountMatchesAllSecrets` fails if this page and the test disagree. +### A session that asked not to be recorded + +The archive is the export's only source, so a session that sent `X-Cortex-Record: off` +exports none of its events from then on. What the archive held from before the opt-out +stays, and exports as usual. A session that asked from its first request exports no +records at all. See +[Keeping a session out of the archive](session-api.md#keeping-a-session-out-of-the-archive). + ### Questions an export answers - Token spend per model diff --git a/praxis/filters/src/record/line.rs b/praxis/filters/src/record/line.rs index f377be9f8..c4df10431 100644 --- a/praxis/filters/src/record/line.rs +++ b/praxis/filters/src/record/line.rs @@ -46,6 +46,8 @@ pub const NOT_SET_IN_PRAXIS_MODE: &[&str] = &[ "parentProject", // An import sets it; every gateway row is live. "source", + // Cortex marks the session before the hop (Store.MarkNoRecord), so the gateway never sets it. + "noRecord", ]; fn is_zero(v: &i64) -> bool {