Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions core/cost/ledger/norecord_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package ledger

import (
"testing"
"time"
)

// A session that asked not to be recorded (SessionEvent.NoRecord, #1923) is kept out of the
// session archive only. Its spend still counts here: ledger rows carry no content, and the cost
// views must stay complete.
func TestWriter_CountsAWithheldSession(t *testing.T) {
dir := t.TempDir()
now := at
w := newTestWriter(t, dir, func() time.Time { return now })

for i := 0; i < 2; i++ {
ev := costedEvent(t, "gw", "m", 0.25, 100, 50)
ev.NoRecord = true
w.Record("s1", ev)
}
now = at.Add(time.Minute)
later := costedEvent(t, "gw", "m", 0.10, 10, 5)
later.At, later.NoRecord = now, true
w.Record("s1", later)
if err := w.sync(); err != nil {
t.Fatalf("sync: %v", err)
}

rows := readAllRows(t, dir)
if len(rows) != 1 || rows[0].Requests != 2 || rows[0].CostMicros != 500_000 {
t.Fatalf("rows = %+v, want one closed minute of 2 requests and 500000 micros", rows)
}
}
89 changes: 89 additions & 0 deletions core/listener/extproc/norecord_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package extproc

import (
"context"
"net/http"
"slices"
"testing"
"time"

corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"

"github.com/rossoctl/cortex/core/listener/skiphost"
"github.com/rossoctl/cortex/core/model"
"github.com/rossoctl/cortex/core/pipeline"
"github.com/rossoctl/cortex/core/session"
)

// removed returns the RemoveHeaders of a request-headers or request-body response.
func removed(t *testing.T, resp *extprocv3.ProcessingResponse) []string {
t.Helper()
var cr *extprocv3.CommonResponse
switch r := resp.Response.(type) {
case *extprocv3.ProcessingResponse_RequestHeaders:
cr = r.RequestHeaders.Response
case *extprocv3.ProcessingResponse_RequestBody:
cr = r.RequestBody.Response
}
if cr == nil || cr.HeaderMutation == nil {
return nil
}
return cr.HeaderMutation.RemoveHeaders
}

// A skip_hosts destination runs no pipeline, but the header is still for Cortex only: both the
// headers phase and the body phase remove it.
func TestNoRecord_ExtProcSkipHostRemovesTheHeader(t *testing.T) {
store := session.New(5*time.Minute, 100, 0)
defer store.Close()
skip, err := skiphost.New([]string{"otel-collector.*.svc.cluster.local"})
if err != nil {
t.Fatal(err)
}
srv, _ := newSkipServer(t, store, skip)
hdrs := func() *corev3.HeaderMap {
return makeHeaders(":authority", "otel-collector.ns.svc.cluster.local:8335", ":path", "/v1/traces",
"x-cortex-record", "off")
}

stream := &mockStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{outboundRequest(hdrs())}}
_ = srv.Process(stream)
if len(stream.responses) != 1 {
t.Fatalf("headers phase: %d responses, want 1", len(stream.responses))
}
if got := removed(t, stream.responses[0]); !slices.Contains(got, "x-cortex-record") {
t.Fatalf("headers phase: RemoveHeaders %v, want x-cortex-record removed", got)
}

resp, pctx := srv.handleOutboundBody(&mockStream{ctx: context.Background()}, hdrs(), []byte(`{}`))
if pctx != nil {
t.Fatal("body phase ran the pipeline for a skipped host")
}
if got := removed(t, resp); !slices.Contains(got, "x-cortex-record") {
t.Fatalf("body phase: RemoveHeaders %v, want x-cortex-record removed", got)
}
}

// ext_proc forwards the wire headers and only the changes it emits. ResolveClient removed
// X-Cortex-Record from pctx.Headers before the listener cloned them, so the diff alone cannot
// see it: withHeaderMutation must remove it explicitly, whatever its value.
func TestNoRecord_ExtProcRemovesTheHeader(t *testing.T) {
for _, v := range []string{"off", "on"} {
h := http.Header{}
h.Set(model.RecordHeader, v)
pctx := &pipeline.Context{Headers: h}
pctx.ResolveClient()
orig := pctx.Headers.Clone() // as handleInbound and handleOutbound do
got := removed(t, withHeaderMutation(allowResponse(), pctx, orig))
if !slices.Contains(got, "x-cortex-record") {
t.Fatalf("value %q: RemoveHeaders = %v, want x-cortex-record", v, got)
}
}
// No header: nothing extra is removed.
pctx := &pipeline.Context{Headers: http.Header{"User-Agent": {"curl/8"}}}
pctx.ResolveClient()
if got := removed(t, withHeaderMutation(allowResponse(), pctx, pctx.Headers.Clone())); slices.Contains(got, "x-cortex-record") {
t.Fatalf("RemoveHeaders = %v with no header sent", got)
}
}
16 changes: 13 additions & 3 deletions core/listener/extproc/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/rossoctl/cortex/core/listener/internal/sessionevent"
"github.com/rossoctl/cortex/core/listener/internal/sseframe"
"github.com/rossoctl/cortex/core/listener/skiphost"
"github.com/rossoctl/cortex/core/model"
"github.com/rossoctl/cortex/core/pipeline"
"github.com/rossoctl/cortex/core/session"
)
Expand Down Expand Up @@ -504,7 +505,9 @@ func (s *Server) handleOutbound(stream extprocv3.ExternalProcessor_ProcessServer
// handleResponseBody) and the deferred RunFinish to no-op as well —
// all four phases are skipped consistently. See ListenerConfig.SkipHosts.
if s.SkipHosts.Match(pctx.Host) {
return passResponse(), nil
// No pipeline ran, so pctx.Headers is its own original and only the X-Cortex-Record
// removal is emitted: the header is for Cortex only, also toward a skipped host (#1923).
return withHeaderMutation(passResponse(), pctx, pctx.Headers), nil
}

if s.Sessions != nil {
Expand Down Expand Up @@ -555,7 +558,7 @@ func (s *Server) handleOutboundBody(stream extprocv3.ExternalProcessor_ProcessSe
if pat, matched := s.SkipHosts.MatchPattern(pctx.Host); matched {
slog.Info("ext_proc: skip_hosts match (body phase) — bypassing pipeline + session recording",
"host", pipeline.SanitizeLogValue(pctx.Host), "pattern", pat, "path", pipeline.SanitizeLogValue(pctx.Path))
return allowBodyResponse(), nil
return withHeaderMutation(allowBodyResponse(), pctx, pctx.Headers), nil // as in handleOutbound
}

if s.Sessions != nil {
Expand Down Expand Up @@ -968,7 +971,9 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte
var set []*corev3.HeaderValueOption
var del []string
for k, vv := range pctx.Headers {
if skip(k) || slices.Equal(orig[k], vv) {
// Never forward X-Cortex-Record (#1923), also when a plugin wrote it back: it is removed
// below, and a set beside the remove would leave the outcome to Envoy's ordering.
if skip(k) || slices.Equal(orig[k], vv) || strings.EqualFold(k, model.RecordHeader) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit — this path defends against a plugin writing X-Cortex-Record back ("also when a plugin wrote it back"), but httpx.SyncRequestHeaders (core/listener/httpx/request.go:34) skips only Content-Length/Content-Encoding — so the same plugin write-back on the forward/reverse proxy paths would forward the header upstream. The same strings.EqualFold(k, model.RecordHeader) in that skip list would make the two listeners agree. Happy to see it in a follow-up; it is outside this PR's diff.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, agreed. Outside this PR's diff, so it's a follow-up: #2005 (skip X-Cortex-Record in httpx.SyncRequestHeaders, as ext_proc does).

Assisted-By: Claude (Anthropic AI) noreply@anthropic.com

continue
}
if len(vv) == 0 {
Expand Down Expand Up @@ -1008,6 +1013,11 @@ func withHeaderMutation(resp *extprocv3.ProcessingResponse, pctx *pipeline.Conte
del = append(del, strings.ToLower(k)) // plugin removed it
}
}
// X-Cortex-Record is for Cortex only (#1923). ResolveClient removed it from pctx.Headers
// before orig was cloned, so the loop above cannot see it: remove it from the wire here.
if pctx.RecordHeaderSeen() {
del = append(del, strings.ToLower(model.RecordHeader))
}
if len(set) == 0 && len(del) == 0 {
return resp
}
Expand Down
10 changes: 10 additions & 0 deletions core/listener/forwardproxy/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,21 @@ func (s *Server) attributeForRecording(pctx *pipeline.Context, trusted string, c
at = s.attribute(clientHeaders, chain)
}
pctx.SetAttribution(at.Proc, at.ClientProc, at.Header)
pctx.SetNamedSession(namedSession(at))
pctx.SetProject(s.projectFor(trusted, m, chain, at))
sid := at.RecordSession(trusted, s.activeSession)
pctx.SetParent(s.parentFor(trusted, m, chain, at, sid))
return sid
}

// namedSession is the session id a session-id header named in at, "" when none did.
func namedSession(at attribution.Attribution) string {
if at.Header == "" {
return ""
}
return at.Session
}

// tunnelSessionID is recordingSessionID for a tunnel's own row. Under client affinity
// resolved is "" only for affinity's ambiguous answer or for no identity at all, and
// neither sends a tunnel row to default: a CONNECT rarely carries a User-Agent, so the
Expand Down Expand Up @@ -179,6 +188,7 @@ func (s *Server) hydrateSession(pctx *pipeline.Context, trusted string, clientHe
at = s.attribute(clientHeaders, chain)
}
pctx.SetAttribution(at.Proc, at.ClientProc, at.Header)
pctx.SetNamedSession(namedSession(at))
pctx.SetProject(s.projectFor(trusted, m, chain, at))
sid, source := at.PluginSession(trusted, s.activeSession)
pctx.SetParent(s.parentFor(trusted, m, chain, at, sid))
Expand Down
Loading
Loading