Repository navigation
Feat: Keep a session out of the archive when its requests carry X-Cortex-Record: off #1959
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
12 commits
Select commit
Hold shift + click to select a range
599f9b1
feat: Keep a session out of the archive when its requests carry X-Cor…
michal-jacovi 2a1a2e8
fix: Keep the header-derived session id out of the archive's queue-fu…
michal-jacovi e0dc212
fix: Let the archive's writer own the opt-out, and never drop a flagg…
michal-jacovi aba849c
fix: Never set X-Cortex-Record through ext_proc, and tighten the opt-…
michal-jacovi 4ee4630
fix: Deliver every flagged event and Withhold, below the reserve, and…
michal-jacovi 7091cf6
fix: Withhold an opaque CONNECT's own session, close the deliver/Clos…
michal-jacovi 72952a7
fix: Keep an opt-out at its place in the archive queue instead of wai…
michal-jacovi 89d558c
test: Cover a row queued while an opt-out is kept, and state the list…
michal-jacovi 7a7a0be
Merge origin/main into record-optout-pr
michal-jacovi 5bb0e5e
docs: Document X-Cortex-Record and noRecord, and report failedClosed
michal-jacovi 905cc55
Merge origin/main into record-optout-pr; guard the hop's opt-out
michal-jacovi 6ce74be
fix: List noRecord as a field the gateway never sets
michal-jacovi File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| package ledger | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| // A session that asked not to be recorded (SessionEvent.NoRecord, #1923) is kept out of the | ||
| // session archive only. Its spend still counts here: ledger rows carry no content, and the cost | ||
| // views must stay complete. | ||
| func TestWriter_CountsAWithheldSession(t *testing.T) { | ||
| dir := t.TempDir() | ||
| now := at | ||
| w := newTestWriter(t, dir, func() time.Time { return now }) | ||
|
|
||
| for i := 0; i < 2; i++ { | ||
| ev := costedEvent(t, "gw", "m", 0.25, 100, 50) | ||
| ev.NoRecord = true | ||
| w.Record("s1", ev) | ||
| } | ||
| now = at.Add(time.Minute) | ||
| later := costedEvent(t, "gw", "m", 0.10, 10, 5) | ||
| later.At, later.NoRecord = now, true | ||
| w.Record("s1", later) | ||
| if err := w.sync(); err != nil { | ||
| t.Fatalf("sync: %v", err) | ||
| } | ||
|
|
||
| rows := readAllRows(t, dir) | ||
| if len(rows) != 1 || rows[0].Requests != 2 || rows[0].CostMicros != 500_000 { | ||
| t.Fatalf("rows = %+v, want one closed minute of 2 requests and 500000 micros", rows) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| package extproc | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
| "slices" | ||
| "testing" | ||
| "time" | ||
|
|
||
| corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" | ||
| extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" | ||
|
|
||
| "github.com/rossoctl/cortex/core/listener/skiphost" | ||
| "github.com/rossoctl/cortex/core/model" | ||
| "github.com/rossoctl/cortex/core/pipeline" | ||
| "github.com/rossoctl/cortex/core/session" | ||
| ) | ||
|
|
||
| // removed returns the RemoveHeaders of a request-headers or request-body response. | ||
| func removed(t *testing.T, resp *extprocv3.ProcessingResponse) []string { | ||
| t.Helper() | ||
| var cr *extprocv3.CommonResponse | ||
| switch r := resp.Response.(type) { | ||
| case *extprocv3.ProcessingResponse_RequestHeaders: | ||
| cr = r.RequestHeaders.Response | ||
| case *extprocv3.ProcessingResponse_RequestBody: | ||
| cr = r.RequestBody.Response | ||
| } | ||
| if cr == nil || cr.HeaderMutation == nil { | ||
| return nil | ||
| } | ||
| return cr.HeaderMutation.RemoveHeaders | ||
| } | ||
|
|
||
| // A skip_hosts destination runs no pipeline, but the header is still for Cortex only: both the | ||
| // headers phase and the body phase remove it. | ||
| func TestNoRecord_ExtProcSkipHostRemovesTheHeader(t *testing.T) { | ||
| store := session.New(5*time.Minute, 100, 0) | ||
| defer store.Close() | ||
| skip, err := skiphost.New([]string{"otel-collector.*.svc.cluster.local"}) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| srv, _ := newSkipServer(t, store, skip) | ||
| hdrs := func() *corev3.HeaderMap { | ||
| return makeHeaders(":authority", "otel-collector.ns.svc.cluster.local:8335", ":path", "/v1/traces", | ||
| "x-cortex-record", "off") | ||
| } | ||
|
|
||
| stream := &mockStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{outboundRequest(hdrs())}} | ||
| _ = srv.Process(stream) | ||
| if len(stream.responses) != 1 { | ||
| t.Fatalf("headers phase: %d responses, want 1", len(stream.responses)) | ||
| } | ||
| if got := removed(t, stream.responses[0]); !slices.Contains(got, "x-cortex-record") { | ||
| t.Fatalf("headers phase: RemoveHeaders %v, want x-cortex-record removed", got) | ||
| } | ||
|
|
||
| resp, pctx := srv.handleOutboundBody(&mockStream{ctx: context.Background()}, hdrs(), []byte(`{}`)) | ||
| if pctx != nil { | ||
| t.Fatal("body phase ran the pipeline for a skipped host") | ||
| } | ||
| if got := removed(t, resp); !slices.Contains(got, "x-cortex-record") { | ||
| t.Fatalf("body phase: RemoveHeaders %v, want x-cortex-record removed", got) | ||
| } | ||
| } | ||
|
|
||
| // ext_proc forwards the wire headers and only the changes it emits. ResolveClient removed | ||
| // X-Cortex-Record from pctx.Headers before the listener cloned them, so the diff alone cannot | ||
| // see it: withHeaderMutation must remove it explicitly, whatever its value. | ||
| func TestNoRecord_ExtProcRemovesTheHeader(t *testing.T) { | ||
| for _, v := range []string{"off", "on"} { | ||
| h := http.Header{} | ||
| h.Set(model.RecordHeader, v) | ||
| pctx := &pipeline.Context{Headers: h} | ||
| pctx.ResolveClient() | ||
| orig := pctx.Headers.Clone() // as handleInbound and handleOutbound do | ||
| got := removed(t, withHeaderMutation(allowResponse(), pctx, orig)) | ||
| if !slices.Contains(got, "x-cortex-record") { | ||
| t.Fatalf("value %q: RemoveHeaders = %v, want x-cortex-record", v, got) | ||
| } | ||
| } | ||
| // No header: nothing extra is removed. | ||
| pctx := &pipeline.Context{Headers: http.Header{"User-Agent": {"curl/8"}}} | ||
| pctx.ResolveClient() | ||
| if got := removed(t, withHeaderMutation(allowResponse(), pctx, pctx.Headers.Clone())); slices.Contains(got, "x-cortex-record") { | ||
| t.Fatalf("RemoveHeaders = %v with no header sent", got) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit — this path defends against a plugin writing
X-Cortex-Recordback ("also when a plugin wrote it back"), buthttpx.SyncRequestHeaders(core/listener/httpx/request.go:34) skips onlyContent-Length/Content-Encoding— so the same plugin write-back on the forward/reverse proxy paths would forward the header upstream. The samestrings.EqualFold(k, model.RecordHeader)in that skip list would make the two listeners agree. Happy to see it in a follow-up; it is outside this PR's diff.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks, agreed. Outside this PR's diff, so it's a follow-up: #2005 (skip
X-Cortex-Recordinhttpx.SyncRequestHeaders, as ext_proc does).Assisted-By: Claude (Anthropic AI) noreply@anthropic.com