From 5b1dc3bd753d3d4ddc29234e538aca62ffd374ef Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:50:27 +0530 Subject: [PATCH] Size active queried series worker pool from GOMAXPROCS The active queried series service sized its worker pool with runtime.NumCPU(), which returns the host's logical CPU count and ignores the CPU cgroup quota. Cortex imports go.uber.org/automaxprocs so that GOMAXPROCS matches the container's CPU limit, and reads container-aware parallelism via runtime.GOMAXPROCS(0) elsewhere (for example the resource monitor's CPU limit and the engine's decoding concurrency default). This one call site was inconsistent: in a CPU-limited container on a many-core host it sized the pool off the host cores instead of the CPU budget. Use runtime.GOMAXPROCS(0) so the pool follows the container CPU quota, matching the rest of the codebase. The min(..., 4) cap is unchanged. Add a regression test asserting the worker count follows GOMAXPROCS. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- CHANGELOG.md | 1 + pkg/ingester/active_queried_series.go | 6 +++-- pkg/ingester/active_queried_series_test.go | 28 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd8bcc7927..a8f5cca9e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ * [BUGFIX] Compactor: Fix spurious `bucket operation fail after retries` error logs emitted during partial block cleanup. #7749 * [BUGFIX] Alertmanager: Fix panic in `validateAlertmanagerConfig` when receiver config traversal encounters nil interface values. #7751 * [BUGFIX] Parquet Converter: Fix `auto_forget_delay` having no effect. The ring lifecycler was created without the auto-forget delegate, so unhealthy instances were never automatically removed from the ring. #7752 +* [BUGFIX] Ingester: Size the active queried series worker pool from `GOMAXPROCS` instead of `runtime.NumCPU()`, so it respects the container CPU quota (set via automaxprocs) rather than the host core count. #7671 ## 1.21.1 2026-06-04 diff --git a/pkg/ingester/active_queried_series.go b/pkg/ingester/active_queried_series.go index 96134a05151..32685d0f911 100644 --- a/pkg/ingester/active_queried_series.go +++ b/pkg/ingester/active_queried_series.go @@ -385,8 +385,10 @@ type ActiveQueriedSeriesService struct { // NewActiveQueriedSeriesService creates a new ActiveQueriedSeriesService service. func NewActiveQueriedSeriesService(logger log.Logger, registerer prometheus.Registerer) *ActiveQueriedSeriesService { - // Cap at 4 workers to avoid excessive goroutines - numWorkers := max(min(runtime.NumCPU()/2, 4), 1) + // Cap at 4 workers to avoid excessive goroutines. Use GOMAXPROCS (which + // automaxprocs sets from the CPU cgroup quota) rather than NumCPU, so the + // pool is sized to the container's CPU budget instead of the host cores. + numWorkers := max(min(runtime.GOMAXPROCS(0)/2, 4), 1) m := &ActiveQueriedSeriesService{ updateChan: make(chan activeQueriedSeriesUpdate, 10000), // Buffered channel to avoid blocking diff --git a/pkg/ingester/active_queried_series_test.go b/pkg/ingester/active_queried_series_test.go index 9fdbd27c4f4..c0360391863 100644 --- a/pkg/ingester/active_queried_series_test.go +++ b/pkg/ingester/active_queried_series_test.go @@ -3,6 +3,7 @@ package ingester import ( "context" "fmt" + "runtime" "sync" "testing" "time" @@ -546,3 +547,30 @@ func TestActiveQueriedSeriesService_NoSendOnClosedChannelOnShutdown(t *testing.T t.Fatalf("producer goroutine panicked during shutdown: %v", panicMsg.Load()) } } + +func TestActiveQueriedSeriesService_NumWorkersFollowsGOMAXPROCS(t *testing.T) { + // The worker pool must be sized from GOMAXPROCS (which automaxprocs derives + // from the CPU cgroup quota) and not from runtime.NumCPU (host cores, which + // ignores cgroup limits). Save and restore GOMAXPROCS around the test. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + + for _, tc := range []struct { + gomaxprocs int + expectedWorkers int + }{ + // GOMAXPROCS(1)/2 == 0, floored to the minimum of 1 worker. Against the + // buggy runtime.NumCPU() this yields min(NumCPU/2, 4) on a multi-core + // host and fails. + {gomaxprocs: 1, expectedWorkers: 1}, + {gomaxprocs: 2, expectedWorkers: 1}, + {gomaxprocs: 4, expectedWorkers: 2}, + // Capped at 4 workers regardless of how many CPUs are available. + {gomaxprocs: 16, expectedWorkers: 4}, + } { + t.Run(fmt.Sprintf("gomaxprocs=%d", tc.gomaxprocs), func(t *testing.T) { + runtime.GOMAXPROCS(tc.gomaxprocs) + svc := NewActiveQueriedSeriesService(log.NewNopLogger(), nil) + assert.Equal(t, tc.expectedWorkers, svc.numWorkers) + }) + } +}