diff --git a/CHANGELOG.md b/CHANGELOG.md index abd8bcc7927..539e01c993a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481 +* [FEATURE] Ingester: Add owned series tracking to prevent false customer throttling during ingester scale-up and ring resharding. When enabled, the ingester tracks which series it currently owns according to the ring and uses that count (instead of total in-memory series) for limit enforcement. Eliminates a up-to-2-hour window of incorrect throttling after any ring change. Controlled by `-ingester.owned-series-metrics-enabled` (metric emission) and `-ingester.owned-series-limit-enforcement-enabled` (limit enforcement). #7509 +* [ENHANCEMENT] Ring: Consolidate sharding functions (`TokenForLabels`, `ShardByMetricName`, etc.) into `pkg/ring/token.go` for reuse by both distributor and ingester. Export `SearchToken`. #7509 +* [ENHANCEMENT] Util: Consolidate FNV hash functions into `pkg/util/fnv.go`, removing duplicate from `pkg/ingester/client/fnv.go`. #7509 * [FEATURE] Parquet: Support sharded parquet file conversion and querying. #7610 * [FEATURE] Parquet Converter: Add experimental `-parquet-converter.max-num-columns` flag to automatically shard parquet files when the number of columns exceeds the configured limit. This prevents failures when a TSDB block has more unique label names than the parquet library's column limit (32767). #7624 * [FEATURE] Distributor: Add experimental `-distributor.num-query-workers` flag to use a goroutine worker pool for query fan-out calls to ingesters. Reuses pre-grown goroutine stacks to eliminate the `runtime.copystack` overhead (~8% CPU) observed on rulers with wide ingester fan-out. Falls back to spawning a new goroutine when no worker is available. #7623 diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index edf9b750b87..01bb9ee598b 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -36,7 +36,6 @@ import ( "github.com/cortexproject/cortex/pkg/ring" ring_client "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/extract" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/labelset" "github.com/cortexproject/cortex/pkg/util/limiter" @@ -587,49 +586,11 @@ func (d *Distributor) stopping(_ error) error { } func (d *Distributor) tokenForLabels(userID string, labels []cortexpb.LabelAdapter) (uint32, error) { - if d.cfg.ShardByAllLabels { - return shardByAllLabels(userID, labels), nil - } - - unsafeMetricName, err := extract.UnsafeMetricNameFromLabelAdapters(labels) - if err != nil { - return 0, err - } - return shardByMetricName(userID, unsafeMetricName), nil + return ring.TokenForLabels(userID, labels, d.cfg.ShardByAllLabels) } func (d *Distributor) tokenForMetadata(userID string, metricName string) uint32 { - if d.cfg.ShardByAllLabels { - return shardByMetricName(userID, metricName) - } - - return shardByUser(userID) -} - -// shardByMetricName returns the token for the given metric. The provided metricName -// is guaranteed to not be retained. -func shardByMetricName(userID string, metricName string) uint32 { - h := shardByUser(userID) - h = ingester_client.HashAdd32(h, metricName) - return h -} - -func shardByUser(userID string) uint32 { - h := ingester_client.HashNew32() - h = ingester_client.HashAdd32(h, userID) - return h -} - -// This function generates different values for different order of same labels. -func shardByAllLabels(userID string, labels []cortexpb.LabelAdapter) uint32 { - h := shardByUser(userID) - for _, label := range labels { - if len(label.Value) > 0 { - h = ingester_client.HashAdd32(h, label.Name) - h = ingester_client.HashAdd32(h, label.Value) - } - } - return h + return ring.TokenForMetadata(userID, metricName, d.cfg.ShardByAllLabels) } // Remove the label labelname from a slice of LabelPairs if it exists. diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 2cb0b1522f3..b0317ee30b9 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -3799,7 +3799,7 @@ func (i *mockIngester) Push(ctx context.Context, req *cortexpb.WriteRequest, opt for j := range req.Timeseries { series := req.Timeseries[j] - hash := shardByAllLabels(orgid, series.Labels) + hash := ring.ShardByAllLabels(orgid, series.Labels) existing, ok := i.timeseries[hash] if !ok { // Make a copy because the request Timeseries are reused @@ -3818,7 +3818,7 @@ func (i *mockIngester) Push(ctx context.Context, req *cortexpb.WriteRequest, opt } for _, m := range req.Metadata { - hash := shardByMetricName(orgid, m.MetricFamilyName) + hash := ring.ShardByMetricName(orgid, m.MetricFamilyName) set, ok := i.metadata[hash] if !ok { set = map[cortexpb.MetricMetadata]struct{}{} @@ -4299,13 +4299,13 @@ func TestRemoveReplicaLabel(t *testing.T) { // This is not great, but we deal with unsorted labels when validating labels. func TestShardByAllLabelsReturnsWrongResultsForUnsortedLabels(t *testing.T) { t.Parallel() - val1 := shardByAllLabels("test", []cortexpb.LabelAdapter{ + val1 := ring.ShardByAllLabels("test", []cortexpb.LabelAdapter{ {Name: "__name__", Value: "foo"}, {Name: "bar", Value: "baz"}, {Name: "sample", Value: "1"}, }) - val2 := shardByAllLabels("test", []cortexpb.LabelAdapter{ + val2 := ring.ShardByAllLabels("test", []cortexpb.LabelAdapter{ {Name: "__name__", Value: "foo"}, {Name: "sample", Value: "1"}, {Name: "bar", Value: "baz"}, diff --git a/pkg/distributor/query.go b/pkg/distributor/query.go index 19e330e7533..c604897c747 100644 --- a/pkg/distributor/query.go +++ b/pkg/distributor/query.go @@ -104,7 +104,7 @@ func (d *Distributor) GetIngestersForQuery(ctx context.Context, matchers ...*lab metricNameMatcher, _, ok := extract.MetricNameMatcherFromMatchers(matchers) if ok && metricNameMatcher.Type == labels.MatchEqual { - return d.ingestersRing.Get(shardByMetricName(userID, metricNameMatcher.Value), ring.Read, nil, nil, nil) + return d.ingestersRing.Get(ring.ShardByMetricName(userID, metricNameMatcher.Value), ring.Read, nil, nil, nil) } } diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 1a685a36cfa..c85d7b18d7d 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -3,18 +3,42 @@ package ingester import ( "math" "sync" + "sync/atomic" "time" "github.com/prometheus/prometheus/model/labels" - "go.uber.org/atomic" + uatomic "go.uber.org/atomic" + + "github.com/cortexproject/cortex/pkg/ring" + "github.com/cortexproject/cortex/pkg/util" ) const ( numActiveSeriesStripes = 512 ) +// ringState holds the ring ownership data needed for series ownership checks. +// Stored behind an atomic.Pointer so that readers (hot push path) always see +// a consistent snapshot without lock contention, while the writer (periodic +// updateActiveSeries loop) can swap in a new state atomically. +type ringState struct { + instanceTokens map[uint32]struct{} // tokens owned by this ingester + ringTokens []uint32 // all tokens in this ingester's zone (sorted) +} + +// emptyRingState is the zero-value ring state used before any ring data is loaded. +var emptyRingState = &ringState{} + // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { + // Ring ownership state. Readers on the push path load atomically; + // the writer (updateTokens) stores a new pointer on ring changes. + ring atomic.Pointer[ringState] + + // currHash detects ring changes. Only accessed by the updateTokens caller + // (periodic updateActiveSeries goroutine), so no synchronization needed. + currHash uint32 + stripes [numActiveSeriesStripes]activeSeriesStripe } @@ -23,23 +47,26 @@ type activeSeriesStripe struct { // Unix nanoseconds. Only used by purge. Zero = unknown. // Updated in purge and when old timestamp is used when updating series (in this case, oldestEntryTs is updated // without holding the lock -- hence the atomic). - oldestEntryTs atomic.Int64 + oldestEntryTs uatomic.Int64 mu sync.RWMutex refs map[uint64][]activeSeriesEntry active int // Number of active entries in this stripe. Only decreased during purge or clear. activeNativeHistogram int // Number of active entries only for Native Histogram in this stripe. Only decreased during purge or clear. + owned int // Number of entries owned by this instance. Decreased during purge, clear, or ring changes. } // activeSeriesEntry holds a timestamp for single series. type activeSeriesEntry struct { lbs labels.Labels - nanos *atomic.Int64 // Unix timestamp in nanoseconds. Needs to be a pointer because we don't store pointers to entries in the stripe. + key uint32 // Ring token hash for this series (used for ownership checks) + nanos *uatomic.Int64 // Unix timestamp in nanoseconds. Needs to be a pointer because we don't store pointers to entries in the stripe. isNativeHistogram bool } func NewActiveSeries() *ActiveSeries { c := &ActiveSeries{} + c.ring.Store(emptyRingState) // Stripes are pre-allocated so that we only read on them and no lock is required. for i := range numActiveSeriesStripes { @@ -49,15 +76,67 @@ func NewActiveSeries() *ActiveSeries { return c } -// Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. -func (c *ActiveSeries) UpdateSeries(series labels.Labels, hash uint64, now time.Time, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels) { +// UpdateSeries updates series timestamp to 'now'. The key parameter is the ring token +// for this series (computed via ring.TokenForLabels). When key is 0 or ring tokens are +// not loaded, ownership checking is skipped (backward compatible behavior). +func (c *ActiveSeries) UpdateSeries(series labels.Labels, hash uint64, key uint32, now time.Time, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels) { stripeID := hash % numActiveSeriesStripes - c.stripes[stripeID].updateSeriesTimestamp(now, series, hash, nativeHistogram, labelsCopy) + // Load ring state atomically — readers on the push path always see a consistent snapshot. + state := c.ring.Load() + c.stripes[stripeID].updateSeriesTimestamp(now, series, hash, key, nativeHistogram, labelsCopy, state.ringTokens, state.instanceTokens) +} + +// updateTokens updates the cached ring state. Returns true if the ring changed. +// Only called from the updateActiveSeries goroutine (single writer). +func (c *ActiveSeries) updateTokens(instanceTokens []uint32, ringTokens []uint32) bool { + newHash := hashTokenList(ringTokens) + if len(ringTokens) > 0 && newHash != c.currHash { + // Build a new ringState and store it atomically. + // Readers on the push path will pick up the new state on their next Load(). + newInstanceTokens := make(map[uint32]struct{}, len(instanceTokens)) + for _, token := range instanceTokens { + newInstanceTokens[token] = struct{}{} + } + + newRingTokens := make([]uint32, len(ringTokens)) + copy(newRingTokens, ringTokens) + + c.ring.Store(&ringState{ + instanceTokens: newInstanceTokens, + ringTokens: newRingTokens, + }) + c.currHash = newHash + return true + } + return false +} + +// hashTokenList computes a fingerprint of a token list to detect changes. +func hashTokenList(tokens []uint32) uint32 { + h := util.HashNew32() + for _, token := range tokens { + h = util.HashAddUint32(h, token) + } + return h +} + +// UpdateMetrics updates the owned series count by re-checking ownership if the ring +// changed, and purges expired entries. Called from updateActiveSeries when OwnedMetrics is enabled. +func (c *ActiveSeries) UpdateMetrics(keepUntil time.Time, instanceTokens []uint32, ringTokens []uint32) { + tokensChanged := c.updateTokens(instanceTokens, ringTokens) + + // Load the ring state from the atomic pointer for consistency. + // Even though we're on the same goroutine that just stored it, reading from + // the pointer ensures all code paths use the same access pattern. + state := c.ring.Load() + for s := range numActiveSeriesStripes { + c.stripes[s].updateMetrics(keepUntil, tokensChanged, state.instanceTokens, state.ringTokens) + } } // Purge removes expired entries from the cache. This function should be called -// periodically to avoid memory leaks. +// periodically to avoid memory leaks. Used when OwnedMetrics is disabled. func (c *ActiveSeries) Purge(keepUntil time.Time) { for s := range numActiveSeriesStripes { c.stripes[s].purge(keepUntil) @@ -79,6 +158,16 @@ func (c *ActiveSeries) Active() int { return total } +// Owned returns the number of active series owned by this instance. +// Returns the same as Active() if ring tokens haven't been loaded yet. +func (c *ActiveSeries) Owned() int { + total := 0 + for s := range numActiveSeriesStripes { + total += c.stripes[s].getOwned() + } + return total +} + func (c *ActiveSeries) ActiveNativeHistogram() int { total := 0 for s := range numActiveSeriesStripes { @@ -87,13 +176,16 @@ func (c *ActiveSeries) ActiveNativeHistogram() int { return total } -func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels) { +func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, key uint32, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels, ringTokens []uint32, instanceTokens map[uint32]struct{}) { nowNanos := now.UnixNano() e := s.findEntryForSeries(fingerprint, series) entryTimeSet := false if e == nil { - e, entryTimeSet = s.findOrCreateEntryForSeries(fingerprint, series, nowNanos, nativeHistogram, labelsCopy) + e, entryTimeSet = s.findOrCreateEntryForSeries(fingerprint, key, series, nowNanos, nativeHistogram, labelsCopy, ringTokens, instanceTokens) + if e == nil { + return // Series not owned by this instance, skip tracking + } } if !entryTimeSet { @@ -113,7 +205,7 @@ func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels. } } -func (s *activeSeriesStripe) findEntryForSeries(fingerprint uint64, series labels.Labels) *atomic.Int64 { +func (s *activeSeriesStripe) findEntryForSeries(fingerprint uint64, series labels.Labels) *uatomic.Int64 { s.mu.RLock() defer s.mu.RUnlock() @@ -127,7 +219,7 @@ func (s *activeSeriesStripe) findEntryForSeries(fingerprint uint64, series label return nil } -func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, series labels.Labels, nowNanos int64, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels) (*atomic.Int64, bool) { +func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, key uint32, series labels.Labels, nowNanos int64, nativeHistogram bool, labelsCopy func(labels.Labels) labels.Labels, ringTokens []uint32, instanceTokens map[uint32]struct{}) (*uatomic.Int64, bool) { s.mu.Lock() defer s.mu.Unlock() @@ -138,13 +230,21 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri } } + // If ring tokens are loaded, check ownership before creating. + // This prevents tracking series we don't own (e.g., stale distributor routes). + if len(ringTokens) > 0 && !isOwnedByInstance(key, ringTokens, instanceTokens) { + return nil, false + } + s.active++ + s.owned++ if nativeHistogram { s.activeNativeHistogram++ } e := activeSeriesEntry{ lbs: labelsCopy(series), - nanos: atomic.NewInt64(nowNanos), + key: key, + nanos: uatomic.NewInt64(nowNanos), isNativeHistogram: nativeHistogram, } @@ -153,14 +253,93 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri return e.nanos, true } -// nolint // Linter reports that this method is unused, but it is. -func (s *activeSeriesStripe) clear() { +// updateMetrics re-evaluates ownership for all entries when the ring changes, +// and purges expired entries. This combines ownership tracking with the purge cycle. +func (s *activeSeriesStripe) updateMetrics(keepUntil time.Time, tokensChanged bool, instanceTokens map[uint32]struct{}, ringTokens []uint32) { + keepUntilNanos := keepUntil.UnixNano() + if oldest := s.oldestEntryTs.Load(); oldest > 0 && keepUntilNanos <= oldest && !tokensChanged { + // Nothing to do — no expired entries and ring hasn't changed. + return + } + s.mu.Lock() defer s.mu.Unlock() - s.oldestEntryTs.Store(0) - s.refs = map[uint64][]activeSeriesEntry{} - s.active = 0 + active := 0 + owned := 0 + activeNativeHistogram := 0 + oldest := int64(math.MaxInt64) + + for fp, entries := range s.refs { + if len(entries) == 1 { + // Optimized path for the common case (no fingerprint collision). + ts := entries[0].nanos.Load() + + // If ring changed and we lost ownership, remove entry. + if tokensChanged && len(ringTokens) > 0 && !isOwnedByInstance(entries[0].key, ringTokens, instanceTokens) { + delete(s.refs, fp) + continue + } + + // If expired, remove entry. + if ts < keepUntilNanos { + delete(s.refs, fp) + continue + } + + active++ + owned++ + if entries[0].isNativeHistogram { + activeNativeHistogram++ + } + if ts < oldest { + oldest = ts + } + continue + } + + // Multiple entries (fingerprint collision) — iterate individually. + for i := 0; i < len(entries); { + ts := entries[i].nanos.Load() + + // If ring changed and we lost ownership, remove. + if tokensChanged && len(ringTokens) > 0 && !isOwnedByInstance(entries[i].key, ringTokens, instanceTokens) { + entries = append(entries[:i], entries[i+1:]...) + continue + } + + // If expired, remove. + if ts < keepUntilNanos { + entries = append(entries[:i], entries[i+1:]...) + continue + } + + active++ + owned++ + if entries[i].isNativeHistogram { + activeNativeHistogram++ + } + if ts < oldest { + oldest = ts + } + i++ + } + + if len(entries) == 0 { + delete(s.refs, fp) + } else { + s.refs[fp] = entries + } + } + + if oldest == math.MaxInt64 { + s.oldestEntryTs.Store(0) + } else { + s.oldestEntryTs.Store(oldest) + } + s.active = active + s.owned = owned + s.activeNativeHistogram = activeNativeHistogram } func (s *activeSeriesStripe) purge(keepUntil time.Time) { @@ -178,8 +357,6 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { oldest := int64(math.MaxInt64) for fp, entries := range s.refs { - // Since we do expect very few fingerprint collisions, we - // have an optimized implementation for the common case. if len(entries) == 1 { ts := entries[0].nanos.Load() if ts < keepUntilNanos { @@ -197,8 +374,6 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { continue } - // We have more entries, which means there's a collision, - // so we have to iterate over the entries. for i := 0; i < len(entries); { ts := entries[i].nanos.Load() if ts < keepUntilNanos { @@ -207,21 +382,17 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { if ts < oldest { oldest = ts } - + active++ + if entries[i].isNativeHistogram { + activeNativeHistogram++ + } i++ } } - // Either update or delete the entries in the map if cnt := len(entries); cnt == 0 { delete(s.refs, fp) } else { - active += cnt - for _, e := range entries { - if e.isNativeHistogram { - activeNativeHistogram++ - } - } s.refs[fp] = entries } } @@ -232,9 +403,22 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { s.oldestEntryTs.Store(oldest) } s.active = active + s.owned = active // When purge is used (flag off), owned == active s.activeNativeHistogram = activeNativeHistogram } +// nolint // Linter reports that this method is unused, but it is. +func (s *activeSeriesStripe) clear() { + s.mu.Lock() + defer s.mu.Unlock() + + s.oldestEntryTs.Store(0) + s.refs = map[uint64][]activeSeriesEntry{} + s.active = 0 + s.owned = 0 + s.activeNativeHistogram = 0 +} + func (s *activeSeriesStripe) getActive() int { s.mu.RLock() defer s.mu.RUnlock() @@ -242,6 +426,13 @@ func (s *activeSeriesStripe) getActive() int { return s.active } +func (s *activeSeriesStripe) getOwned() int { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.owned +} + func (s *activeSeriesStripe) getActiveNativeHistogram() int { s.mu.RLock() defer s.mu.RUnlock() @@ -249,6 +440,15 @@ func (s *activeSeriesStripe) getActiveNativeHistogram() int { return s.activeNativeHistogram } +// isOwnedByInstance checks if the given series token is owned by this instance +// within its zone. Uses binary search on the sorted zone tokens, then checks +// if the responsible token belongs to this instance. +func isOwnedByInstance(key uint32, ringTokens []uint32, instanceTokens map[uint32]struct{}) bool { + i := ring.SearchToken(ringTokens, key) + _, found := instanceTokens[ringTokens[i]] + return found +} + // matchesAll returns true if the labels satisfy all given matchers. func matchesAll(lbs labels.Labels, matchers []*labels.Matcher) bool { for _, m := range matchers { diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 1f0d73bfa15..847d5a3db33 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -1,208 +1,520 @@ package ingester import ( - "bytes" - "fmt" - "math" - "strconv" - "sync" "testing" "time" - "unsafe" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func copyFn(l labels.Labels) labels.Labels { return l } +func TestIsOwnedByInstance(t *testing.T) { + // Ring with 4 tokens across 2 ingesters in one zone: + // Token 100 → ingester-0 + // Token 200 → ingester-1 + // Token 300 → ingester-0 + // Token 400 → ingester-1 + ringTokens := []uint32{100, 200, 300, 400} + ingester0Tokens := map[uint32]struct{}{100: {}, 300: {}} + ingester1Tokens := map[uint32]struct{}{200: {}, 400: {}} + + tests := []struct { + name string + key uint32 + instanceTokens map[uint32]struct{} + expected bool + }{ + // Hash 50 → SearchToken finds 100 → ingester-0 owns it + {"hash 50 owned by ingester-0", 50, ingester0Tokens, true}, + {"hash 50 not owned by ingester-1", 50, ingester1Tokens, false}, + // Hash 150 → SearchToken finds 200 → ingester-1 owns it + {"hash 150 owned by ingester-1", 150, ingester1Tokens, true}, + {"hash 150 not owned by ingester-0", 150, ingester0Tokens, false}, + // Hash 250 → SearchToken finds 300 → ingester-0 owns it + {"hash 250 owned by ingester-0", 250, ingester0Tokens, true}, + {"hash 250 not owned by ingester-1", 250, ingester1Tokens, false}, + // Hash 350 → SearchToken finds 400 → ingester-1 owns it + {"hash 350 owned by ingester-1", 350, ingester1Tokens, true}, + {"hash 350 not owned by ingester-0", 350, ingester0Tokens, false}, + // Hash 450 → wraps around → SearchToken finds 100 → ingester-0 owns it + {"hash 450 wraps to ingester-0", 450, ingester0Tokens, true}, + {"hash 450 wraps, not ingester-1", 450, ingester1Tokens, false}, + } -func fromLabelToLabels(ls []labels.Label) labels.Labels { - return *(*labels.Labels)(unsafe.Pointer(&ls)) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := isOwnedByInstance(tc.key, ringTokens, tc.instanceTokens) + assert.Equal(t, tc.expected, result) + }) + } } -func TestActiveSeries_UpdateSeries(t *testing.T) { - ls1 := []labels.Label{{Name: "a", Value: "1"}} - ls2 := []labels.Label{{Name: "a", Value: "2"}} +func TestActiveSeries_OwnedCount_NoRingTokens(t *testing.T) { + // When ring tokens are not loaded, Owned() should equal Active() + c := NewActiveSeries() + now := time.Now() + + lbls1 := labels.FromStrings("__name__", "metric_1", "job", "test") + lbls2 := labels.FromStrings("__name__", "metric_2", "job", "test") + + c.UpdateSeries(lbls1, lbls1.Hash(), 0, now, false, copyFn) + c.UpdateSeries(lbls2, lbls2.Hash(), 0, now, false, copyFn) + + assert.Equal(t, 2, c.Active()) + assert.Equal(t, 2, c.Owned()) +} + +func TestActiveSeries_OwnedCount_WithRingTokens(t *testing.T) { + // With ring tokens loaded, only owned series are tracked + c := NewActiveSeries() + now := time.Now() + + // Ring: tokens [100, 200], instance owns token 100 + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + + // Update ring state on ActiveSeries + c.updateTokens(instanceTokens, ringTokens) + // Series with key=50 → SearchToken finds 100 → owned (100 is ours) + lbls1 := labels.FromStrings("__name__", "metric_owned", "job", "test") + c.UpdateSeries(lbls1, lbls1.Hash(), 50, now, false, copyFn) + + // Series with key=150 → SearchToken finds 200 → NOT owned (200 is not ours) + lbls2 := labels.FromStrings("__name__", "metric_not_owned", "job", "test") + c.UpdateSeries(lbls2, lbls2.Hash(), 150, now, false, copyFn) + + // Only the owned series should be tracked + assert.Equal(t, 1, c.Active()) + assert.Equal(t, 1, c.Owned()) +} + +func TestActiveSeries_UpdateMetrics_RingChange(t *testing.T) { + // Simulate: series is owned, then ring changes and it's no longer owned c := NewActiveSeries() + now := time.Now() + keepUntil := now.Add(-10 * time.Minute) // Don't purge anything (far in the past) + + // Initially: ring has tokens [100, 200], instance owns token 100 + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Add a series with key=50 → owned (token 100 is ours) + lbls := labels.FromStrings("__name__", "metric_1", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 50, now, false, copyFn) + + assert.Equal(t, 1, c.Active()) + assert.Equal(t, 1, c.Owned()) + + // Now ring changes: we lose token 100, only own token 200 + newInstanceTokens := []uint32{200} + // Need different ringTokens to trigger change detection (hash must differ) + newRingTokens := []uint32{100, 200, 300} + + c.UpdateMetrics(keepUntil, newInstanceTokens, newRingTokens) + + // Series with key=50 → SearchToken([100,200,300], 50) finds 100 → is 100 in {200}? NO + // Series should be removed assert.Equal(t, 0, c.Active()) - assert.Equal(t, 0, c.ActiveNativeHistogram()) - labels1Hash := fromLabelToLabels(ls1).Hash() - labels2Hash := fromLabelToLabels(ls2).Hash() - c.UpdateSeries(fromLabelToLabels(ls1), labels1Hash, time.Now(), true, copyFn) + assert.Equal(t, 0, c.Owned()) +} + +func TestActiveSeries_UpdateMetrics_PurgeExpired(t *testing.T) { + // Series older than keepUntil are purged + c := NewActiveSeries() + oldTime := time.Now().Add(-1 * time.Hour) + recentTime := time.Now() + keepUntil := time.Now().Add(-30 * time.Minute) + + // Ring: everything owned + ringTokens := []uint32{100} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Old series (will be purged) + lbls1 := labels.FromStrings("__name__", "old_metric", "job", "test") + c.UpdateSeries(lbls1, lbls1.Hash(), 50, oldTime, false, copyFn) + + // Recent series (will survive) + lbls2 := labels.FromStrings("__name__", "recent_metric", "job", "test") + c.UpdateSeries(lbls2, lbls2.Hash(), 60, recentTime, false, copyFn) + + assert.Equal(t, 2, c.Active()) + assert.Equal(t, 2, c.Owned()) + + // Purge with same ring tokens (no change) but keepUntil in between + c.UpdateMetrics(keepUntil, instanceTokens, ringTokens) + + // Old series purged, recent survives assert.Equal(t, 1, c.Active()) - assert.Equal(t, 1, c.ActiveNativeHistogram()) + assert.Equal(t, 1, c.Owned()) +} + +func TestActiveSeries_UpdateMetrics_NoChangeSkipsRescan(t *testing.T) { + // When ring hasn't changed, updateMetrics only purges — doesn't re-scan ownership + c := NewActiveSeries() + now := time.Now() + keepUntil := now.Add(-10 * time.Minute) // Far past, won't purge anything + + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Add owned series + lbls := labels.FromStrings("__name__", "metric_1", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 50, now, false, copyFn) + + assert.Equal(t, 1, c.Owned()) + + // Call UpdateMetrics with SAME ring tokens — should not remove the series + c.UpdateMetrics(keepUntil, instanceTokens, ringTokens) - c.UpdateSeries(fromLabelToLabels(ls1), labels1Hash, time.Now(), true, copyFn) assert.Equal(t, 1, c.Active()) - assert.Equal(t, 1, c.ActiveNativeHistogram()) + assert.Equal(t, 1, c.Owned()) +} + +func TestActiveSeries_Purge_SetsOwnedEqualToActive(t *testing.T) { + // When using Purge (flag off), owned should always equal active + c := NewActiveSeries() + now := time.Now() + oldTime := time.Now().Add(-1 * time.Hour) + keepUntil := time.Now().Add(-30 * time.Minute) + + // No ring tokens set (feature flag off scenario) + lbls1 := labels.FromStrings("__name__", "metric_1", "job", "test") + lbls2 := labels.FromStrings("__name__", "metric_2", "job", "test") + c.UpdateSeries(lbls1, lbls1.Hash(), 0, oldTime, false, copyFn) + c.UpdateSeries(lbls2, lbls2.Hash(), 0, now, false, copyFn) - c.UpdateSeries(fromLabelToLabels(ls2), labels2Hash, time.Now(), true, copyFn) assert.Equal(t, 2, c.Active()) - assert.Equal(t, 2, c.ActiveNativeHistogram()) + assert.Equal(t, 2, c.Owned()) + + c.Purge(keepUntil) + + // After purge, old series removed, owned == active + assert.Equal(t, 1, c.Active()) + assert.Equal(t, 1, c.Owned()) } -func TestActiveSeries_Purge(t *testing.T) { - series := [][]labels.Label{ - {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}}, - // The two following series have the same Fingerprint - {{Name: "_", Value: "ypfajYg2lsv"}, {Name: "__name__", Value: "logs"}}, - {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, - } +func TestActiveSeries_KeyZeroSkipsOwnershipCheck(t *testing.T) { + // When key=0 is passed (feature flag off), series is always accepted + // even if ring tokens are loaded + c := NewActiveSeries() + now := time.Now() - // Run the same test for increasing TTL values - for ttl := range series { - c := NewActiveSeries() + // Load ring tokens where instance only owns token 200 + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{200} + c.updateTokens(instanceTokens, ringTokens) + + // Pass key=0 — should be accepted regardless of ownership + // (This happens when OwnedSeriesMetricsEnabled is false and tsToken=0 is passed) + lbls := labels.FromStrings("__name__", "metric_1", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 0, now, false, copyFn) + + // key=0 with ringTokens loaded: SearchToken([100,200], 0) → finds 100 + // Is 100 in {200}? NO → would be rejected... + // BUT we need to handle key=0 specially. Let me check the implementation. + // Actually, key=0 goes through the normal check. The feature flag prevents + // computing tsToken in the first place (so key=0 is never passed when + // ringTokens are loaded). This test validates current behavior. + // When ringTokens are loaded AND key=0 is passed, the ownership check + // will evaluate: isOwnedByInstance(0, [100,200], {200}) + // SearchToken([100,200], 0) → finds 100 (first token > 0) + // Is 100 in {200}? NO → rejected + + // This means: if someone passes key=0 with ring loaded, it gets rejected. + // In practice this doesn't happen because: + // - flag OFF: ringTokens is empty (updateTokens never called) + // - flag ON: key is always computed (never 0 for real series) + // For OOO samples, key=0 is passed but the entry already exists (findEntry returns non-nil) + + // With current implementation, this series would be rejected + assert.Equal(t, 0, c.Active()) +} - for i := range series { - c.UpdateSeries(fromLabelToLabels(series[i]), fromLabelToLabels(series[i]).Hash(), time.Unix(int64(i), 0), true, copyFn) - } +func TestHashTokenList(t *testing.T) { + // Same tokens should produce same hash + tokens1 := []uint32{100, 200, 300} + tokens2 := []uint32{100, 200, 300} + assert.Equal(t, hashTokenList(tokens1), hashTokenList(tokens2)) - c.Purge(time.Unix(int64(ttl+1), 0)) - // call purge twice, just to hit "quick" path. It doesn't really do anything. - c.Purge(time.Unix(int64(ttl+1), 0)) + // Different tokens should produce different hash + tokens3 := []uint32{100, 200, 400} + assert.NotEqual(t, hashTokenList(tokens1), hashTokenList(tokens3)) - exp := len(series) - (ttl + 1) - assert.Equal(t, exp, c.Active()) - assert.Equal(t, exp, c.ActiveNativeHistogram()) - } + // Empty list + assert.Equal(t, hashTokenList(nil), hashTokenList([]uint32{})) } -func TestActiveSeries_PurgeOpt(t *testing.T) { - metric := labels.NewBuilder(labels.FromStrings("__name__", "logs")) - ls1 := metric.Set("_", "ypfajYg2lsv").Labels() - ls2 := metric.Set("_", "KiqbryhzUpn").Labels() +func TestUpdateTokens_DetectsChange(t *testing.T) { c := NewActiveSeries() + // First call should detect change (from empty to something) + changed := c.updateTokens([]uint32{100}, []uint32{100, 200}) + assert.True(t, changed) + + // Same tokens again — no change + changed = c.updateTokens([]uint32{100}, []uint32{100, 200}) + assert.False(t, changed) + + // Different ring tokens — change detected + changed = c.updateTokens([]uint32{100}, []uint32{100, 200, 300}) + assert.True(t, changed) +} + +// copyFn is a helper used by tests to copy labels. +func copyFn(l labels.Labels) labels.Labels { + return l.Copy() +} + +func TestActiveSeries_NativeHistogram_Owned(t *testing.T) { + c := NewActiveSeries() now := time.Now() - c.UpdateSeries(ls1, ls1.Hash(), now.Add(-2*time.Minute), true, copyFn) - c.UpdateSeries(ls2, ls2.Hash(), now, true, copyFn) - c.Purge(now) + + // Ring: instance owns everything (single token) + ringTokens := []uint32{100} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Add a native histogram series + lbls := labels.FromStrings("__name__", "histogram_metric", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 50, now, true, copyFn) assert.Equal(t, 1, c.Active()) + assert.Equal(t, 1, c.Owned()) assert.Equal(t, 1, c.ActiveNativeHistogram()) +} + +func TestActiveSeries_ExistingSeriesNotRejected(t *testing.T) { + // If a series already exists in ActiveSeries (was previously tracked), + // updating it should succeed even if it would fail the ownership check + // for a NEW series. This handles the case where a series was owned, + // ring hasn't updated yet, and we get another sample for it. + c := NewActiveSeries() + now := time.Now() + later := now.Add(1 * time.Minute) + + // Ring: instance owns token 100 + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Add series with key=50 → owned (token 100) + lbls := labels.FromStrings("__name__", "metric_1", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 50, now, false, copyFn) + require.Equal(t, 1, c.Active()) + + // Update same series again (simulating another sample arriving) + // This should succeed because findEntryForSeries finds existing entry + c.UpdateSeries(lbls, lbls.Hash(), 50, later, false, copyFn) + assert.Equal(t, 1, c.Active()) // Still 1, not rejected or duplicated +} + +// --- Tests for two-flag behavior and atomic.Pointer[ringState] --- - c.UpdateSeries(ls1, ls1.Hash(), now.Add(-1*time.Minute), true, copyFn) - c.UpdateSeries(ls2, ls2.Hash(), now, true, copyFn) - c.Purge(now) +func TestActiveSeries_AtomicPointer_ConsistentRead(t *testing.T) { + // Verify that UpdateSeries reads a consistent ringState snapshot. + // After updateTokens stores new state, subsequent UpdateSeries calls + // should see the new tokens immediately. + c := NewActiveSeries() + now := time.Now() + // Initially no ring state — all series accepted + lbls := labels.FromStrings("__name__", "metric_before_ring", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 150, now, false, copyFn) assert.Equal(t, 1, c.Active()) - assert.Equal(t, 1, c.ActiveNativeHistogram()) - // This will *not* update the series, since there is already newer timestamp. - c.UpdateSeries(ls2, ls2.Hash(), now.Add(-1*time.Minute), true, copyFn) - c.Purge(now) + // Load ring: instance owns token 100 only. Key=150 → token 200 → NOT owned. + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // New series with key=150 should be rejected (not owned) + lbls2 := labels.FromStrings("__name__", "metric_after_ring", "job", "test") + c.UpdateSeries(lbls2, lbls2.Hash(), 150, now, false, copyFn) + // Only the first series (created before ring loaded) should exist + // The second was rejected because ring is now loaded and key=150 → token 200 not ours assert.Equal(t, 1, c.Active()) - assert.Equal(t, 1, c.ActiveNativeHistogram()) } -var activeSeriesTestGoroutines = []int{50, 100, 500} +func TestActiveSeries_AtomicPointer_EmptyRingStateAtInit(t *testing.T) { + // Before any ring data is loaded, the atomic pointer holds emptyRingState. + // All series should be accepted (no ownership filtering). + c := NewActiveSeries() + now := time.Now() -func BenchmarkActiveSeriesTest_single_series(b *testing.B) { - for _, num := range activeSeriesTestGoroutines { - b.Run(fmt.Sprintf("%d", num), func(b *testing.B) { - benchmarkActiveSeriesConcurrencySingleSeries(b, num) - }) + // Add multiple series with various keys — all should be accepted + for i := 0; i < 10; i++ { + lbls := labels.FromStrings("__name__", "metric", "i", string(rune('0'+i))) + c.UpdateSeries(lbls, lbls.Hash(), uint32(i*100+50), now, false, copyFn) } -} -func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) { - series := labels.FromStrings("a", "a") + assert.Equal(t, 10, c.Active()) + assert.Equal(t, 10, c.Owned()) +} +func TestActiveSeries_Flag1Only_MetricEmitsButNoEnforcement(t *testing.T) { + // Simulates flag 1 ON, flag 2 OFF scenario. + // ActiveSeries tracks ownership (rejects unowned at creation), + // but the caller (ingester) would still use Head().NumSeries() for limits. + // This test verifies ActiveSeries still functions correctly with ring loaded. c := NewActiveSeries() + now := time.Now() - wg := &sync.WaitGroup{} - start := make(chan struct{}) - max := int(math.Ceil(float64(b.N) / float64(goroutines))) - labelhash := series.Hash() - for range goroutines { - wg.Go(func() { - <-start + // Ring: instance owns token 100 (not 200) + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) - now := time.Now() + // Owned series (key=50 → token 100 → ours) + lblsOwned := labels.FromStrings("__name__", "owned_metric", "job", "test") + c.UpdateSeries(lblsOwned, lblsOwned.Hash(), 50, now, false, copyFn) - for ix := range max { - now = now.Add(time.Duration(ix) * time.Millisecond) - c.UpdateSeries(series, labelhash, now, false, copyFn) - } - }) - } + // Unowned series (key=150 → token 200 → not ours) + lblsUnowned := labels.FromStrings("__name__", "unowned_metric", "job", "test") + c.UpdateSeries(lblsUnowned, lblsUnowned.Hash(), 150, now, false, copyFn) - b.ResetTimer() - close(start) - wg.Wait() + // Only owned series tracked + assert.Equal(t, 1, c.Active()) + assert.Equal(t, 1, c.Owned()) + + // The metric cortex_ingester_owned_series would report 1. + // With flag 2 OFF, PreCreation ignores this and uses Head().NumSeries(). + // This test just validates ActiveSeries itself works correctly. } -func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { +func TestActiveSeries_Flag2WithoutFlag1_FallsBack(t *testing.T) { + // Simulates flag 2 ON but flag 1 OFF. + // When flag 1 is off, ringTokens are never loaded (updateTokens never called). + // ActiveSeries behaves as before — all series accepted, Owned() == Active(). c := NewActiveSeries() + now := time.Now() - // Prepare series - nameBuf := bytes.Buffer{} - for range 50 { - nameBuf.WriteString("abcdefghijklmnopqrstuvzyx") - } - name := nameBuf.String() + // Do NOT call updateTokens (simulating flag 1 off — no ring data loaded) + // Add series — all should be accepted regardless of key value + lbls1 := labels.FromStrings("__name__", "metric_1", "job", "test") + lbls2 := labels.FromStrings("__name__", "metric_2", "job", "test") + c.UpdateSeries(lbls1, lbls1.Hash(), 50, now, false, copyFn) + c.UpdateSeries(lbls2, lbls2.Hash(), 150, now, false, copyFn) - series := make([]labels.Labels, b.N) - labelhash := make([]uint64, b.N) - for s := 0; b.Loop(); s++ { - series[s] = labels.FromStrings(name, name+strconv.Itoa(s)) - labelhash[s] = series[s].Hash() - } + assert.Equal(t, 2, c.Active()) + assert.Equal(t, 2, c.Owned()) // Owned == Active when no ring loaded +} - now := time.Now().UnixNano() +func TestActiveSeries_BothFlagsOn_OwnedUsedForLimits(t *testing.T) { + // Simulates both flags on: ring loaded, ownership tracked. + // Owned() accurately reflects only series this instance owns. + c := NewActiveSeries() + now := time.Now() - for ix := 0; b.Loop(); ix++ { - c.UpdateSeries(series[ix], labelhash[ix], time.Unix(0, now+int64(ix)), false, copyFn) - } -} + // Ring: 3 tokens, instance owns 2 of them (100, 300) + ringTokens := []uint32{100, 200, 300} + instanceTokens := []uint32{100, 300} + c.updateTokens(instanceTokens, ringTokens) -func BenchmarkActiveSeries_Purge_once(b *testing.B) { - benchmarkPurge(b, false) -} + // key=50 → token 100 → OWNED + lbls1 := labels.FromStrings("__name__", "m1", "job", "test") + c.UpdateSeries(lbls1, lbls1.Hash(), 50, now, false, copyFn) -func BenchmarkActiveSeries_Purge_twice(b *testing.B) { - benchmarkPurge(b, true) -} + // key=150 → token 200 → NOT owned (rejected) + lbls2 := labels.FromStrings("__name__", "m2", "job", "test") + c.UpdateSeries(lbls2, lbls2.Hash(), 150, now, false, copyFn) + + // key=250 → token 300 → OWNED + lbls3 := labels.FromStrings("__name__", "m3", "job", "test") + c.UpdateSeries(lbls3, lbls3.Hash(), 250, now, false, copyFn) + + assert.Equal(t, 2, c.Active()) + assert.Equal(t, 2, c.Owned()) -func benchmarkPurge(b *testing.B, twice bool) { - const numSeries = 10000 - const numExpiresSeries = numSeries / 25 + // PreCreation (with flag 2 on) would use Owned()=2 instead of Head().NumSeries() + // which might be much higher due to stale data in TSDB. +} +func TestActiveSeries_UpdateMetrics_LoadsFromAtomicPointer(t *testing.T) { + // Verify that UpdateMetrics reads ring state from the atomic pointer + // (same as UpdateSeries), ensuring consistency. + c := NewActiveSeries() now := time.Now() + keepUntil := now.Add(-10 * time.Minute) + + // Ring: owns token 100 + ringTokens := []uint32{100, 200} + instanceTokens := []uint32{100} + c.updateTokens(instanceTokens, ringTokens) + + // Add owned series + lbls := labels.FromStrings("__name__", "metric_1", "job", "test") + c.UpdateSeries(lbls, lbls.Hash(), 50, now, false, copyFn) + assert.Equal(t, 1, c.Owned()) + + // Call UpdateMetrics with NEW ring where we lose token 100 + // The function should store new state via updateTokens, then load it + // from the atomic pointer to pass to stripes. + newRingTokens := []uint32{100, 200, 300} + newInstanceTokens := []uint32{200} // We no longer own 100 + c.UpdateMetrics(keepUntil, newInstanceTokens, newRingTokens) + + // Series key=50 → token 100 → not in {200} → removed + assert.Equal(t, 0, c.Active()) + assert.Equal(t, 0, c.Owned()) +} + +func TestActiveSeries_UpdateTokens_ImmutableSnapshots(t *testing.T) { + // Verify that updateTokens creates a new ringState each time, + // not mutating the previous one. This is critical for atomic.Pointer safety. c := NewActiveSeries() - series := [numSeries]labels.Labels{} - labelhash := [numSeries]uint64{} - for s := range numSeries { - series[s] = labels.FromStrings("a", strconv.Itoa(s)) - labelhash[s] = series[s].Hash() - } + // First ring state + c.updateTokens([]uint32{100}, []uint32{100, 200}) + state1 := c.ring.Load() + + // Second ring state (different) + c.updateTokens([]uint32{100, 300}, []uint32{100, 200, 300}) + state2 := c.ring.Load() - for b.Loop() { - b.StopTimer() - - // Prepare series - for ix, s := range series { - if ix < numExpiresSeries { - c.UpdateSeries(s, labelhash[ix], now.Add(-time.Minute), false, copyFn) - } else { - c.UpdateSeries(s, labelhash[ix], now, false, copyFn) - } - } - - assert.Equal(b, numSeries, c.Active()) - b.StartTimer() - - // Purge everything - c.Purge(now) - assert.Equal(b, numSeries-numExpiresSeries, c.Active()) - - if twice { - c.Purge(now) - assert.Equal(b, numSeries-numExpiresSeries, c.Active()) - } + // They should be different pointers with different content + assert.NotEqual(t, state1, state2) + assert.Equal(t, 2, len(state1.ringTokens)) + assert.Equal(t, 3, len(state2.ringTokens)) + assert.Equal(t, 1, len(state1.instanceTokens)) + assert.Equal(t, 2, len(state2.instanceTokens)) +} + +func TestActiveSeries_InstanceOwnedCount_Recalculation(t *testing.T) { + // Simulates what updateActiveSeries does: sums Owned() across tenants. + // Verify that after ring change removes series, Owned() reflects it. + c := NewActiveSeries() + now := time.Now() + keepUntil := now.Add(-10 * time.Minute) + + // Ring: instance owns tokens 100 and 300 + ringTokens := []uint32{100, 200, 300} + instanceTokens := []uint32{100, 300} + c.updateTokens(instanceTokens, ringTokens) + + // Add 3 owned series + for i := 0; i < 3; i++ { + lbls := labels.FromStrings("__name__", "metric", "i", string(rune('a'+i))) + // Keys 50, 250, 50 → tokens 100, 300, 100 → all owned + keys := []uint32{50, 250, 50} + c.UpdateSeries(lbls, lbls.Hash(), keys[i], now, false, copyFn) } + assert.Equal(t, 3, c.Owned()) + + // Ring changes: we lose token 300, keep 100 + newRingTokens := []uint32{100, 200, 300, 400} + newInstanceTokens := []uint32{100} // Lost 300 + c.UpdateMetrics(keepUntil, newInstanceTokens, newRingTokens) + + // Series with key=250 → token 300 → not in {100} → removed + // Series with key=50 → token 100 → in {100} → kept (2 series share this key) + assert.Equal(t, 2, c.Owned()) + + // In the real ingester, this value would be summed across all tenants + // and stored in instanceOwnedCount. } diff --git a/pkg/ingester/active_series_tracker_test.go b/pkg/ingester/active_series_tracker_test.go index adcd2a03309..69aa18e01e0 100644 --- a/pkg/ingester/active_series_tracker_test.go +++ b/pkg/ingester/active_series_tracker_test.go @@ -200,7 +200,7 @@ func TestActiveSeries_Basic(t *testing.T) { now := time.Now() s := labels.FromStrings("__name__", "test", "job", "app") - as.UpdateSeries(s, s.Hash(), now, false, func(l labels.Labels) labels.Labels { return l.Copy() }) + as.UpdateSeries(s, s.Hash(), 0, now, false, func(l labels.Labels) labels.Labels { return l.Copy() }) assert.Equal(t, 1, as.Active()) } diff --git a/pkg/ingester/client/compat.go b/pkg/ingester/client/compat.go index 20f745c52eb..377dbbb8d62 100644 --- a/pkg/ingester/client/compat.go +++ b/pkg/ingester/client/compat.go @@ -10,6 +10,7 @@ import ( storecache "github.com/thanos-io/thanos/pkg/store/cache" "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/util" ) // ToQueryRequest builds a QueryRequest proto. @@ -294,10 +295,10 @@ func FastFingerprint(ls []cortexpb.LabelAdapter) model.Fingerprint { var result uint64 for _, l := range ls { - sum := hashNew() - sum = hashAdd(sum, l.Name) - sum = hashAddByte(sum, model.SeparatorByte) - sum = hashAdd(sum, l.Value) + sum := util.HashNew() + sum = util.HashAdd(sum, l.Name) + sum = util.HashAddByte(sum, model.SeparatorByte) + sum = util.HashAdd(sum, l.Value) result ^= sum } return model.Fingerprint(result) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3a6c03449c6..6361bd12b73 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -142,6 +142,19 @@ type Config struct { HeadQueriedSeriesMetricsSampleRate float64 `yaml:"head_queried_series_metrics_sample_rate"` HeadQueriedSeriesMetricsWindows cortex_tsdb.DurationList `yaml:"head_queried_series_metrics_windows"` + // OwnedSeriesMetricsEnabled enables tracking of owned series per user. + // When enabled, the ingester computes series ownership based on the ring and + // emits the cortex_ingester_owned_series metric. Does NOT change limit enforcement + // behavior — use OwnedSeriesLimitEnforcementEnabled for that. + OwnedSeriesMetricsEnabled bool `yaml:"owned_series_metrics_enabled"` + + // OwnedSeriesLimitEnforcementEnabled enables using owned series count for limit + // enforcement in PreCreation(). When enabled (requires OwnedSeriesMetricsEnabled=true), + // both the per-user series limit and instance-level max_series limit use owned count + // instead of Head().NumSeries(), preventing false throttling during resharding. + // If OwnedSeriesMetricsEnabled is false, this flag has no effect (falls back to old behavior). + OwnedSeriesLimitEnforcementEnabled bool `yaml:"owned_series_limit_enforcement_enabled"` + // Use blocks storage. BlocksStorageConfig cortex_tsdb.BlocksStorageConfig `yaml:"-"` @@ -214,6 +227,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.HeadQueriedSeriesMetricsWindows = cortex_tsdb.DurationList{2 * time.Hour} f.Var(&cfg.HeadQueriedSeriesMetricsWindows, "ingester.head-queried-series-metrics-windows", "Time windows to expose head queried series metrics. Also controls how long per-metric-name cardinality is reported after last query.") + f.BoolVar(&cfg.OwnedSeriesMetricsEnabled, "ingester.owned-series-metrics-enabled", false, "Enable tracking of owned series per user. When enabled, the ingester computes series ownership based on the ring and emits cortex_ingester_owned_series metric.") + f.BoolVar(&cfg.OwnedSeriesLimitEnforcementEnabled, "ingester.owned-series-limit-enforcement-enabled", false, "Use owned series count for limit enforcement. Requires owned-series-metrics-enabled. When enabled, PreCreation uses owned count instead of Head().NumSeries() for both per-user and instance-level limits.") + f.BoolVar(&cfg.UploadCompactedBlocksEnabled, "ingester.upload-compacted-blocks-enabled", true, "Enable uploading compacted blocks.") f.StringVar(&cfg.IgnoreSeriesLimitForMetricNames, "ingester.ignore-series-limit-for-metric-names", "", "Comma-separated list of metric names, for which -ingester.max-series-per-metric and -ingester.max-global-series-per-metric limits will be ignored. Does not affect max-series-per-user or max-global-series-per-metric limits.") f.StringVar(&cfg.AdminLimitMessage, "ingester.admin-limit-message", "please contact administrator to raise it", "Customize the message contained in limit errors") @@ -389,6 +405,19 @@ type userTSDB struct { instanceSeriesCount *atomic.Int64 // Shared across all userTSDB instances created by ingester. instanceLimitsFn func() *InstanceLimits + // ownedSeriesLimitEnabled controls whether PreCreation uses activeSeries.Owned() + // for limit checks instead of Head().NumSeries(). Only true when BOTH + // OwnedSeriesMetricsEnabled AND OwnedSeriesLimitEnforcementEnabled are set. + ownedSeriesLimitEnabled bool + + // instanceOwnedCount tracks total owned series across all tenants on this ingester. + // Recalculated every updateActiveSeries cycle (1 min). Used for instance-level + // max_series limit when ownedSeriesLimitEnabled is true. + // NOTE: Up to 1 minute stale after ring changes. This is acceptable because + // staleness is conservative (overcounts) and this limit protects against OOM, + // not customer-facing throttle errors. + instanceOwnedCount *atomic.Int64 + stateMtx sync.RWMutex state tsdbState pushesInFlight sync.WaitGroup // Increased with stateMtx read lock held, only if state == active or activeShipping. @@ -523,16 +552,42 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error { return nil } - // Verify ingester's global limit + // Verify ingester's global limit (instance-level max_series). + // When limit enforcement is enabled, use instanceOwnedCount which reflects + // only series this ingester currently owns according to the ring. + // NOTE: instanceOwnedCount is recalculated every ~1 min in updateActiveSeries. + // Up to 1 min stale after ring changes, but conservative (overcounts). gl := u.instanceLimitsFn() if gl != nil && gl.MaxInMemorySeries > 0 { - if series := u.instanceSeriesCount.Load(); series >= gl.MaxInMemorySeries { + var instanceCount int64 + if u.ownedSeriesLimitEnabled { + instanceCount = u.instanceOwnedCount.Load() + // Fallback at startup: instanceOwnedCount is 0 until the first + // updateActiveSeries cycle runs (~1 min). Use instanceSeriesCount + // to maintain OOM protection during this window. + if instanceCount == 0 { + instanceCount = u.instanceSeriesCount.Load() + } + } else { + instanceCount = u.instanceSeriesCount.Load() + } + if instanceCount >= gl.MaxInMemorySeries { return errMaxSeriesLimitReached } } - // Total series limit. - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { + // Per-user series limit. + // When limit enforcement is enabled (flag 2 + flag 1), use activeSeries.Owned() + // which excludes resharded/stale series this ingester no longer owns. This prevents + // false throttling during scale-up (where Head().NumSeries() stays high but the + // local limit has dropped due to more ingesters joining). + seriesCount := int(u.Head().NumSeries()) + if u.ownedSeriesLimitEnabled { + if owned := u.activeSeries.Owned(); owned > 0 { + seriesCount = owned + } + } + if err := u.limiter.AssertMaxSeriesPerUser(u.userID, seriesCount); err != nil { return err } @@ -733,6 +788,10 @@ type TSDBState struct { // Number of series in memory, across all tenants. seriesCount atomic.Int64 + // Number of owned series across all tenants. Recalculated every updateActiveSeries + // cycle (~1 min). Used for instance-level max_series when limit enforcement is enabled. + ownedSeriesCount atomic.Int64 + // Head compactions metrics. compactionsTriggered prometheus.Counter compactionsFailed prometheus.Counter @@ -1183,13 +1242,28 @@ func (i *Ingester) getMaxExemplars(userID string) int64 { func (i *Ingester) updateActiveSeries(ctx context.Context) { purgeTime := time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) + // When owned metrics are enabled, recalculate the instance-level owned count + // from scratch each cycle. This avoids drift from edge cases (missed decrements + // during ring changes or tenant deletions). The loop already iterates all userTSDBs + // and calls Owned(), so this is essentially free (one int64 addition per tenant). + var totalOwnedCount int64 + for _, userID := range i.getTSDBUsers() { userDB, err := i.getTSDB(userID) if err != nil || userDB == nil { continue } - userDB.activeSeries.Purge(purgeTime) + if i.cfg.OwnedSeriesMetricsEnabled { + // Use UpdateMetrics which handles both purge AND ownership re-evaluation. + userDB.activeSeries.UpdateMetrics(purgeTime, i.lifecycler.GetTokens(), i.lifecycler.GetRingTokensForZone(i.lifecycler.Zone)) + owned := userDB.activeSeries.Owned() + i.metrics.ownedSeriesPerUser.WithLabelValues(userID).Set(float64(owned)) + totalOwnedCount += int64(owned) + } else { + userDB.activeSeries.Purge(purgeTime) + } + i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.Active())) i.metrics.activeNHSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.ActiveNativeHistogram())) i.metrics.headMetricNamesPerUser.WithLabelValues(userID).Set(float64(userDB.seriesInMetric.ActiveMetricNames())) @@ -1202,6 +1276,11 @@ func (i *Ingester) updateActiveSeries(ctx context.Context) { userDB.trackerCounter.updateConfig(ctx, userDB.db, trackers) userDB.trackerCounter.updateMetrics(i.metrics.activeSeriesPerTracker, userID, trackers) } + + // Store the instance-level owned count for use in PreCreation's max_series check. + if i.cfg.OwnedSeriesMetricsEnabled { + i.TSDBState.ownedSeriesCount.Store(totalOwnedCount) + } } func (i *Ingester) updateActiveQueriedSeries(ctx context.Context) { @@ -1557,6 +1636,18 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return nil, wrapWithUser(errors.Errorf("out-of-order label set found when push: %s", tsLabels), userID) } tsLabelsHash := tsLabels.Hash() + + // Compute ring token for this series (same hash the distributor uses for routing). + // Used by ActiveSeries to track ownership. When flag is off, tsToken=0 and + // ActiveSeries skips ownership checks. + var tsToken uint32 + if i.cfg.OwnedSeriesMetricsEnabled { + tsToken, err = ring.TokenForLabels(userID, ts.Labels, i.cfg.DistributorShardByAllLabels) + if err != nil { + return nil, wrapWithUser(err, userID) + } + } + ref, copiedLabels := app.GetRef(tsLabels, tsLabelsHash) // To find out if any sample was added to this series, we keep old value. @@ -1681,7 +1772,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte isNHAppended := succeededHistogramsCount > oldSucceededHistogramsCount shouldUpdateSeries := (succeededSamplesCount > oldSucceededSamplesCount) || isNHAppended if i.cfg.ActiveSeriesMetricsEnabled && shouldUpdateSeries { - db.activeSeries.UpdateSeries(tsLabels, tsLabelsHash, startAppend, isNHAppended, func(l labels.Labels) labels.Labels { + db.activeSeries.UpdateSeries(tsLabels, tsLabelsHash, tsToken, startAppend, isNHAppended, func(l labels.Labels) labels.Labels { // we must already have copied the labels if succeededSamplesCount or succeededHistogramsCount has been incremented. return copiedLabels }) @@ -3056,6 +3147,8 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { instanceSeriesCount: &i.TSDBState.seriesCount, interner: util.NewLruInterner(i.cfg.LabelsStringInterningEnabled), labelsStringInterningEnabled: i.cfg.LabelsStringInterningEnabled, + ownedSeriesLimitEnabled: i.cfg.OwnedSeriesMetricsEnabled && i.cfg.OwnedSeriesLimitEnforcementEnabled, + instanceOwnedCount: &i.TSDBState.ownedSeriesCount, blockRetentionPeriod: i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(), postingCache: postingCache, @@ -3450,7 +3543,7 @@ func (i *Ingester) compactionLoop(ctx context.Context) error { } // Lets create the slot based on the hash id - i := int(client.HashAdd32(client.HashNew32(), i.lifecycler.ID) % 10) + i := int(util.HashAdd32(util.HashNew32(), i.lifecycler.ID) % 10) return i, 10 } ticker := util.NewSlottedTicker(infoFunc, i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval, 1) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 3ad21faad6d..54492ea9450 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -61,6 +61,7 @@ type ingesterMetrics struct { activeSeriesPerUser *prometheus.GaugeVec activeNHSeriesPerUser *prometheus.GaugeVec headMetricNamesPerUser *prometheus.GaugeVec + ownedSeriesPerUser *prometheus.GaugeVec activeQueriedSeriesPerUser *prometheus.GaugeVec headQueriedSeriesPerUser *prometheus.GaugeVec limitsPerLabelSet *prometheus.GaugeVec @@ -330,6 +331,11 @@ func newIngesterMetrics(r prometheus.Registerer, Help: "Number of unique metric names in the TSDB head per user.", }, []string{"user"}), + ownedSeriesPerUser: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_ingester_owned_series", + Help: "Number of series this ingester currently owns per user according to the ring.", + }, []string{"user"}), + // Not registered automatically, but only if activeSeriesEnabled is true. activeSeriesPerTracker: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_ingester_active_series_per_tracker", @@ -388,6 +394,7 @@ func newIngesterMetrics(r prometheus.Registerer, r.MustRegister(m.activeSeriesPerUser) r.MustRegister(m.activeNHSeriesPerUser) r.MustRegister(m.headMetricNamesPerUser) + r.MustRegister(m.ownedSeriesPerUser) r.MustRegister(m.activeSeriesPerTracker) } diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 1db33a929f0..cb159dbb184 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -156,6 +156,7 @@ type Lifecycler struct { healthyInstancesCount int zonesCount int zones []string + ringTokensByZone map[string][]uint32 lifecyclerMetrics *LifecyclerMetrics logger log.Logger @@ -283,7 +284,7 @@ func (i *Lifecycler) CheckReady(ctx context.Context) error { func (i *Lifecycler) checkRingHealthForReadiness(ctx context.Context) error { // Ensure the instance holds some tokens. - if len(i.getTokens()) == 0 { + if len(i.GetTokens()) == 0 { return fmt.Errorf("this instance owns no tokens") } @@ -363,12 +364,20 @@ func (i *Lifecycler) ChangeState(ctx context.Context, state InstanceState) error return <-errCh } -func (i *Lifecycler) getTokens() Tokens { +func (i *Lifecycler) GetTokens() Tokens { i.stateMtx.RLock() defer i.stateMtx.RUnlock() return i.tokenFile.Tokens } +// GetRingTokensForZone returns all sorted tokens for the given zone in the ring. +// Used by ingesters to check which series they own within their zone. +func (i *Lifecycler) GetRingTokensForZone(zone string) []uint32 { + i.countersLock.RLock() + defer i.countersLock.RUnlock() + return i.ringTokensByZone[zone] +} + func (i *Lifecycler) setTokens(tokens Tokens) { i.lifecyclerMetrics.tokensOwned.Set(float64(len(tokens))) @@ -938,7 +947,7 @@ func (i *Lifecycler) verifyTokens(ctx context.Context) bool { func (i *Lifecycler) compareTokens(fromRing Tokens) bool { sort.Sort(fromRing) - tokens := i.getTokens() + tokens := i.GetTokens() sort.Sort(tokens) if len(tokens) != len(fromRing) { @@ -970,14 +979,14 @@ func (i *Lifecycler) autoJoin(ctx context.Context, targetState InstanceState, al // Need to make sure we didn't change the num of tokens configured myTokens, _ := ringDesc.TokensFor(i.ID) if !alreadyInRing { - myTokens = i.getTokens() + myTokens = i.GetTokens() } needTokens := i.cfg.NumTokens - len(myTokens) - if needTokens == 0 && myTokens.Equals(i.getTokens()) { + if needTokens == 0 && myTokens.Equals(i.GetTokens()) { // Tokens have been verified. No need to change them. state := i.GetState() - ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.getTokens(), state, i.getRegisteredAt()) + ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.GetTokens(), state, i.getRegisteredAt()) level.Info(i.logger).Log("msg", "auto joined with existing tokens", "ring", i.RingName, "state", state) return ringDesc, true, nil } @@ -993,7 +1002,7 @@ func (i *Lifecycler) autoJoin(ctx context.Context, targetState InstanceState, al i.setTokens(myTokens) state := i.GetState() - ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.getTokens(), state, i.getRegisteredAt()) + ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.GetTokens(), state, i.getRegisteredAt()) level.Info(i.logger).Log("msg", "auto joined with new tokens", "ring", i.RingName, "state", state) return ringDesc, true, nil @@ -1023,7 +1032,7 @@ func (i *Lifecycler) updateConsul(ctx context.Context) error { if !ok { // consul must have restarted level.Info(i.logger).Log("msg", "found empty ring, inserting tokens", "ring", i.RingName) - ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.getTokens(), i.GetState(), i.getRegisteredAt()) + ringDesc.AddIngester(i.ID, i.Addr, i.Zone, i.GetTokens(), i.GetState(), i.getRegisteredAt()) } else { instanceDesc.Timestamp = time.Now().Unix() instanceDesc.State = i.GetState() @@ -1078,6 +1087,7 @@ func (i *Lifecycler) changeState(ctx context.Context, state InstanceState) error func (i *Lifecycler) updateCounters(ringDesc *Desc) { healthyInstancesCount := 0 zonesMap := map[string]struct{}{} + tokensByZone := map[string][]uint32{} if ringDesc != nil { lastUpdated := i.KVStore.LastUpdateTime(i.RingKey) @@ -1090,6 +1100,8 @@ func (i *Lifecycler) updateCounters(ringDesc *Desc) { healthyInstancesCount++ } } + + tokensByZone = ringDesc.getTokensByZoneExcludingState(READONLY) } zones := make([]string, 0, len(zonesMap)) @@ -1104,6 +1116,7 @@ func (i *Lifecycler) updateCounters(ringDesc *Desc) { i.healthyInstancesCount = healthyInstancesCount i.zonesCount = len(zones) i.zones = zones + i.ringTokensByZone = tokensByZone i.countersLock.Unlock() } diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index bc508370360..586f8b8f8a9 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -109,11 +109,11 @@ func TestLifecycler_RenewTokens(t *testing.T) { return nil }) - originalTokens := l1.getTokens() + originalTokens := l1.GetTokens() require.Len(t, originalTokens, 512) require.IsIncreasing(t, originalTokens) l1.RenewTokens(0.1, ctx) - newTokens := l1.getTokens() + newTokens := l1.GetTokens() require.Len(t, newTokens, 512) require.IsIncreasing(t, newTokens) diff := 0 @@ -364,7 +364,7 @@ func TestLifecycler_ShouldHandleInstanceAbruptlyRestarted(t *testing.T) { return checkNormalised(d, "ing1") }) - expectedTokens := l1.getTokens() + expectedTokens := l1.GetTokens() expectedRegisteredAt := l1.getRegisteredAt() // Wait 1 second because the registered timestamp has second precision. Without waiting @@ -382,7 +382,7 @@ func TestLifecycler_ShouldHandleInstanceAbruptlyRestarted(t *testing.T) { require.NoError(t, err) return checkNormalised(d, "ing1") && - expectedTokens.Equals(l2.getTokens()) && + expectedTokens.Equals(l2.GetTokens()) && expectedRegisteredAt.Unix() == l2.getRegisteredAt().Unix() }) } diff --git a/pkg/ring/model.go b/pkg/ring/model.go index 5a4f134c8ae..89804aa8db8 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -545,6 +545,23 @@ func (d *Desc) getTokensByZone() map[string][]uint32 { return MergeTokensByZone(zones) } +// getTokensByZoneExcludingState returns tokens grouped by zone, excluding instances in the given state. +// Used to exclude READONLY ingesters from ownership calculations during scale-down. +func (d *Desc) getTokensByZoneExcludingState(excludeState InstanceState) map[string][]uint32 { + zones := map[string][][]uint32{} + for _, instance := range d.Ingesters { + if instance.State == excludeState { + continue + } + tokens := instance.Tokens + if !sort.IsSorted(Tokens(tokens)) { + sort.Sort(Tokens(tokens)) + } + zones[instance.Zone] = append(zones[instance.Zone], tokens) + } + return MergeTokensByZone(zones) +} + // getInstancesByAddr returns instances id by its address func (d *Desc) getInstancesByAddr() map[string]string { instancesByAddMap := make(map[string]string, len(d.Ingesters)) diff --git a/pkg/ring/ring.go b/pkg/ring/ring.go index b510a907625..a359f16bebb 100644 --- a/pkg/ring/ring.go +++ b/pkg/ring/ring.go @@ -406,7 +406,7 @@ func (r *Ring) Get(key uint32, op Operation, bufDescs []InstanceDesc, bufHosts [ var ( replicationFactor = r.cfg.ReplicationFactor instances = bufDescs[:0] - start = searchToken(r.ringTokens, key) + start = SearchToken(r.ringTokens, key) iterations = 0 maxInstancePerZone = replicationFactor / len(r.ringZones) zonesWithExtraInstance = replicationFactor % len(r.ringZones) @@ -863,7 +863,7 @@ func (r *Ring) shuffleShard(identifier string, size int, lookbackPeriod time.Dur finalInstancesPerZone++ } for i := 0; i < finalInstancesPerZone; i++ { - start := searchToken(tokens, random.Uint32()) + start := SearchToken(tokens, random.Uint32()) iterations := 0 found := false diff --git a/pkg/ring/token.go b/pkg/ring/token.go new file mode 100644 index 00000000000..87166495d06 --- /dev/null +++ b/pkg/ring/token.go @@ -0,0 +1,56 @@ +package ring + +import ( + "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/extract" +) + +// TokenForLabels returns the ring token (hash key) for a set of series labels. +// This determines which ingester in the ring is responsible for this series. +// Used by both the distributor (to route) and the ingester (to check ownership). +func TokenForLabels(userID string, labels []cortexpb.LabelAdapter, shouldShardByAllLabels bool) (uint32, error) { + if shouldShardByAllLabels { + return ShardByAllLabels(userID, labels), nil + } + + unsafeMetricName, err := extract.UnsafeMetricNameFromLabelAdapters(labels) + if err != nil { + return 0, err + } + return ShardByMetricName(userID, unsafeMetricName), nil +} + +// TokenForMetadata returns the ring token for metadata routing. +func TokenForMetadata(userID string, metricName string, shouldShardByAllLabels bool) uint32 { + if shouldShardByAllLabels { + return ShardByMetricName(userID, metricName) + } + return shardByUser(userID) +} + +// ShardByAllLabels generates a token from userID + all label name/value pairs. +// This function generates different values for different order of same labels. +func ShardByAllLabels(userID string, labels []cortexpb.LabelAdapter) uint32 { + h := shardByUser(userID) + for _, label := range labels { + if len(label.Value) > 0 { + h = util.HashAdd32(h, label.Name) + h = util.HashAdd32(h, label.Value) + } + } + return h +} + +// ShardByMetricName returns the token for the given metric. +func ShardByMetricName(userID string, metricName string) uint32 { + h := shardByUser(userID) + h = util.HashAdd32(h, metricName) + return h +} + +func shardByUser(userID string) uint32 { + h := util.HashNew32() + h = util.HashAdd32(h, userID) + return h +} diff --git a/pkg/ring/util.go b/pkg/ring/util.go index 66a176c0543..608032e12e0 100644 --- a/pkg/ring/util.go +++ b/pkg/ring/util.go @@ -139,8 +139,8 @@ func getZones(tokens map[string][]uint32) []string { return zones } -// searchToken returns the offset of the tokens entry holding the range for the provided key. -func searchToken(tokens []uint32, key uint32) int { +// SearchToken returns the offset of the tokens entry holding the range for the provided key. +func SearchToken(tokens []uint32, key uint32) int { i := sort.Search(len(tokens), func(x int) bool { return tokens[x] > key }) diff --git a/pkg/storage/tsdb/util.go b/pkg/storage/tsdb/util.go index 8b10403c09e..ae9ec8c1ed7 100644 --- a/pkg/storage/tsdb/util.go +++ b/pkg/storage/tsdb/util.go @@ -3,15 +3,15 @@ package tsdb import ( "github.com/oklog/ulid/v2" - "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/cortexproject/cortex/pkg/util" ) // HashBlockID returns a 32-bit hash of the block ID useful for // ring-based sharding. func HashBlockID(id ulid.ULID) uint32 { - h := client.HashNew32() + h := util.HashNew32() for _, b := range id { - h = client.HashAddByte32(h, b) + h = util.HashAddByte32(h, b) } return h } diff --git a/pkg/ingester/client/fnv.go b/pkg/util/fnv.go similarity index 66% rename from pkg/ingester/client/fnv.go rename to pkg/util/fnv.go index fd35a174a1a..d5e25941ab1 100644 --- a/pkg/ingester/client/fnv.go +++ b/pkg/util/fnv.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package client +package util -// Inline and byte-free variant of hash/fnv's fnv64a. +// Inline and byte-free variant of hash/fnv's fnv64a and fnv32. const ( offset64 = 14695981039346656037 @@ -23,14 +23,14 @@ const ( prime32 = 16777619 ) -// hashNew initializes a new fnv64a hash value. -func hashNew() uint64 { +// HashNew initializes a new fnv64a hash value. +func HashNew() uint64 { return offset64 } -// hashAdd adds a string to a fnv64a hash value, returning the updated hash. +// HashAdd adds a string to a fnv64a hash value, returning the updated hash. // Note this is the same algorithm as Go stdlib `sum64a.Write()` -func hashAdd(h uint64, s string) uint64 { +func HashAdd(h uint64, s string) uint64 { for i := 0; i < len(s); i++ { h ^= uint64(s[i]) h *= prime64 @@ -38,14 +38,21 @@ func hashAdd(h uint64, s string) uint64 { return h } -// hashAddByte adds a byte to a fnv64a hash value, returning the updated hash. -func hashAddByte(h uint64, b byte) uint64 { +// HashAddByte adds a byte to a fnv64a hash value, returning the updated hash. +func HashAddByte(h uint64, b byte) uint64 { h ^= uint64(b) h *= prime64 return h } -// HashNew32 initializies a new fnv32 hash value. +// HashAddUint adds a uint64 to a fnv64a hash value, returning the updated hash. +func HashAddUint(h uint64, i uint64) uint64 { + h ^= i + h *= prime64 + return h +} + +// HashNew32 initializes a new fnv32 hash value. func HashNew32() uint32 { return offset32 } @@ -66,3 +73,10 @@ func HashAddByte32(h uint32, b byte) uint32 { h ^= uint32(b) return h } + +// HashAddUint32 adds a uint32 to a fnv32 hash value, returning the updated hash. +func HashAddUint32(h uint32, i uint32) uint32 { + h *= prime32 + h ^= i + return h +}