From 271555b2ada854dfa75db65c5733f01241092f71 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Wed, 23 Sep 2026 06:52:50 +0900 Subject: [PATCH 1/2] Ingester: fix cortex_ingester_ingestion_delay_seconds losing most observations (#7858) * Ingester: fix cortex_ingester_ingestion_delay_seconds losing most observations The histogram was registered with NativeHistogramMinResetDuration: 1, an untyped constant that Go implicitly converts to time.Duration(1), i.e. 1 nanosecond, instead of the intended 1 hour used by every other native histogram in this file. When the native histogram's bucket count exceeds NativeHistogramMaxBucketNumber (100), client_golang's limitBuckets() first tries maybeReset(), which fully resets the histogram (wiping both native and classic bucket counts, keeping only the latest observation) if at least NativeHistogramMinResetDuration has elapsed since the last reset. With an effectively-zero duration, that condition is satisfied on virtually every call, so instead of gracefully reducing resolution (bucket width doubling / zero bucket widening), the histogram repeatedly self-resets and silently drops the large majority of observations. In a 100k-sample simulation this loses ~86% of observations, corrupting both _count/_sum and the classic le="600" bucket that operators alert on for ingestion lag. Fix it to 1 * time.Hour, matching every other histogram in this file. Fixes #7731 Signed-off-by: ankit090701 * Add CHANGELOG.md entry for #7744 Signed-off-by: ankit090701 * Address review feedback from @yeya24 - Drop the CHANGELOG entry: the bug was introduced in #7443, which was merged after the latest release (v1.21.1, 2026-06-04) and has never shipped, so there's nothing for users to be informed about fixing. - Fix check-modernize lint failure: use `for range numObservations` instead of `for i := 0; i < numObservations; i++` in the new test, since the loop index was never used. Signed-off-by: ankit090701 * remove test Signed-off-by: SungJin1212 --------- Signed-off-by: ankit090701 Signed-off-by: SungJin1212 Co-authored-by: ankit090701 (cherry picked from commit f172a5a4d2f259620ee7ff29f4c1b565b5ee0dea) Signed-off-by: Charlie Le --- pkg/ingester/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 3ad21faad6d..e5e038f68cb 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -156,7 +156,7 @@ func newIngesterMetrics(r prometheus.Registerer, Help: "Delay in seconds between sample ingestion time and sample timestamp.", NativeHistogramBucketFactor: 1.1, NativeHistogramMaxBucketNumber: 100, - NativeHistogramMinResetDuration: 1, + NativeHistogramMinResetDuration: 1 * time.Hour, Buckets: []float64{1, 5, 10, 30, 60, 120, 300, 600}, // 1s, 5s, 10s, 30s, 1m, 2m, 5m, 10m }, []string{"user"}), oooLabelsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ From 76ef7bf52e9af249020aea0fb99e0c37117668ce Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Fri, 25 Sep 2026 14:58:49 -0700 Subject: [PATCH 2/2] Pull minio from docker.io/cortexproject instead of quay.io Every integration leg is failing at Preload Images: docker pull quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z Error response from daemon: unauthorized: access to the requested resource is not authorized MinIO has withdrawn its public images from quay.io, after doing the same on Docker Hub (which is why #7837 moved us to quay.io). No mirror still serves the multi-arch RELEASE.2024-05-28T17-19-04Z image. Host an unmodified copy of MinIO's official RELEASE.2024-07-04T14-25-45Z image under docker.io/cortexproject/minio. It was copied with `crane copy`, so its index digest, sha256:5db7e40b69f0c3ad5a878521ff5029468e3070ef146c084dc2540e2d492075c4, is identical to the source and it keeps linux/amd64 and linux/arm64. Its binary reports the commit behind MinIO's signed release tag, and its SLSA provenance records a build of github.com/minio/minio from MinIO's own release commit. The image runs as root like the previous one, so the e2e harness works unchanged. The development docker-compose stacks previously pulled the untagged (latest) image; pin them to the same tag, since that is the only tag we host. Signed-off-by: Charlie Le (cherry picked from commit 6fb32b6da3026562d6424428a5149ad9e7fc3d23) --- .github/workflows/test-build-deploy.yml | 2 +- development/tsdb-blocks-storage-s3-gossip/docker-compose.yml | 2 +- .../tsdb-blocks-storage-s3-single-binary/docker-compose.yml | 2 +- development/tsdb-blocks-storage-s3/docker-compose.yml | 2 +- integration/e2e/images/images.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index a0a97e3a16e..be6479a1d20 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -409,7 +409,7 @@ jobs: done } - retry docker pull quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z + retry docker pull docker.io/cortexproject/minio:RELEASE.2024-07-04T14-25-45Z retry docker pull consul:1.8.4 retry docker pull quay.io/coreos/etcd:v3.5.29 if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then diff --git a/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml b/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml index 0bfbc0f6c1d..ac87160f36a 100644 --- a/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: quay.io/minio/minio + image: docker.io/cortexproject/minio:RELEASE.2024-07-04T14-25-45Z command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml b/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml index ea247084cfa..84d890dde69 100644 --- a/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: quay.io/minio/minio + image: docker.io/cortexproject/minio:RELEASE.2024-07-04T14-25-45Z command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/development/tsdb-blocks-storage-s3/docker-compose.yml b/development/tsdb-blocks-storage-s3/docker-compose.yml index 54d819d745d..bc966244cc6 100644 --- a/development/tsdb-blocks-storage-s3/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: quay.io/minio/minio + image: docker.io/cortexproject/minio:RELEASE.2024-07-04T14-25-45Z command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/integration/e2e/images/images.go b/integration/e2e/images/images.go index c2714676b14..450beef9eea 100644 --- a/integration/e2e/images/images.go +++ b/integration/e2e/images/images.go @@ -8,7 +8,7 @@ package images var ( Memcached = "memcached:1.6.1" Redis = "docker.io/redis:7.0.4-alpine" - Minio = "quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z" + Minio = "docker.io/cortexproject/minio:RELEASE.2024-07-04T14-25-45Z" // Unmodified copy of MinIO's official image; MinIO withdrew its public images. Consul = "consul:1.8.4" ETCD = "quay.io/coreos/etcd:v3.5.29" Prometheus = "quay.io/prometheus/prometheus:v3.9.1"