From 71e7ae4714f1c14f8b57c7f9c6a4c59a75e205dd Mon Sep 17 00:00:00 2001 From: dongjiang Date: Fri, 31 Jul 2026 18:20:45 +0800 Subject: [PATCH] timex: filter unreasonable offset values from kernel adjtimex() overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On KVM/pvclock guests, a race condition in the kernel NTP PLL can cause adjtimex() to return 2^32 ns (4.294967296s) as a transient overflow value, triggering false-positive NodeClockSkewDetected alerts hundreds of times per day despite NTP being correctly synchronized. Add a ±1.0s sanity bound in the timex collector: values exceeding this threshold are discarded (set to 0) with a warning log, as real NTP sync never produces offsets of this magnitude. Apply the same bound in the NodeClockSkewDetected alert expression to provide defense-in-depth at the rule layer. Fixes #3764 Signed-off-by: dongjiang --- collector/timex.go | 9 ++++++++- docs/node-mixin/alerts/alerts.libsonnet | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/collector/timex.go b/collector/timex.go index 6494726ef7..109b6be719 100644 --- a/collector/timex.go +++ b/collector/timex.go @@ -185,8 +185,15 @@ func (c *timexCollector) Update(ch chan<- prometheus.Metric) error { divisor = microSeconds } + offsetSec := float64(timex.Offset) / divisor + if offsetSec > 1.0 || offsetSec < -1.0 { + c.logger.Warn("Discarding unreasonable timex offset value", + "offset_seconds", offsetSec, "status", status) + offsetSec = 0 + } + ch <- c.syncStatus.mustNewConstMetric(syncStatus) - ch <- c.offset.mustNewConstMetric(float64(timex.Offset) / divisor) + ch <- c.offset.mustNewConstMetric(offsetSec) ch <- c.freq.mustNewConstMetric(1 + float64(timex.Freq)/ppm16frac) ch <- c.maxerror.mustNewConstMetric(float64(timex.Maxerror) / microSeconds) ch <- c.esterror.mustNewConstMetric(float64(timex.Esterror) / microSeconds) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 29c934f57b..01d04c7f8d 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -215,12 +215,16 @@ expr: ||| ( node_timex_offset_seconds{%(nodeExporterSelector)s} > 0.05 + and + node_timex_offset_seconds{%(nodeExporterSelector)s} < 1.0 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) >= 0 ) or ( node_timex_offset_seconds{%(nodeExporterSelector)s} < -0.05 + and + node_timex_offset_seconds{%(nodeExporterSelector)s} > -1.0 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) <= 0 )