CNV-87536: add alerts_effective_active_at_timestamp_seconds metric

go.mod

@@ -11,6 +11,8 @@ require (
 	github.com/openshift/library-go v0.0.0-20240905123346-5bdbfe35a6f5
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.0
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.87.0
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.67.4
 	github.com/prometheus/prometheus v0.308.0
 	github.com/sirupsen/logrus v1.9.3
@@ -57,8 +59,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/x448/float16 v0.8.4 // indirect

pkg/k8s/alert_relabel_config_gc.go

@@ -0,0 +1,57 @@
+package k8s
+
+import (
+	"context"
+
+	"github.com/openshift/monitoring-plugin/pkg/managementlabels"
+)
+
+// gcOrphanedARCs deletes AlertRelabelConfigs whose associated alert rule no
+// longer exists. This handles the case where an operator (or manual action)
+// removes rules from a PrometheusRule or deletes the CR entirely — the ARCs
+// that were created by the plugin for classification/drop/stamp become orphans.
+//
+// Only ARCs carrying the plugin's alertRuleId annotation are considered.
+// GitOps-managed ARCs are never deleted automatically; a warning is logged
+// so that operators can clean them up manually.
+//
+// liveRuleIDs must include every alerting-rule ID still present on a
+// PrometheusRule, including platform rules dropped by relabel configs.
+// IDs are recorded in collectAlerts before the drop continue, so a Drop
+// ARC for a disabled rule is not treated as an orphan.
+func (rrm *relabeledRulesManager) gcOrphanedARCs(ctx context.Context, liveRuleIDs map[string]struct{}) {
+	if rrm.alertRelabelConfigs == nil {
+		return
+	}
+
+	arcs, err := rrm.alertRelabelConfigs.List(ctx, "")
+	if err != nil {
+		log.Errorf("orphan ARC GC: failed to list ARCs: %v", err)
+		return
+	}
+
+	for i := range arcs {
+		arc := &arcs[i]
+
+		ruleID, ok := arc.Annotations[managementlabels.ARCAnnotationAlertRuleIDKey]
+		if !ok || ruleID == "" {
+			continue
+		}
+
+		if _, alive := liveRuleIDs[ruleID]; alive {
+			continue
+		}
+
+		if IsManagedByGitOps(arc.Annotations, arc.Labels) {
+			log.Warnf("orphan ARC GC: ARC %s/%s (ruleId=%s) is orphaned but GitOps-managed — skipping deletion, manual cleanup required", arc.Namespace, arc.Name, ruleID)
+			continue
+		}
+
+		if err := rrm.alertRelabelConfigs.Delete(ctx, arc.Namespace, arc.Name); err != nil {
+			log.Errorf("orphan ARC GC: failed to delete ARC %s/%s: %v", arc.Namespace, arc.Name, err)
+			continue
+		}
+
+		log.Infof("orphan ARC GC: deleted orphaned ARC %s/%s (ruleId=%s)", arc.Namespace, arc.Name, ruleID)
+	}
+}

pkg/k8s/alert_relabel_config_gc_test.go

@@ -0,0 +1,188 @@
+package k8s
+
+import (
+	"context"
+	"testing"
+
+	osmv1 "github.com/openshift/api/monitoring/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/openshift/monitoring-plugin/pkg/managementlabels"
+)
+
+type mockARCInterface struct {
+	arcs    map[string]*osmv1.AlertRelabelConfig
+	deleted []string
+}
+
+func (m *mockARCInterface) List(_ context.Context, _ string) ([]osmv1.AlertRelabelConfig, error) {
+	var result []osmv1.AlertRelabelConfig
+	for _, arc := range m.arcs {
+		result = append(result, *arc)
+	}
+	return result, nil
+}
+
+func (m *mockARCInterface) Get(_ context.Context, ns, name string) (*osmv1.AlertRelabelConfig, bool, error) {
+	if arc, ok := m.arcs[ns+"/"+name]; ok {
+		return arc, true, nil
+	}
+	return nil, false, nil
+}
+
+func (m *mockARCInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) {
+	return &arc, nil
+}
+
+func (m *mockARCInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { return nil }
+
+func (m *mockARCInterface) Delete(_ context.Context, ns, name string) error {
+	m.deleted = append(m.deleted, ns+"/"+name)
+	delete(m.arcs, ns+"/"+name)
+	return nil
+}
+
+func newARC(ns, name, ruleID string, annotations, labels map[string]string) *osmv1.AlertRelabelConfig {
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	if ruleID != "" {
+		annotations[managementlabels.ARCAnnotationAlertRuleIDKey] = ruleID
+	}
+	return &osmv1.AlertRelabelConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Namespace:   ns,
+			Annotations: annotations,
+			Labels:      labels,
+		},
+	}
+}
+
+func TestGCOrphanedARCs_DeletesOrphan(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-orphan": newARC("openshift-monitoring", "arc-orphan", "rule-gone", nil, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if len(mock.deleted) != 1 || mock.deleted[0] != "openshift-monitoring/arc-orphan" {
+		t.Fatalf("expected orphan ARC to be deleted, got deleted=%v", mock.deleted)
+	}
+}
+
+func TestGCOrphanedARCs_KeepsLiveRule(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-alive", nil, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{"rule-alive": {}})
+
+	if len(mock.deleted) != 0 {
+		t.Fatalf("expected no deletions, got deleted=%v", mock.deleted)
+	}
+}
+
+func TestGCOrphanedARCs_SkipsGitOpsManaged(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-gone",
+				map[string]string{"argocd.argoproj.io/tracking-id": "some-id"}, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if len(mock.deleted) != 0 {
+		t.Fatalf("expected GitOps-managed ARC to be preserved, got deleted=%v", mock.deleted)
+	}
+}
+
+func TestGCOrphanedARCs_SkipsARCWithoutAnnotation(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if len(mock.deleted) != 0 {
+		t.Fatalf("expected ARC without annotation to be preserved, got deleted=%v", mock.deleted)
+	}
+}
+
+func TestGCOrphanedARCs_MixedScenario(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-live":    newARC("openshift-monitoring", "arc-live", "rule-1", nil, nil),
+			"openshift-monitoring/arc-orphan1": newARC("openshift-monitoring", "arc-orphan1", "rule-deleted-1", nil, nil),
+			"openshift-monitoring/arc-orphan2": newARC("openshift-monitoring", "arc-orphan2", "rule-deleted-2", nil, nil),
+			"openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-deleted-3",
+				map[string]string{"argocd.argoproj.io/tracking-id": "t"}, nil),
+			"openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	liveIDs := map[string]struct{}{"rule-1": {}}
+	rrm.gcOrphanedARCs(context.Background(), liveIDs)
+
+	deletedSet := map[string]bool{}
+	for _, d := range mock.deleted {
+		deletedSet[d] = true
+	}
+
+	if len(mock.deleted) != 2 {
+		t.Fatalf("expected 2 deletions, got %d: %v", len(mock.deleted), mock.deleted)
+	}
+	if !deletedSet["openshift-monitoring/arc-orphan1"] {
+		t.Error("expected arc-orphan1 to be deleted")
+	}
+	if !deletedSet["openshift-monitoring/arc-orphan2"] {
+		t.Error("expected arc-orphan2 to be deleted")
+	}
+	if deletedSet["openshift-monitoring/arc-live"] {
+		t.Error("arc-live should not have been deleted")
+	}
+	if deletedSet["openshift-monitoring/arc-gitops"] {
+		t.Error("arc-gitops should not have been deleted (GitOps-managed)")
+	}
+	if deletedSet["openshift-monitoring/arc-manual"] {
+		t.Error("arc-manual should not have been deleted (no annotation)")
+	}
+}
+
+func TestGCOrphanedARCs_NilInterface(t *testing.T) {
+	rrm := &relabeledRulesManager{alertRelabelConfigs: nil}
+	// Should not panic
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+}
+
+func TestGCOrphanedARCs_NilAnnotations(t *testing.T) {
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-nil": {
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "arc-nil",
+					Namespace: "openshift-monitoring",
+				},
+			},
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if len(mock.deleted) != 0 {
+		t.Fatalf("expected ARC with nil annotations to be preserved, got deleted=%v", mock.deleted)
+	}
+}

pkg/k8s/enrich_active_at_test.go

@@ -0,0 +1,106 @@
+package k8s
+
+import (
+	"testing"
+	"time"
+)
+
+func TestEnrichActiveAt_ReplacesAlertmanagerTimestamp(t *testing.T) {
+	amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC)
+	promTime := time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC)
+
+	amAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "am"},
+		ActiveAt: amTime,
+	}}
+	promAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "prom"},
+		ActiveAt: promTime,
+	}}
+
+	enrichActiveAt(amAlerts, promAlerts)
+
+	if !amAlerts[0].ActiveAt.Equal(promTime) {
+		t.Errorf("expected ActiveAt=%v, got %v", promTime, amAlerts[0].ActiveAt)
+	}
+}
+
+func TestEnrichActiveAt_NoMatchKeepsOriginal(t *testing.T) {
+	amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC)
+
+	amAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "HighCPU", "severity": "critical"},
+		ActiveAt: amTime,
+	}}
+	promAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "DiskFull", "severity": "warning"},
+		ActiveAt: time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC),
+	}}
+
+	enrichActiveAt(amAlerts, promAlerts)
+
+	if !amAlerts[0].ActiveAt.Equal(amTime) {
+		t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt)
+	}
+}
+
+func TestEnrichActiveAt_EmptyPromAlerts(t *testing.T) {
+	amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC)
+
+	amAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "HighCPU"},
+		ActiveAt: amTime,
+	}}
+
+	enrichActiveAt(amAlerts, nil)
+
+	if !amAlerts[0].ActiveAt.Equal(amTime) {
+		t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt)
+	}
+}
+
+func TestEnrichActiveAt_SkipsZeroPromActiveAt(t *testing.T) {
+	amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC)
+
+	amAlerts := []PrometheusAlert{{
+		Labels:   map[string]string{"alertname": "HighCPU"},
+		ActiveAt: amTime,
+	}}
+	promAlerts := []PrometheusAlert{{
+		Labels: map[string]string{"alertname": "HighCPU"},
+	}}
+
+	enrichActiveAt(amAlerts, promAlerts)
+
+	if !amAlerts[0].ActiveAt.Equal(amTime) {
+		t.Errorf("expected ActiveAt to stay %v when prom has zero time, got %v", amTime, amAlerts[0].ActiveAt)
+	}
+}
+
+func TestAlertFingerprint_IgnoresMetadataLabels(t *testing.T) {
+	fp1 := alertFingerprint(map[string]string{
+		"alertname":       "HighCPU",
+		"severity":        "critical",
+		AlertSourceLabel:  "platform",
+		AlertBackendLabel: "am",
+	})
+	fp2 := alertFingerprint(map[string]string{
+		"alertname":       "HighCPU",
+		"severity":        "critical",
+		AlertSourceLabel:  "platform",
+		AlertBackendLabel: "prom",
+	})
+
+	if fp1 != fp2 {
+		t.Errorf("fingerprints should match when only metadata labels differ:\n  fp1=%q\n  fp2=%q", fp1, fp2)
+	}
+}
+
+func TestAlertFingerprint_DifferentLabelsProduceDifferentKeys(t *testing.T) {
+	fp1 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "critical"})
+	fp2 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "warning"})
+
+	if fp1 == fp2 {
+		t.Error("fingerprints should differ when label values differ")
+	}
+}