Add prometheus metrics

chart/bootc-operator/templates/metrics-auth-rbac.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "bootc-operator.fullname" . }}-metrics-auth
+  labels:
+    {{- include "bootc-operator.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "bootc-operator.fullname" . }}-metrics-auth
+  labels:
+    {{- include "bootc-operator.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "bootc-operator.fullname" . }}-metrics-auth
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "bootc-operator.fullname" . }}-controller
+    namespace: {{ .Release.Namespace }}
+{{- end }}

chart/bootc-operator/templates/metrics-reader-rbac.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "bootc-operator.fullname" . }}-metrics-reader
+  labels:
+    {{- include "bootc-operator.labels" . | nindent 4 }}
+rules:
+  - nonResourceURLs:
+      - /metrics
+    verbs:
+      - get
+{{- end }}

chart/bootc-operator/templates/metrics-service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "bootc-operator.fullname" . }}-metrics
+  labels:
+    {{- include "bootc-operator.labels" . | nindent 4 }}
+    app.kubernetes.io/component: controller
+spec:
+  ports:
+    - name: https
+      port: {{ .Values.metrics.service.port }}
+      targetPort: metrics
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: {{ include "bootc-operator.name" . }}
+    app.kubernetes.io/component: controller
+{{- end }}

chart/bootc-operator/templates/metrics-servicemonitor.yaml

@@ -0,0 +1,26 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "bootc-operator.fullname" . }}
+  {{- if .Values.metrics.serviceMonitor.namespace }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+  {{- end }}
+  labels:
+    {{- include "bootc-operator.labels" . | nindent 4 }}
+spec:
+  endpoints:
+    - port: https
+      scheme: https
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      tlsConfig:
+        insecureSkipVerify: true
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "bootc-operator.name" . }}
+      app.kubernetes.io/component: controller
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+{{- end }}

chart/bootc-operator/values.yaml

@@ -10,3 +10,14 @@ imagePullSecrets: []
 
 controller:
   extraArgs: []
+
+metrics:
+  enabled: true
+  service:
+    port: 8443
+  serviceMonitor:
+    # When enabled, bind the metrics-reader ClusterRole to your
+    # Prometheus ServiceAccount so it can access /metrics.
+    enabled: false
+    interval: 30s
+    namespace: ""

cmd/controller/main.go

@@ -15,6 +15,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
 	"github.com/bootc-dev/bootc-operator/internal/controller"
@@ -47,6 +49,8 @@ func init() {
 func main() {
 	var enableLeaderElection bool
 	var probeAddr string
+	var metricsAddr string
+	var metricsCertDir string
 	var tagResolutionInterval time.Duration
 	var allowInsecureRegistry bool
 	flag.StringVar(
@@ -55,6 +59,18 @@ func main() {
 		":8081",
 		"The address the probe endpoint binds to.",
 	)
+	flag.StringVar(
+		&metricsAddr,
+		"metrics-bind-address",
+		":8443",
+		"The address the metrics endpoint binds to.",
+	)
+	flag.StringVar(
+		&metricsCertDir,
+		"metrics-cert-dir",
+		"",
+		"The directory containing TLS certificates for the metrics server. If empty, self-signed certificates are generated.",
+	)
 	flag.DurationVar(
 		&tagResolutionInterval,
 		"tag-resolution-interval",
@@ -88,9 +104,19 @@ func main() {
 		version.GitCommit,
 	)
 
+	metricsOpts := metricsserver.Options{
+		BindAddress:    metricsAddr,
+		SecureServing:  true,
+		FilterProvider: filters.WithAuthenticationAndAuthorization,
+	}
+	if metricsCertDir != "" {
+		metricsOpts.CertDir = metricsCertDir
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		HealthProbeBindAddress: probeAddr,
+		Metrics:                metricsOpts,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "bootc-operator.bootc.dev",
 	})
@@ -99,6 +125,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err := controller.RegisterMetrics(mgr.GetCache()); err != nil {
+		setupLog.Error(err, "Failed to register metrics collector")
+		os.Exit(1)
+	}
+
 	kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "Failed to create Kubernetes clientset")

config/manager/manager.yaml

@@ -70,6 +70,7 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --metrics-bind-address=:8443
         image: ghcr.io/bootc-dev/bootc-operator:latest
         name: manager
         env:
@@ -78,6 +79,9 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         ports:
+        - containerPort: 8443
+          name: metrics
+          protocol: TCP
         - containerPort: 8081
           name: health
           protocol: TCP

config/rbac/kustomization.yaml

@@ -7,3 +7,6 @@ resources:
 - daemon_service_account.yaml
 - daemon_role.yaml
 - daemon_role_binding.yaml
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml

config/rbac/metrics_auth_role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-auth-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/rbac/metrics_auth_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-auth-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/metrics_reader_role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get

docs/src/SUMMARY.md

@@ -8,3 +8,4 @@
 - [Helm Chart](helm.md)
 - [Operations](operations/index.md)
   - [Managing a pool](operations/pool.md)
+  - [Monitoring](operations/monitoring.md)

docs/src/helm.md

@@ -141,3 +141,8 @@ make helm
 | `image.pullPolicy` | not set | Kubernetes image pull policy |
 | `imagePullSecrets` | `[]` | List of pull secret references |
 | `controller.extraArgs` | `[]` | Extra arguments for the controller |
+| `metrics.enabled` | `true` | Deploy the metrics Service |
+| `metrics.service.port` | `8443` | Service port for the metrics endpoint |
+| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource |
+| `metrics.serviceMonitor.interval` | `30s` | Scrape interval |
+| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor |

docs/src/operations/monitoring.md

@@ -0,0 +1,84 @@
+# Monitoring
+
+The Bootc Operator exposes Prometheus metrics on a secure HTTPS
+endpoint. This guide covers how to set up Prometheus and Grafana to
+scrape and visualize the operator metrics.
+
+## Exposed metrics
+
+The operator exposes the following custom metrics:
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `bootcoperator_pool_nodes_total` | Gauge | `pool` | Total number of nodes in a BootcNodePool |
+| `bootcoperator_pool_nodes_updated` | Gauge | `pool` | Nodes running the target digest |
+| `bootcoperator_pool_nodes_updating` | Gauge | `pool` | Nodes actively being updated |
+| `bootcoperator_pool_nodes_degraded` | Gauge | `pool` | Nodes in a degraded state |
+| `bootcoperator_pool_rollout_paused` | Gauge | `pool` | Whether rollout is paused (0 or 1) |
+| `bootcoperator_node_booted_image_info` | Gauge | `node`, `pool`, `image`, `digest`, `version` | Info metric for the booted OS image |
+
+In addition to these custom metrics, the operator exposes standard
+controller-runtime metrics (workqueue depth, reconcile duration, etc.).
+
+## Enabling the ServiceMonitor
+
+The Helm chart includes a `ServiceMonitor` resource that tells
+Prometheus where to scrape. Enable it during install or upgrade:
+
+```bash
+helm upgrade --install bootc-operator \
+  oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator \
+  --namespace bootc-operator \
+  --set metrics.serviceMonitor.enabled=true
+```
+
+Verify the ServiceMonitor was created:
+
+```bash
+kubectl -n bootc-operator get servicemonitor
+```
+
+## Installing Prometheus and Grafana
+
+The recommended way to deploy Prometheus and Grafana is the
+[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
+Helm chart:
+
+```bash
+helm repo add prometheus-community \
+  https://prometheus-community.github.io/helm-charts
+helm repo update
+
+helm install monitoring prometheus-community/kube-prometheus-stack \
+  --namespace monitoring --create-namespace \
+  --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false
+```
+
+Setting `serviceMonitorSelectorNilUsesHelmValues=false` allows
+Prometheus to discover ServiceMonitors from all namespaces, including
+`bootc-operator`.
+
+## Granting Prometheus access to the metrics endpoint
+
+The operator metrics endpoint requires authentication. The Helm chart
+creates a `ClusterRole` called `<release>-metrics-reader` that grants
+`GET` access to `/metrics`. Bind it to the Prometheus ServiceAccount:
+
+```bash
+kubectl create clusterrolebinding prometheus-metrics-reader \
+  --clusterrole=bootc-operator-metrics-reader \
+  --serviceaccount=monitoring:monitoring-kube-prometheus-prometheus
+```
+
+Adjust the ServiceAccount name if your Prometheus installation uses a
+different one.
+
+## Helm values reference
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| `metrics.enabled` | `true` | Deploy the metrics Service |
+| `metrics.service.port` | `8443` | Service port for the metrics endpoint |
+| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource |
+| `metrics.serviceMonitor.interval` | `30s` | Scrape interval |
+| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor (defaults to release namespace) |