diff --git a/chart/bootc-operator/templates/metrics-auth-rbac.yaml b/chart/bootc-operator/templates/metrics-auth-rbac.yaml new file mode 100644 index 00000000..d6b9eefa --- /dev/null +++ b/chart/bootc-operator/templates/metrics-auth-rbac.yaml @@ -0,0 +1,36 @@ +{{- if .Values.metrics.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-auth + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-auth + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "bootc-operator.fullname" . }}-metrics-auth +subjects: + - kind: ServiceAccount + name: {{ include "bootc-operator.fullname" . }}-controller + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-reader-rbac.yaml b/chart/bootc-operator/templates/metrics-reader-rbac.yaml new file mode 100644 index 00000000..ee02b6ba --- /dev/null +++ b/chart/bootc-operator/templates/metrics-reader-rbac.yaml @@ -0,0 +1,13 @@ +{{- if .Values.metrics.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-reader + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +rules: + - nonResourceURLs: + - /metrics + verbs: + - get +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-service.yaml b/chart/bootc-operator/templates/metrics-service.yaml new file mode 100644 index 00000000..48a91945 --- /dev/null +++ b/chart/bootc-operator/templates/metrics-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: controller +spec: + ports: + - name: https + port: {{ .Values.metrics.service.port }} + targetPort: metrics + protocol: TCP + selector: + app.kubernetes.io/name: {{ include "bootc-operator.name" . }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-servicemonitor.yaml b/chart/bootc-operator/templates/metrics-servicemonitor.yaml new file mode 100644 index 00000000..085166b5 --- /dev/null +++ b/chart/bootc-operator/templates/metrics-servicemonitor.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "bootc-operator.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +spec: + endpoints: + - port: https + scheme: https + interval: {{ .Values.metrics.serviceMonitor.interval }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + app.kubernetes.io/name: {{ include "bootc-operator.name" . }} + app.kubernetes.io/component: controller + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/values.yaml b/chart/bootc-operator/values.yaml index 9aaacc54..cec56320 100644 --- a/chart/bootc-operator/values.yaml +++ b/chart/bootc-operator/values.yaml @@ -10,3 +10,14 @@ imagePullSecrets: [] controller: extraArgs: [] + +metrics: + enabled: true + service: + port: 8443 + serviceMonitor: + # When enabled, bind the metrics-reader ClusterRole to your + # Prometheus ServiceAccount so it can access /metrics. + enabled: false + interval: 30s + namespace: "" diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f15223d3..f45b88c8 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -15,6 +15,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" "github.com/bootc-dev/bootc-operator/internal/controller" @@ -47,6 +49,8 @@ func init() { func main() { var enableLeaderElection bool var probeAddr string + var metricsAddr string + var metricsCertDir string var tagResolutionInterval time.Duration var allowInsecureRegistry bool flag.StringVar( @@ -55,6 +59,18 @@ func main() { ":8081", "The address the probe endpoint binds to.", ) + flag.StringVar( + &metricsAddr, + "metrics-bind-address", + ":8443", + "The address the metrics endpoint binds to.", + ) + flag.StringVar( + &metricsCertDir, + "metrics-cert-dir", + "", + "The directory containing TLS certificates for the metrics server. If empty, self-signed certificates are generated.", + ) flag.DurationVar( &tagResolutionInterval, "tag-resolution-interval", @@ -88,9 +104,19 @@ func main() { version.GitCommit, ) + metricsOpts := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + } + if metricsCertDir != "" { + metricsOpts.CertDir = metricsCertDir + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, HealthProbeBindAddress: probeAddr, + Metrics: metricsOpts, LeaderElection: enableLeaderElection, LeaderElectionID: "bootc-operator.bootc.dev", }) @@ -99,6 +125,11 @@ func main() { os.Exit(1) } + if err := controller.RegisterMetrics(mgr.GetCache()); err != nil { + setupLog.Error(err, "Failed to register metrics collector") + os.Exit(1) + } + kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) if err != nil { setupLog.Error(err, "Failed to create Kubernetes clientset") diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e34d6944..e867509b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -70,6 +70,7 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8443 image: ghcr.io/bootc-dev/bootc-operator:latest name: manager env: @@ -78,6 +79,9 @@ spec: fieldRef: fieldPath: metadata.namespace ports: + - containerPort: 8443 + name: metrics + protocol: TCP - containerPort: 8081 name: health protocol: TCP diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 809aa028..d248afd5 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,3 +7,6 @@ resources: - daemon_service_account.yaml - daemon_role.yaml - daemon_role_binding.yaml +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000..32d2e4ec --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000..e775d67f --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000..03ce329d --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index fbfaf433..efb6f73a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,3 +8,4 @@ - [Helm Chart](helm.md) - [Operations](operations/index.md) - [Managing a pool](operations/pool.md) + - [Monitoring](operations/monitoring.md) diff --git a/docs/src/helm.md b/docs/src/helm.md index 91a73e49..df17a3ec 100644 --- a/docs/src/helm.md +++ b/docs/src/helm.md @@ -141,3 +141,8 @@ make helm | `image.pullPolicy` | not set | Kubernetes image pull policy | | `imagePullSecrets` | `[]` | List of pull secret references | | `controller.extraArgs` | `[]` | Extra arguments for the controller | +| `metrics.enabled` | `true` | Deploy the metrics Service | +| `metrics.service.port` | `8443` | Service port for the metrics endpoint | +| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource | +| `metrics.serviceMonitor.interval` | `30s` | Scrape interval | +| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor | diff --git a/docs/src/operations/monitoring.md b/docs/src/operations/monitoring.md new file mode 100644 index 00000000..380777b3 --- /dev/null +++ b/docs/src/operations/monitoring.md @@ -0,0 +1,84 @@ +# Monitoring + +The Bootc Operator exposes Prometheus metrics on a secure HTTPS +endpoint. This guide covers how to set up Prometheus and Grafana to +scrape and visualize the operator metrics. + +## Exposed metrics + +The operator exposes the following custom metrics: + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `bootcoperator_pool_nodes_total` | Gauge | `pool` | Total number of nodes in a BootcNodePool | +| `bootcoperator_pool_nodes_updated` | Gauge | `pool` | Nodes running the target digest | +| `bootcoperator_pool_nodes_updating` | Gauge | `pool` | Nodes actively being updated | +| `bootcoperator_pool_nodes_degraded` | Gauge | `pool` | Nodes in a degraded state | +| `bootcoperator_pool_rollout_paused` | Gauge | `pool` | Whether rollout is paused (0 or 1) | +| `bootcoperator_node_booted_image_info` | Gauge | `node`, `pool`, `image`, `digest`, `version` | Info metric for the booted OS image | + +In addition to these custom metrics, the operator exposes standard +controller-runtime metrics (workqueue depth, reconcile duration, etc.). + +## Enabling the ServiceMonitor + +The Helm chart includes a `ServiceMonitor` resource that tells +Prometheus where to scrape. Enable it during install or upgrade: + +```bash +helm upgrade --install bootc-operator \ + oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator \ + --namespace bootc-operator \ + --set metrics.serviceMonitor.enabled=true +``` + +Verify the ServiceMonitor was created: + +```bash +kubectl -n bootc-operator get servicemonitor +``` + +## Installing Prometheus and Grafana + +The recommended way to deploy Prometheus and Grafana is the +[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) +Helm chart: + +```bash +helm repo add prometheus-community \ + https://prometheus-community.github.io/helm-charts +helm repo update + +helm install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring --create-namespace \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +``` + +Setting `serviceMonitorSelectorNilUsesHelmValues=false` allows +Prometheus to discover ServiceMonitors from all namespaces, including +`bootc-operator`. + +## Granting Prometheus access to the metrics endpoint + +The operator metrics endpoint requires authentication. The Helm chart +creates a `ClusterRole` called `-metrics-reader` that grants +`GET` access to `/metrics`. Bind it to the Prometheus ServiceAccount: + +```bash +kubectl create clusterrolebinding prometheus-metrics-reader \ + --clusterrole=bootc-operator-metrics-reader \ + --serviceaccount=monitoring:monitoring-kube-prometheus-prometheus +``` + +Adjust the ServiceAccount name if your Prometheus installation uses a +different one. + +## Helm values reference + +| Key | Default | Description | +|-----|---------|-------------| +| `metrics.enabled` | `true` | Deploy the metrics Service | +| `metrics.service.port` | `8443` | Service port for the metrics endpoint | +| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource | +| `metrics.serviceMonitor.interval` | `30s` | Scrape interval | +| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor (defaults to release namespace) | diff --git a/go.mod b/go.mod index 7b8502a0..4f6a4748 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( github.com/go-logr/logr v1.4.4 github.com/google/go-containerregistry v0.22.1 github.com/onsi/gomega v1.44.0 + github.com/prometheus/client_golang v1.25.0 + github.com/prometheus/client_model v0.6.3 k8s.io/api v0.37.1 k8s.io/apiextensions-apiserver v0.37.1 k8s.io/apimachinery v0.37.1 @@ -22,8 +24,10 @@ require ( ) require ( + cel.dev/expr v0.25.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect @@ -38,6 +42,7 @@ require ( github.com/aws/smithy-go v1.28.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -46,8 +51,10 @@ require ( github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/go-openapi/jsonreference v1.0.0 // indirect @@ -64,9 +71,11 @@ require ( github.com/go-openapi/swag/typeutils v0.27.1 // indirect github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.1 // indirect @@ -81,8 +90,6 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.25.0 // indirect - github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.72.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -91,10 +98,20 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.23.0 // indirect @@ -103,15 +120,21 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gotest.tools/v3 v3.5.2 // indirect + k8s.io/apiserver v0.37.1 // indirect k8s.io/cli-runtime v0.37.1 // indirect k8s.io/component-base v0.37.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + k8s.io/streaming v0.37.1 // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.21.1 // indirect sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect diff --git a/go.sum b/go.sum index 82079f68..1c587040 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= @@ -40,6 +44,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= @@ -65,14 +71,19 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= @@ -111,8 +122,12 @@ github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAg github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.29.2 h1:ZtDxkeiMmz0mxbKDYiNkE5Lk7V5edMRcaaDf2jX002k= +github.com/google/cel-go v0.29.2/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -126,6 +141,8 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -196,6 +213,26 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -207,6 +244,8 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= @@ -228,6 +267,14 @@ golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -246,6 +293,8 @@ k8s.io/apiextensions-apiserver v0.37.1 h1:7fIQG8eThDSTVYBWg/DOpI8v5wYfKCo0N8/TDM k8s.io/apiextensions-apiserver v0.37.1/go.mod h1:1Q3ujWwHgnhBoGsdmGORflwK1sDFmuaqH1OsMDWk2Gs= k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= +k8s.io/apiserver v0.37.1 h1:dUsNwjsvItyVKNpC1N0AhQ0OBSiUJKfWz004vPifOe0= +k8s.io/apiserver v0.37.1/go.mod h1:EWXW6PntGfOC2uD9GGVobnOExgFoX1z0hZo7f7Rs7Ng= k8s.io/cli-runtime v0.37.1 h1:3mir5bM4XjMJHW3SMRk++3Ylf4YQne0XFFsW+IGT85U= k8s.io/cli-runtime v0.37.1/go.mod h1:g3VQOm71f//aNbp79g0az9jBRTrvPjAHz7WxVysbZ4M= k8s.io/client-go v0.37.1 h1:QTv/5ha4jAHtW9qxxVBkQVFBRDb4jHfFopQqqMdc+wM= @@ -258,8 +307,12 @@ k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3d k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= k8s.io/kubectl v0.37.1 h1:n1VCIntOJiX943uByXCThSZQmE+4v9jGS8S9u94/aHU= k8s.io/kubectl v0.37.1/go.mod h1:66cc4Bz8PxBTxlrHzBuzXEzGIytAnNasKQPeEcpuaV8= +k8s.io/streaming v0.37.1 h1:TpzVfQeFuVndn2g9mFqxy1UcUYPwDzqjUmwR/IzJCWc= +k8s.io/streaming v0.37.1/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ= sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/hack/sync-helm.sh b/hack/sync-helm.sh index f55df976..246a883d 100755 --- a/hack/sync-helm.sh +++ b/hack/sync-helm.sh @@ -90,6 +90,20 @@ sed -i '/^ image: {{ include "bootc-operator\.image" \. }}/i\ - {{ . | quote }}\ {{- end }}' "$CHART_DIR/templates/controller-deployment.yaml" +# Make --metrics-bind-address conditional on metrics.enabled +sed -i 's|^\( - --metrics-bind-address=\):8443$| {{- if .Values.metrics.enabled }}\n\1:{{ .Values.metrics.service.port }}\n {{- end }}|' \ + "$CHART_DIR/templates/controller-deployment.yaml" + +# Make metrics containerPort conditional on metrics.enabled +python3 -c " +import re, sys +text = open(sys.argv[1]).read() +pattern = r' - containerPort: 8443\n name: metrics\n protocol: TCP\n' +repl = ' {{- if .Values.metrics.enabled }}\n - containerPort: {{ .Values.metrics.service.port }}\n name: metrics\n protocol: TCP\n {{- end }}\n' +text = re.sub(pattern, repl, text) +open(sys.argv[1], 'w').write(text) +" "$CHART_DIR/templates/controller-deployment.yaml" + gen_workload "$CONFIG_DIR/daemon/daemon.yaml" DaemonSet daemon \ "$CHART_DIR/templates/daemon-daemonset.yaml" echo "Synced workloads" diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 00000000..3ca3e1ad --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" +) + +type objectLister interface { + List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error +} + +var ( + poolNodesTotal = prometheus.NewDesc( + "bootcoperator_pool_nodes_total", + "Total number of nodes in a BootcNodePool.", + []string{"pool"}, nil, + ) + poolNodesUpdated = prometheus.NewDesc( + "bootcoperator_pool_nodes_updated", + "Number of nodes running the target digest.", + []string{"pool"}, nil, + ) + poolNodesUpdating = prometheus.NewDesc( + "bootcoperator_pool_nodes_updating", + "Number of nodes actively being updated.", + []string{"pool"}, nil, + ) + poolNodesDegraded = prometheus.NewDesc( + "bootcoperator_pool_nodes_degraded", + "Number of nodes in a degraded state.", + []string{"pool"}, nil, + ) + poolRolloutPaused = prometheus.NewDesc( + "bootcoperator_pool_rollout_paused", + "Whether rollout is paused for a BootcNodePool (0 or 1).", + []string{"pool"}, nil, + ) + nodeBootedImageInfo = prometheus.NewDesc( + "bootcoperator_node_booted_image_info", + "Info metric for the currently booted OS image on a node.", + []string{"node", "pool", "image", "digest", "version"}, nil, + ) +) + +type metricsCollector struct { + lister objectLister +} + +func newMetricsCollector(lister objectLister) *metricsCollector { + return &metricsCollector{lister: lister} +} + +func (c *metricsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- poolNodesTotal + ch <- poolNodesUpdated + ch <- poolNodesUpdating + ch <- poolNodesDegraded + ch <- poolRolloutPaused + ch <- nodeBootedImageInfo +} + +func (c *metricsCollector) Collect(ch chan<- prometheus.Metric) { + log := logf.Log.WithName("metrics") + ctx := context.Background() + + var pools bootcv1alpha1.BootcNodePoolList + if err := c.lister.List(ctx, &pools); err != nil { + return + } + + for i := range pools.Items { + pool := &pools.Items[i] + name := pool.Name + + emitGauge(ch, log, poolNodesTotal, float64(pool.Status.NodeCount), name) + emitGauge(ch, log, poolNodesUpdated, float64(pool.Status.UpdatedCount), name) + emitGauge(ch, log, poolNodesUpdating, float64(pool.Status.UpdatingCount), name) + emitGauge(ch, log, poolNodesDegraded, float64(pool.Status.DegradedCount), name) + + var paused float64 + if pool.Spec.Rollout != nil && pool.Spec.Rollout.Paused { + paused = 1 + } + emitGauge(ch, log, poolRolloutPaused, paused, name) + } + + var nodes bootcv1alpha1.BootcNodeList + if err := c.lister.List(ctx, &nodes); err != nil { + return + } + + for i := range nodes.Items { + node := &nodes.Items[i] + if node.Status.Booted == nil { + continue + } + + poolName := ownerPoolName(node) + emitGauge(ch, log, nodeBootedImageInfo, 1, + node.Name, + poolName, + node.Status.Booted.Image, + node.Status.Booted.ImageDigest, + node.Status.Booted.Version, + ) + } +} + +func emitGauge( + ch chan<- prometheus.Metric, + log logr.Logger, + desc *prometheus.Desc, + value float64, + labels ...string, +) { + m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, value, labels...) + if err != nil { + log.Error(err, "Failed to create metric", "metric", desc.String()) + return + } + ch <- m +} + +func RegisterMetrics(lister objectLister) error { + return metrics.Registry.Register(newMetricsCollector(lister)) +} + +func ownerPoolName(node *bootcv1alpha1.BootcNode) string { + for _, ref := range node.OwnerReferences { + if ref.Kind == "BootcNodePool" { + return ref.Name + } + } + return "" +} diff --git a/internal/controller/metrics_envtest_test.go b/internal/controller/metrics_envtest_test.go new file mode 100644 index 00000000..6f4bbc99 --- /dev/null +++ b/internal/controller/metrics_envtest_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +func TestMetricsFromEnvtestCache(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest not available") + } + if testCache == nil { + t.Skip("cache not available") + } + g := NewWithT(t) + ctx := context.Background() + + pool := testutil.NewPool( + "metrics-envtest", + testutil.ImageDigestRefA, + testutil.WithWorkerSelector(), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() (int64, error) { + var fetched bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched) + return fetched.Status.ObservedGeneration, err + }).WithTimeout(10 * time.Second).WithPolling(200 * time.Millisecond).Should(Equal(pool.Generation)) + + reg := prometheus.NewPedanticRegistry() + collector := newMetricsCollector(testCache) + g.Expect(reg.Register(collector)).To(Succeed()) + + families, err := reg.Gather() + g.Expect(err).ToNot(HaveOccurred()) + + f := findFamily(families, "bootcoperator_pool_nodes_total") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).ToNot(BeEmpty()) + + found := false + for _, m := range f.GetMetric() { + if labelValue(m, "pool") == pool.Name { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "expected pool %q in metrics", pool.Name) +} + +func TestMetricsNodeInfoFromEnvtestCache(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest not available") + } + if testCache == nil { + t.Skip("cache not available") + } + g := NewWithT(t) + ctx := context.Background() + + pool := testutil.NewPool("metrics-node-envtest", testutil.ImageDigestRefA, + testutil.WithNodeSelector(map[string]string{"envtest-metrics": "true"}), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() (string, error) { + var fetched bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched) + return fetched.Name, err + }).WithTimeout(10 * time.Second).WithPolling(200 * time.Millisecond).Should(Equal(pool.Name)) + + var fetched bootcv1alpha1.BootcNodePool + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched)).To(Succeed()) + + node := &bootcv1alpha1.BootcNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-envtest-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: bootcv1alpha1.GroupVersion.String(), + Kind: "BootcNodePool", + Name: fetched.Name, + UID: fetched.UID, + }}, + }, + Spec: bootcv1alpha1.BootcNodeSpec{ + DesiredImage: testutil.ImageDigestRefA, + DesiredImageState: bootcv1alpha1.DesiredImageStateStaged, + }, + } + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + node.Status.Booted = &bootcv1alpha1.ImageInfo{ + Image: testutil.ImageDigestRefA, + ImageDigest: testutil.DigestA, + Version: "1.0.0", + } + g.Expect(k8sClient.Status().Update(ctx, node)).To(Succeed()) + + reg := prometheus.NewPedanticRegistry() + collector := newMetricsCollector(testCache) + g.Expect(reg.Register(collector)).To(Succeed()) + + g.Eventually(func() (string, error) { + families, err := reg.Gather() + if err != nil { + return "", err + } + f := findFamily(families, "bootcoperator_node_booted_image_info") + if f == nil { + return "", nil + } + for _, m := range f.GetMetric() { + if labelValue(m, "node") == node.Name { + return labelValue(m, "pool"), nil + } + } + return "", nil + }).WithTimeout(10 * time.Second).WithPolling(500 * time.Millisecond).Should(Equal(pool.Name)) +} diff --git a/internal/controller/metrics_test.go b/internal/controller/metrics_test.go new file mode 100644 index 00000000..c9facc99 --- /dev/null +++ b/internal/controller/metrics_test.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +type fakeCache struct { + pools []bootcv1alpha1.BootcNodePool + nodes []bootcv1alpha1.BootcNode +} + +func newFakeCache(pools []bootcv1alpha1.BootcNodePool, nodes []bootcv1alpha1.BootcNode) *fakeCache { + return &fakeCache{pools: pools, nodes: nodes} +} + +func (f *fakeCache) List(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + switch v := list.(type) { + case *bootcv1alpha1.BootcNodePoolList: + v.Items = f.pools + case *bootcv1alpha1.BootcNodeList: + v.Items = f.nodes + default: + return fmt.Errorf("unexpected list type: %T", list) + } + return nil +} + +func collectMetrics(t *testing.T, c prometheus.Collector) []*dto.MetricFamily { + t.Helper() + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + t.Fatalf("failed to register collector: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + return families +} + +func findFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, f := range families { + if f.GetName() == name { + return f + } + } + return nil +} + +func labelValue(m *dto.Metric, name string) string { + for _, l := range m.GetLabel() { + if l.GetName() == name { + return l.GetValue() + } + } + return "" +} + +func TestCollectorPoolGauges(t *testing.T) { + g := NewWithT(t) + + const poolName = "prod" + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(poolName, testutil.ImageDigestRefA, + testutil.WithWorkerSelector(), + testutil.WithPaused(true), + ), + } + pools[0].Status = bootcv1alpha1.BootcNodePoolStatus{ + NodeCount: 5, + UpdatedCount: 3, + UpdatingCount: 1, + DegradedCount: 1, + } + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + tests := []struct { + name string + value float64 + }{ + {"bootcoperator_pool_nodes_total", 5}, + {"bootcoperator_pool_nodes_updated", 3}, + {"bootcoperator_pool_nodes_updating", 1}, + {"bootcoperator_pool_nodes_degraded", 1}, + {"bootcoperator_pool_rollout_paused", 1}, + } + + for _, tt := range tests { + f := findFamily(families, tt.name) + g.Expect(f).ToNot(BeNil(), "metric %s not found", tt.name) + g.Expect(f.GetMetric()).To(HaveLen(1)) + m := f.GetMetric()[0] + g.Expect(m.GetGauge().GetValue()).To(Equal(tt.value), "metric %s", tt.name) + g.Expect(labelValue(m, "pool")).To(Equal(poolName)) + } +} + +func TestCollectorPoolRolloutNotPaused(t *testing.T) { + g := NewWithT(t) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool("staging", testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + } + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_pool_rollout_paused") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()[0].GetGauge().GetValue()).To(Equal(float64(0))) +} + +func TestCollectorNodeInfoMetric(t *testing.T) { + g := NewWithT(t) + + const ( + poolName = "prod" + nodeName = "worker-1" + version = "1.0.0" + ) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(poolName, testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + } + pools[0].UID = "pool-uid-1" + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode(nodeName, testutil.ImageDigestRefA, + testutil.WithBootedImage(testutil.ImageDigestRefA), + testutil.WithBootedDigest(testutil.DigestA), + testutil.WithBootedVersion(version), + ), + } + nodes[0].OwnerReferences = []metav1.OwnerReference{{ + APIVersion: bootcv1alpha1.GroupVersion.String(), + Kind: "BootcNodePool", + Name: poolName, + UID: pools[0].UID, + }} + + c := newMetricsCollector(newFakeCache(pools, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(1)) + m := f.GetMetric()[0] + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(1))) + g.Expect(labelValue(m, "node")).To(Equal(nodeName)) + g.Expect(labelValue(m, "pool")).To(Equal(poolName)) + g.Expect(labelValue(m, "image")).To(Equal(testutil.ImageDigestRefA)) + g.Expect(labelValue(m, "digest")).To(Equal(testutil.DigestA)) + g.Expect(labelValue(m, "version")).To(Equal(version)) +} + +// Orphaned nodes (no BootcNodePool owner) still emit a metric with an empty pool label. +func TestCollectorNodeWithoutOwnerHasEmptyPool(t *testing.T) { + g := NewWithT(t) + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode("orphan", testutil.ImageDigestRefA, + testutil.WithBootedImage(testutil.ImageDigestRefA), + testutil.WithBootedDigest(testutil.DigestA), + ), + } + + c := newMetricsCollector(newFakeCache(nil, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(1)) + g.Expect(labelValue(f.GetMetric()[0], "pool")).To(Equal("")) +} + +func TestCollectorNodeWithoutBootedStatus(t *testing.T) { + g := NewWithT(t) + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode("no-boot", testutil.ImageDigestRefA), + } + + c := newMetricsCollector(newFakeCache(nil, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).To(BeNil()) +} + +func TestCollectorMultiplePools(t *testing.T) { + g := NewWithT(t) + + const ( + prodPool = "prod" + stagingPool = "staging" + ) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(prodPool, testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + *testutil.NewPool(stagingPool, testutil.ImageDigestRefB, testutil.WithNodeSelector(testutil.ControlPlaneLabels())), + } + pools[0].Status.NodeCount = 3 + pools[1].Status.NodeCount = 2 + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_pool_nodes_total") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(2)) + + for _, m := range f.GetMetric() { + switch labelValue(m, "pool") { + case prodPool: + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(3))) + case stagingPool: + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(2))) + default: + t.Errorf("unexpected pool label: %s", labelValue(m, "pool")) + } + } +} + +func TestCollectorEmptyCache(t *testing.T) { + g := NewWithT(t) + + c := newMetricsCollector(newFakeCache(nil, nil)) + families := collectMetrics(t, c) + + g.Expect(findFamily(families, "bootcoperator_pool_nodes_total")).To(BeNil()) + g.Expect(findFamily(families, "bootcoperator_node_booted_image_info")).To(BeNil()) +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index a85cb868..08c480d3 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -27,6 +27,7 @@ var ( testEnv *envtest.Environment k8sClient client.Client testReconciler *BootcNodePoolReconciler + testCache objectLister ) // TODO: TestMain starts envtest and the manager unconditionally, which means @@ -63,7 +64,7 @@ func TestMain(m *testing.M) { mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, Metrics: metricsserver.Options{ - BindAddress: "0", // disable metrics server in tests + BindAddress: "0", }, }) if err != nil { @@ -83,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + testCache = mgr.GetCache() + testReconciler = &BootcNodePoolReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go new file mode 100644 index 00000000..84cee19a --- /dev/null +++ b/test/e2e/metrics_test.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +const ( + metricsServiceName = "bootc-operator-metrics" + metricsReaderRole = "bootc-operator-metrics-reader" +) + +// TestMetricsEndpoint creates a pool, then scrapes the operator metrics +// endpoint from inside the cluster (the same way Prometheus would) and +// verifies that the expected pool and node metrics are present. +func TestMetricsEndpoint(t *testing.T) { + e2eutil.Providers(t, "bink") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + nodeName := env.AddNode(t) + + ctx := context.Background() + + pool := env.NewPool("metrics", env.NodeImageDigestedPullSpec()) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), + ) + + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageDigest())) + + t.Logf("Node %q is Idle; scraping metrics endpoint", nodeName) + + sa, crb := createMetricsScraper(t, g, ctx, env.Client, env.TestID()) + t.Cleanup(func() { + _ = env.Client.Delete(ctx, crb) + _ = env.Client.Delete(ctx, sa) + }) + + metricsOutput := scrapeMetrics(t, g, ctx, env.TestID(), sa.Name) + t.Logf("Scraped %d bytes from metrics endpoint", len(metricsOutput)) + + poolName := pool.Name + expectedMetrics := []string{ + fmt.Sprintf(`bootcoperator_pool_nodes_total{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_updated{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_updating{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_degraded{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_rollout_paused{pool="%s"}`, poolName), + "bootcoperator_node_booted_image_info{", + } + for _, metric := range expectedMetrics { + g.Expect(metricsOutput).To(ContainSubstring(metric), + "expected metric %q in output", metric) + } + + var bn bootcv1alpha1.BootcNode + g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + g.Expect(metricsOutput).To(ContainSubstring( + fmt.Sprintf(`node="%s"`, nodeName)), + "expected node label in bootcoperator_node_booted_image_info", + ) +} + +// createMetricsScraper creates a ServiceAccount and ClusterRoleBinding +// that grants it the metrics-reader ClusterRole. +func createMetricsScraper( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, + testID string, +) (*corev1.ServiceAccount, *rbacv1.ClusterRoleBinding) { + t.Helper() + + saName := testID + "-metrics-scraper" + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: testutil.OperatorNamespaceName, + }, + } + g.Expect(c.Create(ctx, sa)).To(Succeed()) + + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: metricsReaderRole, + }, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: saName, + Namespace: testutil.OperatorNamespaceName, + }}, + } + g.Expect(c.Create(ctx, crb)).To(Succeed()) + + return sa, crb +} + +// scrapeMetrics runs a curl pod with the scraper SA to hit the metrics +// endpoint from inside the cluster, the same way Prometheus would. +func scrapeMetrics( + t *testing.T, + g Gomega, + ctx context.Context, + testID string, + saName string, +) string { + t.Helper() + + kubeconfigPath := os.Getenv("KUBECONFIG") + metricsURL := fmt.Sprintf( + "https://%s.%s.svc:%d/metrics", + metricsServiceName, testutil.OperatorNamespaceName, 8443, + ) + + curlImage := "docker.io/curlimages/curl:latest" + podName := testID + "-curl" + args := []string{ + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "run", podName, + "--image=" + curlImage, + "--restart=Never", + "--overrides", fmt.Sprintf(`{ + "spec": { + "serviceAccountName": "%s", + "nodeSelector": { + "node-role.kubernetes.io/control-plane": "" + }, + "tolerations": [{ + "key": "node-role.kubernetes.io/control-plane", + "operator": "Exists", + "effect": "NoSchedule" + }], + "containers": [{ + "name": "curl", + "image": "%s", + "command": ["sh", "-c", + "curl -sf -k -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" %s" + ] + }] + } + }`, saName, curlImage, metricsURL), + } + + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred(), + "failed to create curl pod: %s", string(out)) + + t.Cleanup(func() { + del := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "delete", "pod", podName, "--ignore-not-found", + ) + _ = del.Run() + }) + + g.Eventually(func() (corev1.PodPhase, error) { + getCmd := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "get", "pod", podName, + "-o", "jsonpath={.status.phase}", + ) + out, err := getCmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("kubectl get pod: %s: %w", string(out), err) + } + return corev1.PodPhase(strings.TrimSpace(string(out))), nil + }).WithTimeout(2*time.Minute).WithPolling(3*time.Second).Should( + Equal(corev1.PodSucceeded), + "curl pod should complete successfully", + ) + + logCmd := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "logs", podName, + ) + logOut, err := logCmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred(), + "failed to read curl pod logs: %s", string(logOut)) + + return strings.TrimSpace(string(logOut)) +} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 90172488..ebd058b0 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -67,9 +67,11 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // the released image via Helm. Disable metrics because the + // released binary does not support --metrics-bind-address. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) + helmInstallOperator(t, chartPath, releasedImg, extraArgs, + "--set", "metrics.enabled=false") waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Released operator %s installed via Helm", releaseTag) @@ -175,7 +177,12 @@ func uninstallOperator( } // helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +func helmInstallOperator( + t *testing.T, + chartPath, img string, + extraArgs []string, + helmSetArgs ...string, +) { t.Helper() args := []string{ "install", helmReleaseName, chartPath, @@ -183,17 +190,24 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string "--create-namespace", } args = append(args, helmImageArgs(t, img, extraArgs)...) + args = append(args, helmSetArgs...) helmRun(t, args...) } // helmUpgradeOperator upgrades the operator via helm upgrade. -func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { +func helmUpgradeOperator( + t *testing.T, + chartPath, img string, + extraArgs []string, + helmSetArgs ...string, +) { t.Helper() args := []string{ "upgrade", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, } args = append(args, helmImageArgs(t, img, extraArgs)...) + args = append(args, helmSetArgs...) helmRun(t, args...) } @@ -255,7 +269,8 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { for _, arg := range c.Args { switch { case arg == "--leader-elect", - strings.HasPrefix(arg, "--health-probe-bind-address="): + strings.HasPrefix(arg, "--health-probe-bind-address="), + strings.HasPrefix(arg, "--metrics-bind-address="): default: extra = append(extra, arg) } diff --git a/test/util/builders.go b/test/util/builders.go index ef826a0f..badd3e5f 100644 --- a/test/util/builders.go +++ b/test/util/builders.go @@ -176,9 +176,30 @@ func NewNode(name, desiredImage string, opts ...NodeOption) *bootcv1alpha1.Bootc // WithBootedDigest sets the booted image digest in the node's status. func WithBootedDigest(digest string) NodeOption { return func(node *bootcv1alpha1.BootcNode) { - node.Status.Booted = &bootcv1alpha1.ImageInfo{ - ImageDigest: digest, + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} } + node.Status.Booted.ImageDigest = digest + } +} + +// WithBootedImage sets the booted image pullspec in the node's status. +func WithBootedImage(image string) NodeOption { + return func(node *bootcv1alpha1.BootcNode) { + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} + } + node.Status.Booted.Image = image + } +} + +// WithBootedVersion sets the booted image version in the node's status. +func WithBootedVersion(version string) NodeOption { + return func(node *bootcv1alpha1.BootcNode) { + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} + } + node.Status.Booted.Version = version } }