From e035eadc822274e4f2e31774f965c27cb1bc93bb Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Tue, 20 Oct 2026 14:11:31 +0000 Subject: [PATCH 1/8] feat: add Prometheus metrics collector for pools and nodes Expose pool-level gauges (nodes total, updated, updating, degraded, rollout paused) and a node-level info metric for the booted OS image. The collector reads from the controller-runtime cache on each scrape. Configure the metrics server with HTTPS and authentication/authorization on port 8443, with flags for bind address and TLS cert directory. Assisted-by: AI Signed-off-by: Alice Frosi --- cmd/controller/main.go | 31 +++++++ go.mod | 27 ++++++- go.sum | 53 ++++++++++++ internal/controller/metrics.go | 144 +++++++++++++++++++++++++++++++++ test/util/builders.go | 5 +- 5 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 internal/controller/metrics.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f15223d3..f45b88c8 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -15,6 +15,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" "github.com/bootc-dev/bootc-operator/internal/controller" @@ -47,6 +49,8 @@ func init() { func main() { var enableLeaderElection bool var probeAddr string + var metricsAddr string + var metricsCertDir string var tagResolutionInterval time.Duration var allowInsecureRegistry bool flag.StringVar( @@ -55,6 +59,18 @@ func main() { ":8081", "The address the probe endpoint binds to.", ) + flag.StringVar( + &metricsAddr, + "metrics-bind-address", + ":8443", + "The address the metrics endpoint binds to.", + ) + flag.StringVar( + &metricsCertDir, + "metrics-cert-dir", + "", + "The directory containing TLS certificates for the metrics server. If empty, self-signed certificates are generated.", + ) flag.DurationVar( &tagResolutionInterval, "tag-resolution-interval", @@ -88,9 +104,19 @@ func main() { version.GitCommit, ) + metricsOpts := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + } + if metricsCertDir != "" { + metricsOpts.CertDir = metricsCertDir + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, HealthProbeBindAddress: probeAddr, + Metrics: metricsOpts, LeaderElection: enableLeaderElection, LeaderElectionID: "bootc-operator.bootc.dev", }) @@ -99,6 +125,11 @@ func main() { os.Exit(1) } + if err := controller.RegisterMetrics(mgr.GetCache()); err != nil { + setupLog.Error(err, "Failed to register metrics collector") + os.Exit(1) + } + kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) if err != nil { setupLog.Error(err, "Failed to create Kubernetes clientset") diff --git a/go.mod b/go.mod index 7b8502a0..4f6a4748 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( github.com/go-logr/logr v1.4.4 github.com/google/go-containerregistry v0.22.1 github.com/onsi/gomega v1.44.0 + github.com/prometheus/client_golang v1.25.0 + github.com/prometheus/client_model v0.6.3 k8s.io/api v0.37.1 k8s.io/apiextensions-apiserver v0.37.1 k8s.io/apimachinery v0.37.1 @@ -22,8 +24,10 @@ require ( ) require ( + cel.dev/expr v0.25.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect @@ -38,6 +42,7 @@ require ( github.com/aws/smithy-go v1.28.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -46,8 +51,10 @@ require ( github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/go-openapi/jsonreference v1.0.0 // indirect @@ -64,9 +71,11 @@ require ( github.com/go-openapi/swag/typeutils v0.27.1 // indirect github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.1 // indirect @@ -81,8 +90,6 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.25.0 // indirect - github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.72.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -91,10 +98,20 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.23.0 // indirect @@ -103,15 +120,21 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gotest.tools/v3 v3.5.2 // indirect + k8s.io/apiserver v0.37.1 // indirect k8s.io/cli-runtime v0.37.1 // indirect k8s.io/component-base v0.37.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + k8s.io/streaming v0.37.1 // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.21.1 // indirect sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect diff --git a/go.sum b/go.sum index 82079f68..1c587040 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= @@ -40,6 +44,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= @@ -65,14 +71,19 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= @@ -111,8 +122,12 @@ github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAg github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.29.2 h1:ZtDxkeiMmz0mxbKDYiNkE5Lk7V5edMRcaaDf2jX002k= +github.com/google/cel-go v0.29.2/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -126,6 +141,8 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -196,6 +213,26 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -207,6 +244,8 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= @@ -228,6 +267,14 @@ golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -246,6 +293,8 @@ k8s.io/apiextensions-apiserver v0.37.1 h1:7fIQG8eThDSTVYBWg/DOpI8v5wYfKCo0N8/TDM k8s.io/apiextensions-apiserver v0.37.1/go.mod h1:1Q3ujWwHgnhBoGsdmGORflwK1sDFmuaqH1OsMDWk2Gs= k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= +k8s.io/apiserver v0.37.1 h1:dUsNwjsvItyVKNpC1N0AhQ0OBSiUJKfWz004vPifOe0= +k8s.io/apiserver v0.37.1/go.mod h1:EWXW6PntGfOC2uD9GGVobnOExgFoX1z0hZo7f7Rs7Ng= k8s.io/cli-runtime v0.37.1 h1:3mir5bM4XjMJHW3SMRk++3Ylf4YQne0XFFsW+IGT85U= k8s.io/cli-runtime v0.37.1/go.mod h1:g3VQOm71f//aNbp79g0az9jBRTrvPjAHz7WxVysbZ4M= k8s.io/client-go v0.37.1 h1:QTv/5ha4jAHtW9qxxVBkQVFBRDb4jHfFopQqqMdc+wM= @@ -258,8 +307,12 @@ k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3d k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= k8s.io/kubectl v0.37.1 h1:n1VCIntOJiX943uByXCThSZQmE+4v9jGS8S9u94/aHU= k8s.io/kubectl v0.37.1/go.mod h1:66cc4Bz8PxBTxlrHzBuzXEzGIytAnNasKQPeEcpuaV8= +k8s.io/streaming v0.37.1 h1:TpzVfQeFuVndn2g9mFqxy1UcUYPwDzqjUmwR/IzJCWc= +k8s.io/streaming v0.37.1/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ= sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 00000000..3ca3e1ad --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" +) + +type objectLister interface { + List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error +} + +var ( + poolNodesTotal = prometheus.NewDesc( + "bootcoperator_pool_nodes_total", + "Total number of nodes in a BootcNodePool.", + []string{"pool"}, nil, + ) + poolNodesUpdated = prometheus.NewDesc( + "bootcoperator_pool_nodes_updated", + "Number of nodes running the target digest.", + []string{"pool"}, nil, + ) + poolNodesUpdating = prometheus.NewDesc( + "bootcoperator_pool_nodes_updating", + "Number of nodes actively being updated.", + []string{"pool"}, nil, + ) + poolNodesDegraded = prometheus.NewDesc( + "bootcoperator_pool_nodes_degraded", + "Number of nodes in a degraded state.", + []string{"pool"}, nil, + ) + poolRolloutPaused = prometheus.NewDesc( + "bootcoperator_pool_rollout_paused", + "Whether rollout is paused for a BootcNodePool (0 or 1).", + []string{"pool"}, nil, + ) + nodeBootedImageInfo = prometheus.NewDesc( + "bootcoperator_node_booted_image_info", + "Info metric for the currently booted OS image on a node.", + []string{"node", "pool", "image", "digest", "version"}, nil, + ) +) + +type metricsCollector struct { + lister objectLister +} + +func newMetricsCollector(lister objectLister) *metricsCollector { + return &metricsCollector{lister: lister} +} + +func (c *metricsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- poolNodesTotal + ch <- poolNodesUpdated + ch <- poolNodesUpdating + ch <- poolNodesDegraded + ch <- poolRolloutPaused + ch <- nodeBootedImageInfo +} + +func (c *metricsCollector) Collect(ch chan<- prometheus.Metric) { + log := logf.Log.WithName("metrics") + ctx := context.Background() + + var pools bootcv1alpha1.BootcNodePoolList + if err := c.lister.List(ctx, &pools); err != nil { + return + } + + for i := range pools.Items { + pool := &pools.Items[i] + name := pool.Name + + emitGauge(ch, log, poolNodesTotal, float64(pool.Status.NodeCount), name) + emitGauge(ch, log, poolNodesUpdated, float64(pool.Status.UpdatedCount), name) + emitGauge(ch, log, poolNodesUpdating, float64(pool.Status.UpdatingCount), name) + emitGauge(ch, log, poolNodesDegraded, float64(pool.Status.DegradedCount), name) + + var paused float64 + if pool.Spec.Rollout != nil && pool.Spec.Rollout.Paused { + paused = 1 + } + emitGauge(ch, log, poolRolloutPaused, paused, name) + } + + var nodes bootcv1alpha1.BootcNodeList + if err := c.lister.List(ctx, &nodes); err != nil { + return + } + + for i := range nodes.Items { + node := &nodes.Items[i] + if node.Status.Booted == nil { + continue + } + + poolName := ownerPoolName(node) + emitGauge(ch, log, nodeBootedImageInfo, 1, + node.Name, + poolName, + node.Status.Booted.Image, + node.Status.Booted.ImageDigest, + node.Status.Booted.Version, + ) + } +} + +func emitGauge( + ch chan<- prometheus.Metric, + log logr.Logger, + desc *prometheus.Desc, + value float64, + labels ...string, +) { + m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, value, labels...) + if err != nil { + log.Error(err, "Failed to create metric", "metric", desc.String()) + return + } + ch <- m +} + +func RegisterMetrics(lister objectLister) error { + return metrics.Registry.Register(newMetricsCollector(lister)) +} + +func ownerPoolName(node *bootcv1alpha1.BootcNode) string { + for _, ref := range node.OwnerReferences { + if ref.Kind == "BootcNodePool" { + return ref.Name + } + } + return "" +} diff --git a/test/util/builders.go b/test/util/builders.go index ef826a0f..8650a19c 100644 --- a/test/util/builders.go +++ b/test/util/builders.go @@ -176,9 +176,10 @@ func NewNode(name, desiredImage string, opts ...NodeOption) *bootcv1alpha1.Bootc // WithBootedDigest sets the booted image digest in the node's status. func WithBootedDigest(digest string) NodeOption { return func(node *bootcv1alpha1.BootcNode) { - node.Status.Booted = &bootcv1alpha1.ImageInfo{ - ImageDigest: digest, + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} } + node.Status.Booted.ImageDigest = digest } } From f9b0a40690936e77674132fec2717b72b0ae9b7c Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Tue, 20 Oct 2026 14:11:38 +0000 Subject: [PATCH 2/8] test: add unit tests for metrics collector Cover pool gauges, paused/not-paused rollout, node info metric, orphan nodes, missing booted status, multiple pools, and empty cache. Assisted-by: AI Signed-off-by: Alice Frosi --- internal/controller/metrics_test.go | 248 ++++++++++++++++++++++++++++ test/util/builders.go | 20 +++ 2 files changed, 268 insertions(+) create mode 100644 internal/controller/metrics_test.go diff --git a/internal/controller/metrics_test.go b/internal/controller/metrics_test.go new file mode 100644 index 00000000..c9facc99 --- /dev/null +++ b/internal/controller/metrics_test.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +type fakeCache struct { + pools []bootcv1alpha1.BootcNodePool + nodes []bootcv1alpha1.BootcNode +} + +func newFakeCache(pools []bootcv1alpha1.BootcNodePool, nodes []bootcv1alpha1.BootcNode) *fakeCache { + return &fakeCache{pools: pools, nodes: nodes} +} + +func (f *fakeCache) List(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + switch v := list.(type) { + case *bootcv1alpha1.BootcNodePoolList: + v.Items = f.pools + case *bootcv1alpha1.BootcNodeList: + v.Items = f.nodes + default: + return fmt.Errorf("unexpected list type: %T", list) + } + return nil +} + +func collectMetrics(t *testing.T, c prometheus.Collector) []*dto.MetricFamily { + t.Helper() + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + t.Fatalf("failed to register collector: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + return families +} + +func findFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, f := range families { + if f.GetName() == name { + return f + } + } + return nil +} + +func labelValue(m *dto.Metric, name string) string { + for _, l := range m.GetLabel() { + if l.GetName() == name { + return l.GetValue() + } + } + return "" +} + +func TestCollectorPoolGauges(t *testing.T) { + g := NewWithT(t) + + const poolName = "prod" + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(poolName, testutil.ImageDigestRefA, + testutil.WithWorkerSelector(), + testutil.WithPaused(true), + ), + } + pools[0].Status = bootcv1alpha1.BootcNodePoolStatus{ + NodeCount: 5, + UpdatedCount: 3, + UpdatingCount: 1, + DegradedCount: 1, + } + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + tests := []struct { + name string + value float64 + }{ + {"bootcoperator_pool_nodes_total", 5}, + {"bootcoperator_pool_nodes_updated", 3}, + {"bootcoperator_pool_nodes_updating", 1}, + {"bootcoperator_pool_nodes_degraded", 1}, + {"bootcoperator_pool_rollout_paused", 1}, + } + + for _, tt := range tests { + f := findFamily(families, tt.name) + g.Expect(f).ToNot(BeNil(), "metric %s not found", tt.name) + g.Expect(f.GetMetric()).To(HaveLen(1)) + m := f.GetMetric()[0] + g.Expect(m.GetGauge().GetValue()).To(Equal(tt.value), "metric %s", tt.name) + g.Expect(labelValue(m, "pool")).To(Equal(poolName)) + } +} + +func TestCollectorPoolRolloutNotPaused(t *testing.T) { + g := NewWithT(t) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool("staging", testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + } + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_pool_rollout_paused") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()[0].GetGauge().GetValue()).To(Equal(float64(0))) +} + +func TestCollectorNodeInfoMetric(t *testing.T) { + g := NewWithT(t) + + const ( + poolName = "prod" + nodeName = "worker-1" + version = "1.0.0" + ) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(poolName, testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + } + pools[0].UID = "pool-uid-1" + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode(nodeName, testutil.ImageDigestRefA, + testutil.WithBootedImage(testutil.ImageDigestRefA), + testutil.WithBootedDigest(testutil.DigestA), + testutil.WithBootedVersion(version), + ), + } + nodes[0].OwnerReferences = []metav1.OwnerReference{{ + APIVersion: bootcv1alpha1.GroupVersion.String(), + Kind: "BootcNodePool", + Name: poolName, + UID: pools[0].UID, + }} + + c := newMetricsCollector(newFakeCache(pools, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(1)) + m := f.GetMetric()[0] + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(1))) + g.Expect(labelValue(m, "node")).To(Equal(nodeName)) + g.Expect(labelValue(m, "pool")).To(Equal(poolName)) + g.Expect(labelValue(m, "image")).To(Equal(testutil.ImageDigestRefA)) + g.Expect(labelValue(m, "digest")).To(Equal(testutil.DigestA)) + g.Expect(labelValue(m, "version")).To(Equal(version)) +} + +// Orphaned nodes (no BootcNodePool owner) still emit a metric with an empty pool label. +func TestCollectorNodeWithoutOwnerHasEmptyPool(t *testing.T) { + g := NewWithT(t) + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode("orphan", testutil.ImageDigestRefA, + testutil.WithBootedImage(testutil.ImageDigestRefA), + testutil.WithBootedDigest(testutil.DigestA), + ), + } + + c := newMetricsCollector(newFakeCache(nil, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(1)) + g.Expect(labelValue(f.GetMetric()[0], "pool")).To(Equal("")) +} + +func TestCollectorNodeWithoutBootedStatus(t *testing.T) { + g := NewWithT(t) + + nodes := []bootcv1alpha1.BootcNode{ + *testutil.NewNode("no-boot", testutil.ImageDigestRefA), + } + + c := newMetricsCollector(newFakeCache(nil, nodes)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_node_booted_image_info") + g.Expect(f).To(BeNil()) +} + +func TestCollectorMultiplePools(t *testing.T) { + g := NewWithT(t) + + const ( + prodPool = "prod" + stagingPool = "staging" + ) + + pools := []bootcv1alpha1.BootcNodePool{ + *testutil.NewPool(prodPool, testutil.ImageDigestRefA, testutil.WithWorkerSelector()), + *testutil.NewPool(stagingPool, testutil.ImageDigestRefB, testutil.WithNodeSelector(testutil.ControlPlaneLabels())), + } + pools[0].Status.NodeCount = 3 + pools[1].Status.NodeCount = 2 + + c := newMetricsCollector(newFakeCache(pools, nil)) + families := collectMetrics(t, c) + + f := findFamily(families, "bootcoperator_pool_nodes_total") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).To(HaveLen(2)) + + for _, m := range f.GetMetric() { + switch labelValue(m, "pool") { + case prodPool: + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(3))) + case stagingPool: + g.Expect(m.GetGauge().GetValue()).To(Equal(float64(2))) + default: + t.Errorf("unexpected pool label: %s", labelValue(m, "pool")) + } + } +} + +func TestCollectorEmptyCache(t *testing.T) { + g := NewWithT(t) + + c := newMetricsCollector(newFakeCache(nil, nil)) + families := collectMetrics(t, c) + + g.Expect(findFamily(families, "bootcoperator_pool_nodes_total")).To(BeNil()) + g.Expect(findFamily(families, "bootcoperator_node_booted_image_info")).To(BeNil()) +} diff --git a/test/util/builders.go b/test/util/builders.go index 8650a19c..badd3e5f 100644 --- a/test/util/builders.go +++ b/test/util/builders.go @@ -183,6 +183,26 @@ func WithBootedDigest(digest string) NodeOption { } } +// WithBootedImage sets the booted image pullspec in the node's status. +func WithBootedImage(image string) NodeOption { + return func(node *bootcv1alpha1.BootcNode) { + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} + } + node.Status.Booted.Image = image + } +} + +// WithBootedVersion sets the booted image version in the node's status. +func WithBootedVersion(version string) NodeOption { + return func(node *bootcv1alpha1.BootcNode) { + if node.Status.Booted == nil { + node.Status.Booted = &bootcv1alpha1.ImageInfo{} + } + node.Status.Booted.Version = version + } +} + // WithNodeCondition appends a condition to the node's status. func WithNodeCondition(condType string, status metav1.ConditionStatus, reason string) NodeOption { return func(node *bootcv1alpha1.BootcNode) { From c0cded011e4a407093a9ea92e3521bb273d20506 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Tue, 20 Oct 2026 14:11:47 +0000 Subject: [PATCH 3/8] feat: add metrics Service, ServiceMonitor, and RBAC Add kustomize and Helm chart resources for the metrics endpoint: a Service targeting port 8443, an opt-in ServiceMonitor, RBAC for the operator SA to perform token/access reviews, and a metrics-reader ClusterRole for Prometheus scraping. Assisted-by: AI Signed-off-by: Alice Frosi --- .../templates/metrics-auth-rbac.yaml | 36 +++++++++++++++++++ .../templates/metrics-reader-rbac.yaml | 13 +++++++ .../templates/metrics-service.yaml | 18 ++++++++++ .../templates/metrics-servicemonitor.yaml | 26 ++++++++++++++ chart/bootc-operator/values.yaml | 11 ++++++ config/manager/manager.yaml | 4 +++ config/rbac/kustomization.yaml | 3 ++ config/rbac/metrics_auth_role.yaml | 17 +++++++++ config/rbac/metrics_auth_role_binding.yaml | 12 +++++++ config/rbac/metrics_reader_role.yaml | 9 +++++ 10 files changed, 149 insertions(+) create mode 100644 chart/bootc-operator/templates/metrics-auth-rbac.yaml create mode 100644 chart/bootc-operator/templates/metrics-reader-rbac.yaml create mode 100644 chart/bootc-operator/templates/metrics-service.yaml create mode 100644 chart/bootc-operator/templates/metrics-servicemonitor.yaml create mode 100644 config/rbac/metrics_auth_role.yaml create mode 100644 config/rbac/metrics_auth_role_binding.yaml create mode 100644 config/rbac/metrics_reader_role.yaml diff --git a/chart/bootc-operator/templates/metrics-auth-rbac.yaml b/chart/bootc-operator/templates/metrics-auth-rbac.yaml new file mode 100644 index 00000000..d6b9eefa --- /dev/null +++ b/chart/bootc-operator/templates/metrics-auth-rbac.yaml @@ -0,0 +1,36 @@ +{{- if .Values.metrics.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-auth + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-auth + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "bootc-operator.fullname" . }}-metrics-auth +subjects: + - kind: ServiceAccount + name: {{ include "bootc-operator.fullname" . }}-controller + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-reader-rbac.yaml b/chart/bootc-operator/templates/metrics-reader-rbac.yaml new file mode 100644 index 00000000..ee02b6ba --- /dev/null +++ b/chart/bootc-operator/templates/metrics-reader-rbac.yaml @@ -0,0 +1,13 @@ +{{- if .Values.metrics.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics-reader + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +rules: + - nonResourceURLs: + - /metrics + verbs: + - get +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-service.yaml b/chart/bootc-operator/templates/metrics-service.yaml new file mode 100644 index 00000000..48a91945 --- /dev/null +++ b/chart/bootc-operator/templates/metrics-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "bootc-operator.fullname" . }}-metrics + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: controller +spec: + ports: + - name: https + port: {{ .Values.metrics.service.port }} + targetPort: metrics + protocol: TCP + selector: + app.kubernetes.io/name: {{ include "bootc-operator.name" . }} + app.kubernetes.io/component: controller +{{- end }} diff --git a/chart/bootc-operator/templates/metrics-servicemonitor.yaml b/chart/bootc-operator/templates/metrics-servicemonitor.yaml new file mode 100644 index 00000000..085166b5 --- /dev/null +++ b/chart/bootc-operator/templates/metrics-servicemonitor.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "bootc-operator.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "bootc-operator.labels" . | nindent 4 }} +spec: + endpoints: + - port: https + scheme: https + interval: {{ .Values.metrics.serviceMonitor.interval }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + app.kubernetes.io/name: {{ include "bootc-operator.name" . }} + app.kubernetes.io/component: controller + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/values.yaml b/chart/bootc-operator/values.yaml index 9aaacc54..cec56320 100644 --- a/chart/bootc-operator/values.yaml +++ b/chart/bootc-operator/values.yaml @@ -10,3 +10,14 @@ imagePullSecrets: [] controller: extraArgs: [] + +metrics: + enabled: true + service: + port: 8443 + serviceMonitor: + # When enabled, bind the metrics-reader ClusterRole to your + # Prometheus ServiceAccount so it can access /metrics. + enabled: false + interval: 30s + namespace: "" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e34d6944..e867509b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -70,6 +70,7 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8443 image: ghcr.io/bootc-dev/bootc-operator:latest name: manager env: @@ -78,6 +79,9 @@ spec: fieldRef: fieldPath: metadata.namespace ports: + - containerPort: 8443 + name: metrics + protocol: TCP - containerPort: 8081 name: health protocol: TCP diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 809aa028..d248afd5 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,3 +7,6 @@ resources: - daemon_service_account.yaml - daemon_role.yaml - daemon_role_binding.yaml +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000..32d2e4ec --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000..e775d67f --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000..03ce329d --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get From e9a14966eb851909973fda07995b814b82509d88 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Tue, 20 Oct 2026 14:11:55 +0000 Subject: [PATCH 4/8] test: add envtest tests for metrics collector Extend the envtest suite to expose the manager cache and add two tests that create pools and nodes against the envtest API server, then verify the collector emits the expected metrics. Assisted-by: AI Signed-off-by: Alice Frosi --- internal/controller/metrics_envtest_test.go | 134 ++++++++++++++++++++ internal/controller/suite_test.go | 5 +- 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 internal/controller/metrics_envtest_test.go diff --git a/internal/controller/metrics_envtest_test.go b/internal/controller/metrics_envtest_test.go new file mode 100644 index 00000000..6f4bbc99 --- /dev/null +++ b/internal/controller/metrics_envtest_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +func TestMetricsFromEnvtestCache(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest not available") + } + if testCache == nil { + t.Skip("cache not available") + } + g := NewWithT(t) + ctx := context.Background() + + pool := testutil.NewPool( + "metrics-envtest", + testutil.ImageDigestRefA, + testutil.WithWorkerSelector(), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() (int64, error) { + var fetched bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched) + return fetched.Status.ObservedGeneration, err + }).WithTimeout(10 * time.Second).WithPolling(200 * time.Millisecond).Should(Equal(pool.Generation)) + + reg := prometheus.NewPedanticRegistry() + collector := newMetricsCollector(testCache) + g.Expect(reg.Register(collector)).To(Succeed()) + + families, err := reg.Gather() + g.Expect(err).ToNot(HaveOccurred()) + + f := findFamily(families, "bootcoperator_pool_nodes_total") + g.Expect(f).ToNot(BeNil()) + g.Expect(f.GetMetric()).ToNot(BeEmpty()) + + found := false + for _, m := range f.GetMetric() { + if labelValue(m, "pool") == pool.Name { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "expected pool %q in metrics", pool.Name) +} + +func TestMetricsNodeInfoFromEnvtestCache(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest not available") + } + if testCache == nil { + t.Skip("cache not available") + } + g := NewWithT(t) + ctx := context.Background() + + pool := testutil.NewPool("metrics-node-envtest", testutil.ImageDigestRefA, + testutil.WithNodeSelector(map[string]string{"envtest-metrics": "true"}), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() (string, error) { + var fetched bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched) + return fetched.Name, err + }).WithTimeout(10 * time.Second).WithPolling(200 * time.Millisecond).Should(Equal(pool.Name)) + + var fetched bootcv1alpha1.BootcNodePool + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &fetched)).To(Succeed()) + + node := &bootcv1alpha1.BootcNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-envtest-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: bootcv1alpha1.GroupVersion.String(), + Kind: "BootcNodePool", + Name: fetched.Name, + UID: fetched.UID, + }}, + }, + Spec: bootcv1alpha1.BootcNodeSpec{ + DesiredImage: testutil.ImageDigestRefA, + DesiredImageState: bootcv1alpha1.DesiredImageStateStaged, + }, + } + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + node.Status.Booted = &bootcv1alpha1.ImageInfo{ + Image: testutil.ImageDigestRefA, + ImageDigest: testutil.DigestA, + Version: "1.0.0", + } + g.Expect(k8sClient.Status().Update(ctx, node)).To(Succeed()) + + reg := prometheus.NewPedanticRegistry() + collector := newMetricsCollector(testCache) + g.Expect(reg.Register(collector)).To(Succeed()) + + g.Eventually(func() (string, error) { + families, err := reg.Gather() + if err != nil { + return "", err + } + f := findFamily(families, "bootcoperator_node_booted_image_info") + if f == nil { + return "", nil + } + for _, m := range f.GetMetric() { + if labelValue(m, "node") == node.Name { + return labelValue(m, "pool"), nil + } + } + return "", nil + }).WithTimeout(10 * time.Second).WithPolling(500 * time.Millisecond).Should(Equal(pool.Name)) +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index a85cb868..08c480d3 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -27,6 +27,7 @@ var ( testEnv *envtest.Environment k8sClient client.Client testReconciler *BootcNodePoolReconciler + testCache objectLister ) // TODO: TestMain starts envtest and the manager unconditionally, which means @@ -63,7 +64,7 @@ func TestMain(m *testing.M) { mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, Metrics: metricsserver.Options{ - BindAddress: "0", // disable metrics server in tests + BindAddress: "0", }, }) if err != nil { @@ -83,6 +84,8 @@ func TestMain(m *testing.M) { os.Exit(1) } + testCache = mgr.GetCache() + testReconciler = &BootcNodePoolReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), From 0363f0e24ff7c5d9459733833439e24645951814 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Wed, 21 Oct 2026 10:27:21 +0000 Subject: [PATCH 5/8] test: add e2e test for metrics endpoint scraping Create a ServiceAccount with the metrics-reader ClusterRole and run a curl pod to scrape the metrics endpoint from inside the cluster, the same way Prometheus would. Verify pool and node metrics are present after a pool reaches steady state. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/metrics_test.go | 209 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 test/e2e/metrics_test.go diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go new file mode 100644 index 00000000..30605293 --- /dev/null +++ b/test/e2e/metrics_test.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +const ( + metricsServiceName = "bootc-operator-metrics" + metricsReaderRole = "bootc-operator-metrics-reader" +) + +// TestMetricsEndpoint creates a pool, then scrapes the operator metrics +// endpoint from inside the cluster (the same way Prometheus would) and +// verifies that the expected pool and node metrics are present. +func TestMetricsEndpoint(t *testing.T) { + e2eutil.Providers(t, "bink") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + nodeName := env.AddNode(t) + + ctx := context.Background() + + pool := env.NewPool("metrics", env.NodeImageDigestedPullSpec()) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute, + HaveField("Booted", imageMatchesDigest(env.NodeImageDigest())), + ) + + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageDigest())) + + t.Logf("Node %q is Idle; scraping metrics endpoint", nodeName) + + sa, crb := createMetricsScraper(t, g, ctx, env.Client, env.TestID()) + t.Cleanup(func() { + _ = env.Client.Delete(ctx, crb) + _ = env.Client.Delete(ctx, sa) + }) + + metricsOutput := scrapeMetrics(t, g, ctx, env.TestID(), sa.Name) + t.Logf("Scraped %d bytes from metrics endpoint", len(metricsOutput)) + + poolName := pool.Name + expectedMetrics := []string{ + fmt.Sprintf(`bootcoperator_pool_nodes_total{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_updated{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_updating{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_nodes_degraded{pool="%s"}`, poolName), + fmt.Sprintf(`bootcoperator_pool_rollout_paused{pool="%s"}`, poolName), + "bootcoperator_node_booted_image_info{", + } + for _, metric := range expectedMetrics { + g.Expect(metricsOutput).To(ContainSubstring(metric), + "expected metric %q in output", metric) + } + + var bn bootcv1alpha1.BootcNode + g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + g.Expect(metricsOutput).To(ContainSubstring( + fmt.Sprintf(`node="%s"`, nodeName)), + "expected node label in bootcoperator_node_booted_image_info", + ) +} + +// createMetricsScraper creates a ServiceAccount and ClusterRoleBinding +// that grants it the metrics-reader ClusterRole. +func createMetricsScraper( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, + testID string, +) (*corev1.ServiceAccount, *rbacv1.ClusterRoleBinding) { + t.Helper() + + saName := testID + "-metrics-scraper" + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: testutil.OperatorNamespaceName, + }, + } + g.Expect(c.Create(ctx, sa)).To(Succeed()) + + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: metricsReaderRole, + }, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: saName, + Namespace: testutil.OperatorNamespaceName, + }}, + } + g.Expect(c.Create(ctx, crb)).To(Succeed()) + + return sa, crb +} + +// scrapeMetrics runs a curl pod with the scraper SA to hit the metrics +// endpoint from inside the cluster, the same way Prometheus would. +func scrapeMetrics( + t *testing.T, + g Gomega, + ctx context.Context, + testID string, + saName string, +) string { + t.Helper() + + kubeconfigPath := os.Getenv("KUBECONFIG") + metricsURL := fmt.Sprintf( + "https://%s.%s.svc:%d/metrics", + metricsServiceName, testutil.OperatorNamespaceName, 8443, + ) + + curlImage := "docker.io/curlimages/curl:latest" + podName := testID + "-curl" + args := []string{ + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "run", podName, + "--image=" + curlImage, + "--restart=Never", + "--overrides", fmt.Sprintf(`{ + "spec": { + "serviceAccountName": "%s", + "containers": [{ + "name": "curl", + "image": "%s", + "command": ["sh", "-c", + "curl -sf -k -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" %s" + ] + }] + } + }`, saName, curlImage, metricsURL), + } + + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred(), + "failed to create curl pod: %s", string(out)) + + t.Cleanup(func() { + del := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "delete", "pod", podName, "--ignore-not-found", + ) + _ = del.Run() + }) + + g.Eventually(func() (corev1.PodPhase, error) { + getCmd := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "get", "pod", podName, + "-o", "jsonpath={.status.phase}", + ) + out, err := getCmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("kubectl get pod: %s: %w", string(out), err) + } + return corev1.PodPhase(strings.TrimSpace(string(out))), nil + }).WithTimeout(2*time.Minute).WithPolling(3*time.Second).Should( + Equal(corev1.PodSucceeded), + "curl pod should complete successfully", + ) + + logCmd := exec.CommandContext(ctx, "kubectl", + "--kubeconfig", kubeconfigPath, + "-n", testutil.OperatorNamespaceName, + "logs", podName, + ) + logOut, err := logCmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred(), + "failed to read curl pod logs: %s", string(logOut)) + + return strings.TrimSpace(string(logOut)) +} From b7d8f4432d6b5bdc46607c9810b817214777f4f0 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Wed, 21 Oct 2026 14:51:19 +0200 Subject: [PATCH 6/8] docs: add docs for monitoring Document how to configure the monitoring using Prometheus and Grafana. Assisted-by: AI Signed-off-by: Alice Frosi --- docs/src/SUMMARY.md | 1 + docs/src/helm.md | 5 ++ docs/src/operations/monitoring.md | 84 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 docs/src/operations/monitoring.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index fbfaf433..efb6f73a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,3 +8,4 @@ - [Helm Chart](helm.md) - [Operations](operations/index.md) - [Managing a pool](operations/pool.md) + - [Monitoring](operations/monitoring.md) diff --git a/docs/src/helm.md b/docs/src/helm.md index 91a73e49..df17a3ec 100644 --- a/docs/src/helm.md +++ b/docs/src/helm.md @@ -141,3 +141,8 @@ make helm | `image.pullPolicy` | not set | Kubernetes image pull policy | | `imagePullSecrets` | `[]` | List of pull secret references | | `controller.extraArgs` | `[]` | Extra arguments for the controller | +| `metrics.enabled` | `true` | Deploy the metrics Service | +| `metrics.service.port` | `8443` | Service port for the metrics endpoint | +| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource | +| `metrics.serviceMonitor.interval` | `30s` | Scrape interval | +| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor | diff --git a/docs/src/operations/monitoring.md b/docs/src/operations/monitoring.md new file mode 100644 index 00000000..380777b3 --- /dev/null +++ b/docs/src/operations/monitoring.md @@ -0,0 +1,84 @@ +# Monitoring + +The Bootc Operator exposes Prometheus metrics on a secure HTTPS +endpoint. This guide covers how to set up Prometheus and Grafana to +scrape and visualize the operator metrics. + +## Exposed metrics + +The operator exposes the following custom metrics: + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `bootcoperator_pool_nodes_total` | Gauge | `pool` | Total number of nodes in a BootcNodePool | +| `bootcoperator_pool_nodes_updated` | Gauge | `pool` | Nodes running the target digest | +| `bootcoperator_pool_nodes_updating` | Gauge | `pool` | Nodes actively being updated | +| `bootcoperator_pool_nodes_degraded` | Gauge | `pool` | Nodes in a degraded state | +| `bootcoperator_pool_rollout_paused` | Gauge | `pool` | Whether rollout is paused (0 or 1) | +| `bootcoperator_node_booted_image_info` | Gauge | `node`, `pool`, `image`, `digest`, `version` | Info metric for the booted OS image | + +In addition to these custom metrics, the operator exposes standard +controller-runtime metrics (workqueue depth, reconcile duration, etc.). + +## Enabling the ServiceMonitor + +The Helm chart includes a `ServiceMonitor` resource that tells +Prometheus where to scrape. Enable it during install or upgrade: + +```bash +helm upgrade --install bootc-operator \ + oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator \ + --namespace bootc-operator \ + --set metrics.serviceMonitor.enabled=true +``` + +Verify the ServiceMonitor was created: + +```bash +kubectl -n bootc-operator get servicemonitor +``` + +## Installing Prometheus and Grafana + +The recommended way to deploy Prometheus and Grafana is the +[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) +Helm chart: + +```bash +helm repo add prometheus-community \ + https://prometheus-community.github.io/helm-charts +helm repo update + +helm install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring --create-namespace \ + --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +``` + +Setting `serviceMonitorSelectorNilUsesHelmValues=false` allows +Prometheus to discover ServiceMonitors from all namespaces, including +`bootc-operator`. + +## Granting Prometheus access to the metrics endpoint + +The operator metrics endpoint requires authentication. The Helm chart +creates a `ClusterRole` called `-metrics-reader` that grants +`GET` access to `/metrics`. Bind it to the Prometheus ServiceAccount: + +```bash +kubectl create clusterrolebinding prometheus-metrics-reader \ + --clusterrole=bootc-operator-metrics-reader \ + --serviceaccount=monitoring:monitoring-kube-prometheus-prometheus +``` + +Adjust the ServiceAccount name if your Prometheus installation uses a +different one. + +## Helm values reference + +| Key | Default | Description | +|-----|---------|-------------| +| `metrics.enabled` | `true` | Deploy the metrics Service | +| `metrics.service.port` | `8443` | Service port for the metrics endpoint | +| `metrics.serviceMonitor.enabled` | `false` | Create a ServiceMonitor resource | +| `metrics.serviceMonitor.interval` | `30s` | Scrape interval | +| `metrics.serviceMonitor.namespace` | `""` | Namespace for the ServiceMonitor (defaults to release namespace) | From 078bf8d3c17b4c75d24cc5ac2b3bbbe148c3384d Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 22 Oct 2026 09:00:31 +0000 Subject: [PATCH 7/8] fix: gate metrics flag and port on metrics.enabled in Helm chart The released operator binary does not support --metrics-bind-address, so deploying it with the current Helm chart crashes the operator. Gate the flag and metrics port in sync-helm.sh on metrics.enabled, and disable metrics when installing the released image in the upgrade test. Also filter --metrics-bind-address from controllerExtraArgs to prevent duplication. Assisted-by: AI Signed-off-by: Alice Frosi --- hack/sync-helm.sh | 14 ++++++++++++++ test/e2e/upgrade_test.go | 25 ++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/hack/sync-helm.sh b/hack/sync-helm.sh index f55df976..246a883d 100755 --- a/hack/sync-helm.sh +++ b/hack/sync-helm.sh @@ -90,6 +90,20 @@ sed -i '/^ image: {{ include "bootc-operator\.image" \. }}/i\ - {{ . | quote }}\ {{- end }}' "$CHART_DIR/templates/controller-deployment.yaml" +# Make --metrics-bind-address conditional on metrics.enabled +sed -i 's|^\( - --metrics-bind-address=\):8443$| {{- if .Values.metrics.enabled }}\n\1:{{ .Values.metrics.service.port }}\n {{- end }}|' \ + "$CHART_DIR/templates/controller-deployment.yaml" + +# Make metrics containerPort conditional on metrics.enabled +python3 -c " +import re, sys +text = open(sys.argv[1]).read() +pattern = r' - containerPort: 8443\n name: metrics\n protocol: TCP\n' +repl = ' {{- if .Values.metrics.enabled }}\n - containerPort: {{ .Values.metrics.service.port }}\n name: metrics\n protocol: TCP\n {{- end }}\n' +text = re.sub(pattern, repl, text) +open(sys.argv[1], 'w').write(text) +" "$CHART_DIR/templates/controller-deployment.yaml" + gen_workload "$CONFIG_DIR/daemon/daemon.yaml" DaemonSet daemon \ "$CHART_DIR/templates/daemon-daemonset.yaml" echo "Synced workloads" diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 90172488..ebd058b0 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -67,9 +67,11 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // the released image via Helm. Disable metrics because the + // released binary does not support --metrics-bind-address. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) + helmInstallOperator(t, chartPath, releasedImg, extraArgs, + "--set", "metrics.enabled=false") waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Released operator %s installed via Helm", releaseTag) @@ -175,7 +177,12 @@ func uninstallOperator( } // helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +func helmInstallOperator( + t *testing.T, + chartPath, img string, + extraArgs []string, + helmSetArgs ...string, +) { t.Helper() args := []string{ "install", helmReleaseName, chartPath, @@ -183,17 +190,24 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string "--create-namespace", } args = append(args, helmImageArgs(t, img, extraArgs)...) + args = append(args, helmSetArgs...) helmRun(t, args...) } // helmUpgradeOperator upgrades the operator via helm upgrade. -func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { +func helmUpgradeOperator( + t *testing.T, + chartPath, img string, + extraArgs []string, + helmSetArgs ...string, +) { t.Helper() args := []string{ "upgrade", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, } args = append(args, helmImageArgs(t, img, extraArgs)...) + args = append(args, helmSetArgs...) helmRun(t, args...) } @@ -255,7 +269,8 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { for _, arg := range c.Args { switch { case arg == "--leader-elect", - strings.HasPrefix(arg, "--health-probe-bind-address="): + strings.HasPrefix(arg, "--health-probe-bind-address="), + strings.HasPrefix(arg, "--metrics-bind-address="): default: extra = append(extra, arg) } From be5de9cfea9e4606bebac43bb75903c3bc8aeec0 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 22 Oct 2026 10:49:30 +0000 Subject: [PATCH 8/8] fix: schedule metrics curl pod on the controller node The curl pod used to scrape the metrics endpoint was scheduled on worker nodes, where Calico CNI is not fully initialized. This caused FailedCreatePodSandBox errors and curl failures on K8s 1.36 and 1.37, while 1.35 passed only due to timing. Pin the curl pod to the control-plane node with a nodeSelector and toleration so it runs where networking is stable and the metrics service is reachable. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/metrics_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go index 30605293..84cee19a 100644 --- a/test/e2e/metrics_test.go +++ b/test/e2e/metrics_test.go @@ -154,6 +154,14 @@ func scrapeMetrics( "--overrides", fmt.Sprintf(`{ "spec": { "serviceAccountName": "%s", + "nodeSelector": { + "node-role.kubernetes.io/control-plane": "" + }, + "tolerations": [{ + "key": "node-role.kubernetes.io/control-plane", + "operator": "Exists", + "effect": "NoSchedule" + }], "containers": [{ "name": "curl", "image": "%s",