From 7e0c5cf83d415130d1b12bb29f409315d88a82e6 Mon Sep 17 00:00:00 2001 From: Danielle Mustillo Date: Tue, 17 Nov 2026 14:10:14 -0500 Subject: [PATCH 1/3] atecontroller: scope controller cache to deployments managed by workerpools The WorkerPool controller currently caches all Deployments in a cluster. Therefore the controller's memory incorrectly grows proportionately to the number of Deployments in the cluster, not to the size of the resources managed by agent-substrate as is expected. Before this change, on very large clusters with thousands of unrelated deployments, the controller's memory can grow to consume hundreds megabytes needlessly. This commit changes the ate-controller cache to only cache Deployments with the label "ate.dev/worker-pool", a label already added to every Deployment managed by the WorkerPool controller. Signed-off-by: Danielle Mustillo --- .../internal/controllers/cache.go | 46 +++++++++ .../internal/controllers/cache_test.go | 99 +++++++++++++++++++ .../controllers/egressmitmtrust_controller.go | 2 +- .../internal/controllers/workerpool_apply.go | 3 +- .../controllers/workerpool_controller.go | 2 + .../controllers/workerpool_controller_test.go | 4 + cmd/atecontroller/main.go | 16 +-- cmd/atecontroller/main_test.go | 33 +++++-- 8 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 cmd/atecontroller/internal/controllers/cache.go create mode 100644 cmd/atecontroller/internal/controllers/cache_test.go diff --git a/cmd/atecontroller/internal/controllers/cache.go b/cmd/atecontroller/internal/controllers/cache.go new file mode 100644 index 0000000000..a251059ef9 --- /dev/null +++ b/cmd/atecontroller/internal/controllers/cache.go @@ -0,0 +1,46 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CacheOptions limits the manager cache to the egressMITMCAPool Secret and to +// Deployments that carry workerPoolLabel. Other kinds are cached in full. +func CacheOptions(egressMITMCAPool types.NamespacedName) cache.Options { + hasWorkerPoolLabel, err := labels.NewRequirement(workerPoolLabel, selection.Exists, nil) + utilruntime.Must(err) + return cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Namespaces: map[string]cache.Config{ + egressMITMCAPool.Namespace: { + FieldSelector: fields.OneTermEqualSelector("metadata.name", egressMITMCAPool.Name), + }, + }, + }, + &appsv1.Deployment{}: {Label: labels.NewSelector().Add(*hasWorkerPoolLabel)}, + }, + } +} diff --git a/cmd/atecontroller/internal/controllers/cache_test.go b/cmd/atecontroller/internal/controllers/cache_test.go new file mode 100644 index 0000000000..94f5b12101 --- /dev/null +++ b/cmd/atecontroller/internal/controllers/cache_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TestDeploymentCacheScope checks that the manager cache holds a pool's +// Deployment and no Deployment without the ate.dev/worker-pool label. +func TestDeploymentCacheScope(t *testing.T) { + t.Parallel() + ctx := t.Context() + const label = "ate.dev/worker-pool" + wp := makeWorkerPool("cache-scope", "default", 2, "worker:v1") + + // The bystander is created first, so a cache of every Deployment holds it + // before it holds the labeled Deployment of the pool. + bystander := applyDeploymentWithoutLabels(t, "cache-scope-bystander", wp.Namespace, map[string]string{"app": "bystander"}, "someone-else") + applyDeploymentWithoutLabels(t, wp.Name, wp.Namespace, map[string]string{label: wp.Name}, workerPoolFieldOwner) + + if err := k8sClient.Create(ctx, wp); err != nil { + t.Fatalf("create WorkerPool: %v", err) + } + deleteOnCleanup(t, wp) + + eventually(t, func(ctx context.Context) (bool, error) { + dep := &appsv1.Deployment{} + if err := cachedClient.Get(ctx, client.ObjectKeyFromObject(wp), dep); err != nil { + return false, nil + } + return dep.Labels[label] == wp.Name && metav1.IsControlledBy(dep, wp), nil + }) + + err := cachedClient.Get(ctx, client.ObjectKeyFromObject(bystander), &appsv1.Deployment{}) + if !k8errors.IsNotFound(err) { + t.Errorf("cached read of a Deployment without %s: err = %v, want NotFound", label, err) + } + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(bystander), &appsv1.Deployment{}); err != nil { + t.Fatalf("live read of the bystander Deployment: %v", err) + } + + dep, err := getDeployment(ctx, wp) + if err != nil { + t.Fatalf("get Deployment: %v", err) + } + strip := client.RawPatch(types.MergePatchType, []byte(`{"metadata":{"labels":{"`+label+`":null}}}`)) + if err := k8sClient.Patch(ctx, dep, strip); err != nil { + t.Fatalf("strip %s: %v", label, err) + } + if _, has := dep.Labels[label]; has { + t.Fatalf("label %s survived the patch", label) + } + eventually(t, func(ctx context.Context) (bool, error) { + dep, err := getDeployment(ctx, wp) + return err == nil && dep.Labels[label] == wp.Name, nil + }) +} + +// applyDeploymentWithoutLabels applies a Deployment whose metadata carries no +// labels, and deletes it on cleanup. +func applyDeploymentWithoutLabels(t *testing.T, name, namespace string, podLabels map[string]string, fieldOwner string) *appsv1.Deployment { + t.Helper() + ac := appsv1ac.Deployment(name, namespace). + WithSpec(appsv1ac.DeploymentSpec(). + WithSelector(metav1ac.LabelSelector().WithMatchLabels(podLabels)). + WithTemplate(corev1ac.PodTemplateSpec(). + WithLabels(podLabels). + WithSpec(corev1ac.PodSpec(). + WithContainers(corev1ac.Container().WithName("worker").WithImage("worker:v0"))))) + if err := k8sClient.Apply(t.Context(), ac, client.FieldOwner(fieldOwner)); err != nil { + t.Fatalf("apply Deployment %s: %v", name, err) + } + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}} + deleteOnCleanup(t, dep) + return dep +} diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index fe5bfd22a3..36d91ee173 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -58,7 +58,7 @@ func EgressMITMCAPoolRef(systemNamespace string) types.NamespacedName { } // resourceNames covers list and watch because the manager's cache selects the -// Secret by metadata.name (managerOptions in main.go), which the apiserver +// Secret by metadata.name (CacheOptions), which the apiserver // authorizes as a request for that one name. Without the selector the grant // would deny them. // diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 1244d6498a..356c5a4430 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -93,7 +93,8 @@ type workerOTelSettings struct { MetricsExporter string } -// workerPoolLabel names the WorkerPool on each of its worker pods. +// workerPoolLabel names the WorkerPool on its Deployment, worker pods and +// NetworkPolicy. CacheOptions caches only Deployments that carry it. const workerPoolLabel = "ate.dev/worker-pool" const ( diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 23b1906a4f..eadd53fe30 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -163,6 +163,8 @@ func (r *WorkerPoolReconciler) reconcileWorkerPool(ctx context.Context, wp *atev log := log.FromContext(ctx) log.Info("Reconciling worker pool") + // The cache holds only Deployments with workerPoolLabel (CacheOptions). One + // without it reads as not found here, and the apply at the end labels it. existing := &appsv1.Deployment{} if err := r.Get(ctx, types.NamespacedName{Name: wp.Name, Namespace: wp.Namespace}, existing); err != nil { if !k8errors.IsNotFound(err) { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 3b6d3502ba..5bfe2604dc 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -53,6 +53,8 @@ import ( var ( cfg *rest.Config k8sClient client.Client + // cachedClient reads from the manager cache, as the reconcilers do. + cachedClient client.Client ) func TestMain(m *testing.M) { @@ -74,11 +76,13 @@ func TestMain(m *testing.M) { Scheme: scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", + Cache: CacheOptions(EgressMITMCAPoolRef(installdefaults.SystemNamespace)), }) if err != nil { fmt.Fprintf(os.Stderr, "manager creation failed: %v\n", err) os.Exit(1) } + cachedClient = mgr.GetClient() if err := (&WorkerPoolReconciler{ Client: mgr.GetClient(), diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d99fb97c81..d9fb560fe6 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -36,16 +36,12 @@ import ( "go.ate.dev/substrate/pkg/client/informers/externalversions" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -134,17 +130,7 @@ func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) ctrl. return ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsBindAddr(servePull)}, - Cache: cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Namespaces: map[string]cache.Config{ - egressMITMCAPool.Namespace: { - FieldSelector: fields.OneTermEqualSelector("metadata.name", egressMITMCAPool.Name), - }, - }, - }, - }, - }, + Cache: controllers.CacheOptions(egressMITMCAPool), } } diff --git a/cmd/atecontroller/main_test.go b/cmd/atecontroller/main_test.go index 0795aaf1c1..8564ee7d8f 100644 --- a/cmd/atecontroller/main_test.go +++ b/cmd/atecontroller/main_test.go @@ -22,7 +22,9 @@ import ( "github.com/spf13/pflag" prombridge "go.opentelemetry.io/contrib/bridges/prometheus" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/cache" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -128,27 +130,40 @@ func TestManagerOptionsMetricsListener(t *testing.T) { } } -func TestManagerOptionsScopeSecretCache(t *testing.T) { +func TestManagerOptionsScopeCache(t *testing.T) { t.Parallel() ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} opts := managerOptions(ref, true) - var byObject cache.ByObject - found := false + var secrets, deployments cache.ByObject + hasSecrets, hasDeployments := false, false for obj, by := range opts.Cache.ByObject { - if _, isSecret := obj.(*corev1.Secret); isSecret { - byObject, found = by, true + switch obj.(type) { + case *corev1.Secret: + secrets, hasSecrets = by, true + case *appsv1.Deployment: + deployments, hasDeployments = by, true } } - if !found { + if !hasSecrets { t.Fatal("no cache scope for Secrets") } - cfg, ok := byObject.Namespaces[ref.Namespace] - if !ok || len(byObject.Namespaces) != 1 { - t.Fatalf("Secret cache namespaces = %v, want only %q", byObject.Namespaces, ref.Namespace) + cfg, ok := secrets.Namespaces[ref.Namespace] + if !ok || len(secrets.Namespaces) != 1 { + t.Fatalf("Secret cache namespaces = %v, want only %q", secrets.Namespaces, ref.Namespace) } if got := cfg.FieldSelector.String(); got != "metadata.name="+ref.Name { t.Errorf("Secret field selector = %q, want %q", got, "metadata.name="+ref.Name) } + + if !hasDeployments || deployments.Label == nil { + t.Fatal("no label-selected cache scope for Deployments") + } + if !deployments.Label.Matches(labels.Set{"ate.dev/worker-pool": "pool"}) { + t.Errorf("Deployment label selector %q does not match the label ate.dev/worker-pool", deployments.Label) + } + if deployments.Label.Matches(labels.Set{}) { + t.Errorf("Deployment label selector %q matches a Deployment with no labels", deployments.Label) + } } From 291b06bfc0b99187ed7c3fc36aa0a0c87983d3e6 Mon Sep 17 00:00:00 2001 From: Danielle Mustillo Date: Wed, 18 Nov 2026 15:14:40 -0500 Subject: [PATCH 2/3] small refactors around unit tests & comments Signed-off-by: Danielle Mustillo --- .../internal/controllers/cache_test.go | 45 +++++++++++++++++++ .../controllers/workerpool_controller.go | 2 - .../controllers/workerpool_controller_test.go | 2 - cmd/atecontroller/main_test.go | 34 ++++---------- 4 files changed, 54 insertions(+), 29 deletions(-) diff --git a/cmd/atecontroller/internal/controllers/cache_test.go b/cmd/atecontroller/internal/controllers/cache_test.go index 94f5b12101..566c87222a 100644 --- a/cmd/atecontroller/internal/controllers/cache_test.go +++ b/cmd/atecontroller/internal/controllers/cache_test.go @@ -19,15 +19,60 @@ import ( "testing" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" ) +// cachedClient reads from the manager cache, as the reconcilers do. TestMain sets it. +var cachedClient client.Client + +// TestCacheOptionsScope checks the Secret and Deployment rules of CacheOptions. +func TestCacheOptionsScope(t *testing.T) { + t.Parallel() + + ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} + opts := CacheOptions(ref) + + var secrets, deployments cache.ByObject + hasSecrets, hasDeployments := false, false + for obj, by := range opts.ByObject { + switch obj.(type) { + case *corev1.Secret: + secrets, hasSecrets = by, true + case *appsv1.Deployment: + deployments, hasDeployments = by, true + } + } + if !hasSecrets { + t.Fatal("no cache scope for Secrets") + } + scope, ok := secrets.Namespaces[ref.Namespace] + if !ok || len(secrets.Namespaces) != 1 { + t.Fatalf("Secret cache namespaces = %v, want only %q", secrets.Namespaces, ref.Namespace) + } + if got := scope.FieldSelector.String(); got != "metadata.name="+ref.Name { + t.Errorf("Secret field selector = %q, want %q", got, "metadata.name="+ref.Name) + } + + if !hasDeployments || deployments.Label == nil { + t.Fatal("no label-selected cache scope for Deployments") + } + if !deployments.Label.Matches(labels.Set{"ate.dev/worker-pool": "pool"}) { + t.Errorf("Deployment label selector %q does not match the label ate.dev/worker-pool", deployments.Label) + } + if deployments.Label.Matches(labels.Set{}) { + t.Errorf("Deployment label selector %q matches a Deployment with no labels", deployments.Label) + } +} + // TestDeploymentCacheScope checks that the manager cache holds a pool's // Deployment and no Deployment without the ate.dev/worker-pool label. func TestDeploymentCacheScope(t *testing.T) { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index eadd53fe30..23b1906a4f 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -163,8 +163,6 @@ func (r *WorkerPoolReconciler) reconcileWorkerPool(ctx context.Context, wp *atev log := log.FromContext(ctx) log.Info("Reconciling worker pool") - // The cache holds only Deployments with workerPoolLabel (CacheOptions). One - // without it reads as not found here, and the apply at the end labels it. existing := &appsv1.Deployment{} if err := r.Get(ctx, types.NamespacedName{Name: wp.Name, Namespace: wp.Namespace}, existing); err != nil { if !k8errors.IsNotFound(err) { diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 5bfe2604dc..c61725a770 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -53,8 +53,6 @@ import ( var ( cfg *rest.Config k8sClient client.Client - // cachedClient reads from the manager cache, as the reconcilers do. - cachedClient client.Client ) func TestMain(m *testing.M) { diff --git a/cmd/atecontroller/main_test.go b/cmd/atecontroller/main_test.go index 8564ee7d8f..5b2f936ead 100644 --- a/cmd/atecontroller/main_test.go +++ b/cmd/atecontroller/main_test.go @@ -24,9 +24,7 @@ import ( prombridge "go.opentelemetry.io/contrib/bridges/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/cache" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) @@ -130,40 +128,26 @@ func TestManagerOptionsMetricsListener(t *testing.T) { } } +// TestManagerOptionsScopeCache checks that the manager uses +// controllers.CacheOptions. The rules are tested next to CacheOptions. func TestManagerOptionsScopeCache(t *testing.T) { t.Parallel() - ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} - opts := managerOptions(ref, true) + opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, true) - var secrets, deployments cache.ByObject hasSecrets, hasDeployments := false, false - for obj, by := range opts.Cache.ByObject { + for obj := range opts.Cache.ByObject { switch obj.(type) { case *corev1.Secret: - secrets, hasSecrets = by, true + hasSecrets = true case *appsv1.Deployment: - deployments, hasDeployments = by, true + hasDeployments = true } } if !hasSecrets { - t.Fatal("no cache scope for Secrets") + t.Error("no cache scope for Secrets") } - cfg, ok := secrets.Namespaces[ref.Namespace] - if !ok || len(secrets.Namespaces) != 1 { - t.Fatalf("Secret cache namespaces = %v, want only %q", secrets.Namespaces, ref.Namespace) - } - if got := cfg.FieldSelector.String(); got != "metadata.name="+ref.Name { - t.Errorf("Secret field selector = %q, want %q", got, "metadata.name="+ref.Name) - } - - if !hasDeployments || deployments.Label == nil { - t.Fatal("no label-selected cache scope for Deployments") - } - if !deployments.Label.Matches(labels.Set{"ate.dev/worker-pool": "pool"}) { - t.Errorf("Deployment label selector %q does not match the label ate.dev/worker-pool", deployments.Label) - } - if deployments.Label.Matches(labels.Set{}) { - t.Errorf("Deployment label selector %q matches a Deployment with no labels", deployments.Label) + if !hasDeployments { + t.Error("no cache scope for Deployments") } } From 280d8b69320480b07c33a98f090a96d27bc21598 Mon Sep 17 00:00:00 2001 From: Danielle Mustillo Date: Wed, 18 Nov 2026 15:22:40 -0500 Subject: [PATCH 3/3] explicit error handling Signed-off-by: Danielle Mustillo --- cmd/atecontroller/internal/controllers/cache.go | 11 +++++++---- .../internal/controllers/cache_test.go | 5 ++++- .../controllers/workerpool_controller_test.go | 7 ++++++- cmd/atecontroller/main.go | 17 +++++++++++++---- cmd/atecontroller/main_test.go | 10 ++++++++-- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/cmd/atecontroller/internal/controllers/cache.go b/cmd/atecontroller/internal/controllers/cache.go index a251059ef9..5fb2b5c6d5 100644 --- a/cmd/atecontroller/internal/controllers/cache.go +++ b/cmd/atecontroller/internal/controllers/cache.go @@ -15,22 +15,25 @@ package controllers import ( + "fmt" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" ) // CacheOptions limits the manager cache to the egressMITMCAPool Secret and to // Deployments that carry workerPoolLabel. Other kinds are cached in full. -func CacheOptions(egressMITMCAPool types.NamespacedName) cache.Options { +func CacheOptions(egressMITMCAPool types.NamespacedName) (cache.Options, error) { hasWorkerPoolLabel, err := labels.NewRequirement(workerPoolLabel, selection.Exists, nil) - utilruntime.Must(err) + if err != nil { + return cache.Options{}, fmt.Errorf("failed to select Deployments by label %q: %w", workerPoolLabel, err) + } return cache.Options{ ByObject: map[client.Object]cache.ByObject{ &corev1.Secret{}: { @@ -42,5 +45,5 @@ func CacheOptions(egressMITMCAPool types.NamespacedName) cache.Options { }, &appsv1.Deployment{}: {Label: labels.NewSelector().Add(*hasWorkerPoolLabel)}, }, - } + }, nil } diff --git a/cmd/atecontroller/internal/controllers/cache_test.go b/cmd/atecontroller/internal/controllers/cache_test.go index 566c87222a..533194cecc 100644 --- a/cmd/atecontroller/internal/controllers/cache_test.go +++ b/cmd/atecontroller/internal/controllers/cache_test.go @@ -39,7 +39,10 @@ func TestCacheOptionsScope(t *testing.T) { t.Parallel() ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} - opts := CacheOptions(ref) + opts, err := CacheOptions(ref) + if err != nil { + t.Fatalf("CacheOptions: %v", err) + } var secrets, deployments cache.ByObject hasSecrets, hasDeployments := false, false diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index c61725a770..5604a83d6e 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -70,11 +70,16 @@ func TestMain(m *testing.M) { os.Exit(1) } + cacheOptions, err := CacheOptions(EgressMITMCAPoolRef(installdefaults.SystemNamespace)) + if err != nil { + fmt.Fprintf(os.Stderr, "cache options failed: %v\n", err) + os.Exit(1) + } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", - Cache: CacheOptions(EgressMITMCAPoolRef(installdefaults.SystemNamespace)), + Cache: cacheOptions, }) if err != nil { fmt.Fprintf(os.Stderr, "manager creation failed: %v\n", err) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d9fb560fe6..3ec790c6a4 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -126,12 +126,16 @@ func metricsBindAddr(servePull bool) string { } // managerOptions configures the controller-runtime manager. -func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) ctrl.Options { +func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) (ctrl.Options, error) { + cacheOptions, err := controllers.CacheOptions(egressMITMCAPool) + if err != nil { + return ctrl.Options{}, err + } return ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsBindAddr(servePull)}, - Cache: controllers.CacheOptions(egressMITMCAPool), - } + Cache: cacheOptions, + }, nil } func main() { @@ -231,7 +235,12 @@ func main() { // EgressMITMTrustReconciler watches the Secret `egress-mitm-ca-pool`. systemNamespace := installdefaults.NamespaceFromPodEnv() egressMITMCAPool := controllers.EgressMITMCAPoolRef(systemNamespace) - mgr, err := ctrl.NewManager(k8sConfig, managerOptions(egressMITMCAPool, servePull)) + mgrOptions, err := managerOptions(egressMITMCAPool, servePull) + if err != nil { + setupLog.Error(err, "unable to configure manager") + os.Exit(1) + } + mgr, err := ctrl.NewManager(k8sConfig, mgrOptions) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/cmd/atecontroller/main_test.go b/cmd/atecontroller/main_test.go index 5b2f936ead..5ecb44d077 100644 --- a/cmd/atecontroller/main_test.go +++ b/cmd/atecontroller/main_test.go @@ -112,7 +112,10 @@ func TestManagerOptionsMetricsListener(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, tt.servePull) + opts, err := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, tt.servePull) + if err != nil { + t.Fatalf("managerOptions: %v", err) + } if opts.Metrics.BindAddress != tt.wantAddr { t.Errorf("metrics address = %q, want %q", opts.Metrics.BindAddress, tt.wantAddr) @@ -133,7 +136,10 @@ func TestManagerOptionsMetricsListener(t *testing.T) { func TestManagerOptionsScopeCache(t *testing.T) { t.Parallel() - opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, true) + opts, err := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, true) + if err != nil { + t.Fatalf("managerOptions: %v", err) + } hasSecrets, hasDeployments := false, false for obj := range opts.Cache.ByObject {