diff --git a/cmd/atecontroller/internal/controllers/cache.go b/cmd/atecontroller/internal/controllers/cache.go new file mode 100644 index 0000000000..5fb2b5c6d5 --- /dev/null +++ b/cmd/atecontroller/internal/controllers/cache.go @@ -0,0 +1,49 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CacheOptions limits the manager cache to the egressMITMCAPool Secret and to +// Deployments that carry workerPoolLabel. Other kinds are cached in full. +func CacheOptions(egressMITMCAPool types.NamespacedName) (cache.Options, error) { + hasWorkerPoolLabel, err := labels.NewRequirement(workerPoolLabel, selection.Exists, nil) + if err != nil { + return cache.Options{}, fmt.Errorf("failed to select Deployments by label %q: %w", workerPoolLabel, err) + } + return cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Namespaces: map[string]cache.Config{ + egressMITMCAPool.Namespace: { + FieldSelector: fields.OneTermEqualSelector("metadata.name", egressMITMCAPool.Name), + }, + }, + }, + &appsv1.Deployment{}: {Label: labels.NewSelector().Add(*hasWorkerPoolLabel)}, + }, + }, nil +} diff --git a/cmd/atecontroller/internal/controllers/cache_test.go b/cmd/atecontroller/internal/controllers/cache_test.go new file mode 100644 index 0000000000..533194cecc --- /dev/null +++ b/cmd/atecontroller/internal/controllers/cache_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// cachedClient reads from the manager cache, as the reconcilers do. TestMain sets it. +var cachedClient client.Client + +// TestCacheOptionsScope checks the Secret and Deployment rules of CacheOptions. +func TestCacheOptionsScope(t *testing.T) { + t.Parallel() + + ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} + opts, err := CacheOptions(ref) + if err != nil { + t.Fatalf("CacheOptions: %v", err) + } + + var secrets, deployments cache.ByObject + hasSecrets, hasDeployments := false, false + for obj, by := range opts.ByObject { + switch obj.(type) { + case *corev1.Secret: + secrets, hasSecrets = by, true + case *appsv1.Deployment: + deployments, hasDeployments = by, true + } + } + if !hasSecrets { + t.Fatal("no cache scope for Secrets") + } + scope, ok := secrets.Namespaces[ref.Namespace] + if !ok || len(secrets.Namespaces) != 1 { + t.Fatalf("Secret cache namespaces = %v, want only %q", secrets.Namespaces, ref.Namespace) + } + if got := scope.FieldSelector.String(); got != "metadata.name="+ref.Name { + t.Errorf("Secret field selector = %q, want %q", got, "metadata.name="+ref.Name) + } + + if !hasDeployments || deployments.Label == nil { + t.Fatal("no label-selected cache scope for Deployments") + } + if !deployments.Label.Matches(labels.Set{"ate.dev/worker-pool": "pool"}) { + t.Errorf("Deployment label selector %q does not match the label ate.dev/worker-pool", deployments.Label) + } + if deployments.Label.Matches(labels.Set{}) { + t.Errorf("Deployment label selector %q matches a Deployment with no labels", deployments.Label) + } +} + +// TestDeploymentCacheScope checks that the manager cache holds a pool's +// Deployment and no Deployment without the ate.dev/worker-pool label. +func TestDeploymentCacheScope(t *testing.T) { + t.Parallel() + ctx := t.Context() + const label = "ate.dev/worker-pool" + wp := makeWorkerPool("cache-scope", "default", 2, "worker:v1") + + // The bystander is created first, so a cache of every Deployment holds it + // before it holds the labeled Deployment of the pool. + bystander := applyDeploymentWithoutLabels(t, "cache-scope-bystander", wp.Namespace, map[string]string{"app": "bystander"}, "someone-else") + applyDeploymentWithoutLabels(t, wp.Name, wp.Namespace, map[string]string{label: wp.Name}, workerPoolFieldOwner) + + if err := k8sClient.Create(ctx, wp); err != nil { + t.Fatalf("create WorkerPool: %v", err) + } + deleteOnCleanup(t, wp) + + eventually(t, func(ctx context.Context) (bool, error) { + dep := &appsv1.Deployment{} + if err := cachedClient.Get(ctx, client.ObjectKeyFromObject(wp), dep); err != nil { + return false, nil + } + return dep.Labels[label] == wp.Name && metav1.IsControlledBy(dep, wp), nil + }) + + err := cachedClient.Get(ctx, client.ObjectKeyFromObject(bystander), &appsv1.Deployment{}) + if !k8errors.IsNotFound(err) { + t.Errorf("cached read of a Deployment without %s: err = %v, want NotFound", label, err) + } + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(bystander), &appsv1.Deployment{}); err != nil { + t.Fatalf("live read of the bystander Deployment: %v", err) + } + + dep, err := getDeployment(ctx, wp) + if err != nil { + t.Fatalf("get Deployment: %v", err) + } + strip := client.RawPatch(types.MergePatchType, []byte(`{"metadata":{"labels":{"`+label+`":null}}}`)) + if err := k8sClient.Patch(ctx, dep, strip); err != nil { + t.Fatalf("strip %s: %v", label, err) + } + if _, has := dep.Labels[label]; has { + t.Fatalf("label %s survived the patch", label) + } + eventually(t, func(ctx context.Context) (bool, error) { + dep, err := getDeployment(ctx, wp) + return err == nil && dep.Labels[label] == wp.Name, nil + }) +} + +// applyDeploymentWithoutLabels applies a Deployment whose metadata carries no +// labels, and deletes it on cleanup. +func applyDeploymentWithoutLabels(t *testing.T, name, namespace string, podLabels map[string]string, fieldOwner string) *appsv1.Deployment { + t.Helper() + ac := appsv1ac.Deployment(name, namespace). + WithSpec(appsv1ac.DeploymentSpec(). + WithSelector(metav1ac.LabelSelector().WithMatchLabels(podLabels)). + WithTemplate(corev1ac.PodTemplateSpec(). + WithLabels(podLabels). + WithSpec(corev1ac.PodSpec(). + WithContainers(corev1ac.Container().WithName("worker").WithImage("worker:v0"))))) + if err := k8sClient.Apply(t.Context(), ac, client.FieldOwner(fieldOwner)); err != nil { + t.Fatalf("apply Deployment %s: %v", name, err) + } + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}} + deleteOnCleanup(t, dep) + return dep +} diff --git a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go index fe5bfd22a3..36d91ee173 100644 --- a/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go +++ b/cmd/atecontroller/internal/controllers/egressmitmtrust_controller.go @@ -58,7 +58,7 @@ func EgressMITMCAPoolRef(systemNamespace string) types.NamespacedName { } // resourceNames covers list and watch because the manager's cache selects the -// Secret by metadata.name (managerOptions in main.go), which the apiserver +// Secret by metadata.name (CacheOptions), which the apiserver // authorizes as a request for that one name. Without the selector the grant // would deny them. // diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 1244d6498a..356c5a4430 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -93,7 +93,8 @@ type workerOTelSettings struct { MetricsExporter string } -// workerPoolLabel names the WorkerPool on each of its worker pods. +// workerPoolLabel names the WorkerPool on its Deployment, worker pods and +// NetworkPolicy. CacheOptions caches only Deployments that carry it. const workerPoolLabel = "ate.dev/worker-pool" const ( diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index 3b6d3502ba..5604a83d6e 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -70,15 +70,22 @@ func TestMain(m *testing.M) { os.Exit(1) } + cacheOptions, err := CacheOptions(EgressMITMCAPoolRef(installdefaults.SystemNamespace)) + if err != nil { + fmt.Fprintf(os.Stderr, "cache options failed: %v\n", err) + os.Exit(1) + } mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", + Cache: cacheOptions, }) if err != nil { fmt.Fprintf(os.Stderr, "manager creation failed: %v\n", err) os.Exit(1) } + cachedClient = mgr.GetClient() if err := (&WorkerPoolReconciler{ Client: mgr.GetClient(), diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index d99fb97c81..3ec790c6a4 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -36,16 +36,12 @@ import ( "go.ate.dev/substrate/pkg/client/informers/externalversions" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -130,22 +126,16 @@ func metricsBindAddr(servePull bool) string { } // managerOptions configures the controller-runtime manager. -func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) ctrl.Options { +func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) (ctrl.Options, error) { + cacheOptions, err := controllers.CacheOptions(egressMITMCAPool) + if err != nil { + return ctrl.Options{}, err + } return ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsBindAddr(servePull)}, - Cache: cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Namespaces: map[string]cache.Config{ - egressMITMCAPool.Namespace: { - FieldSelector: fields.OneTermEqualSelector("metadata.name", egressMITMCAPool.Name), - }, - }, - }, - }, - }, - } + Cache: cacheOptions, + }, nil } func main() { @@ -245,7 +235,12 @@ func main() { // EgressMITMTrustReconciler watches the Secret `egress-mitm-ca-pool`. systemNamespace := installdefaults.NamespaceFromPodEnv() egressMITMCAPool := controllers.EgressMITMCAPoolRef(systemNamespace) - mgr, err := ctrl.NewManager(k8sConfig, managerOptions(egressMITMCAPool, servePull)) + mgrOptions, err := managerOptions(egressMITMCAPool, servePull) + if err != nil { + setupLog.Error(err, "unable to configure manager") + os.Exit(1) + } + mgr, err := ctrl.NewManager(k8sConfig, mgrOptions) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/cmd/atecontroller/main_test.go b/cmd/atecontroller/main_test.go index 0795aaf1c1..5ecb44d077 100644 --- a/cmd/atecontroller/main_test.go +++ b/cmd/atecontroller/main_test.go @@ -22,9 +22,9 @@ import ( "github.com/spf13/pflag" prombridge "go.opentelemetry.io/contrib/bridges/prometheus" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/cache" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) @@ -112,7 +112,10 @@ func TestManagerOptionsMetricsListener(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, tt.servePull) + opts, err := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, tt.servePull) + if err != nil { + t.Fatalf("managerOptions: %v", err) + } if opts.Metrics.BindAddress != tt.wantAddr { t.Errorf("metrics address = %q, want %q", opts.Metrics.BindAddress, tt.wantAddr) @@ -128,27 +131,29 @@ func TestManagerOptionsMetricsListener(t *testing.T) { } } -func TestManagerOptionsScopeSecretCache(t *testing.T) { +// TestManagerOptionsScopeCache checks that the manager uses +// controllers.CacheOptions. The rules are tested next to CacheOptions. +func TestManagerOptionsScopeCache(t *testing.T) { t.Parallel() - ref := types.NamespacedName{Namespace: "ate-system", Name: "pool"} - opts := managerOptions(ref, true) + opts, err := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, true) + if err != nil { + t.Fatalf("managerOptions: %v", err) + } - var byObject cache.ByObject - found := false - for obj, by := range opts.Cache.ByObject { - if _, isSecret := obj.(*corev1.Secret); isSecret { - byObject, found = by, true + hasSecrets, hasDeployments := false, false + for obj := range opts.Cache.ByObject { + switch obj.(type) { + case *corev1.Secret: + hasSecrets = true + case *appsv1.Deployment: + hasDeployments = true } } - if !found { - t.Fatal("no cache scope for Secrets") - } - cfg, ok := byObject.Namespaces[ref.Namespace] - if !ok || len(byObject.Namespaces) != 1 { - t.Fatalf("Secret cache namespaces = %v, want only %q", byObject.Namespaces, ref.Namespace) + if !hasSecrets { + t.Error("no cache scope for Secrets") } - if got := cfg.FieldSelector.String(); got != "metadata.name="+ref.Name { - t.Errorf("Secret field selector = %q, want %q", got, "metadata.name="+ref.Name) + if !hasDeployments { + t.Error("no cache scope for Deployments") } }