diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 403af343..3e370b4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,9 @@ jobs: - name: Unit tests run: make unit + - name: Helm template tests + run: make helm-test + - name: Vet run: make vet diff --git a/.gitignore b/.gitignore index 01dc05d8..c1d5cf8b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ chart/bootc-operator/charts/ chart/bootc-operator/Chart.lock chart/bootc-operator-crds/templates/ chart/bootc-operator/templates/controller-clusterrole.yaml +chart/bootc-operator/templates/controller-role.yaml +chart/bootc-operator/templates/controller-rolebinding.yaml chart/bootc-operator/templates/daemon-clusterrole.yaml chart/bootc-operator/templates/leader-election-role.yaml chart/bootc-operator/templates/controller-deployment.yaml diff --git a/Makefile b/Makefile index 52605716..f6a9f1a9 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,14 @@ BINK_NODE_DISK_IMAGE ?= ghcr.io/bootc-dev/bink/node:v$(DEFAULT_KUBE_MINOR)-fedor BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node E2E_REGISTRY_USER ?= e2e-user E2E_REGISTRY_PASSWORD ?= e2e-password -# Released operator for upgrade testing (TestOperatorUpgrade). -# Override RELEASED_OPERATOR_IMG for downstream images. +# Extra namespace deploy-bink allows pull secrets from (TestPullSecretNamespaces). +E2E_PULL_SECRET_NAMESPACE ?= bootc-e2e-pull-secrets +# Released operator for upgrade testing (TestOperatorUpgrade): the published +# chart and the release tag, whose version without the leading "v" is the +# chart version. The released image is the one the chart deploys. Override +# RELEASED_OPERATOR_CHART for downstream charts. RELEASED_OPERATOR_TAG ?= v0.2.2 -RELEASED_OPERATOR_IMG ?= ghcr.io/bootc-dev/bootc-operator:$(RELEASED_OPERATOR_TAG) +RELEASED_OPERATOR_CHART ?= oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator IMG_BINK_RELEASED ?= registry.cluster.local:5000/bootc-operator-released:latest # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. YEAR ?= $(shell date +%Y) @@ -62,6 +66,10 @@ manifests: controller-gen ## Generate CRD and RBAC manifests. helm: manifests yq ## Regenerate Helm chart from Kustomize manifests. YQ="$(YQ)" ./hack/sync-helm.sh +.PHONY: helm-test +helm-test: helm helm-tool ## Render the Helm chart and check the output. + HELM="$(HELM)" YQ="$(YQ)" ./hack/test-helm.sh + .PHONY: generate generate: controller-gen ## Generate DeepCopy method implementations. "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt",year=$(YEAR) paths="./..." @@ -119,8 +127,10 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= E2E_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \ E2E_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ + E2E_PULL_SECRET_NAMESPACE=$(E2E_PULL_SECRET_NAMESPACE) \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . # EKS e2e settings @@ -152,7 +162,7 @@ e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \ E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(RELEASED_OPERATOR_IMG)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ $(if $(EKS_IMAGE_PULL_SECRET),E2E_IMAGE_PULL_SECRET=$(EKS_IMAGE_PULL_SECRET)) \ E2E_NODE_PULL_SECRET="$(EKS_NODE_PULL_SECRET)" \ ARTIFACTS="$(ARTIFACTS)" \ @@ -192,13 +202,14 @@ build-update-image: ## Build derived node images for update testing and push to podman push --tls-verify=false localhost:5000/node:update2 .PHONY: push-released-operator-image -push-released-operator-image: ## Pull released operator image and push to bink registry for upgrade testing. - @if [ -z "$(RELEASED_OPERATOR_IMG)" ]; then \ - echo "Error: RELEASED_OPERATOR_IMG must be set (e.g. ghcr.io/bootc-dev/bootc-operator:v0.1.0)"; \ - exit 1; \ - fi - podman pull $(RELEASED_OPERATOR_IMG) - podman push --tls-verify=false $(RELEASED_OPERATOR_IMG) localhost:5000/bootc-operator-released:latest +push-released-operator-image: yq ## Copy the image of the released chart to the bink registry for upgrade testing. + img=$$(helm template bootc-operator $(RELEASED_OPERATOR_CHART) \ + --version $(RELEASED_OPERATOR_TAG:v%=%) \ + --kube-version $(DEFAULT_KUBE_MINOR).0 | \ + $(YQ) 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "manager") | .image') && \ + test -n "$$img" && \ + podman pull "$$img" && \ + podman push --tls-verify=false "$$img" localhost:5000/bootc-operator-released:latest ##@ Deployment @@ -256,9 +267,11 @@ start-bink: seed-node-image ## Start a bink cluster (idempotent). kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m .PHONY: deploy-bink -deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). +deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_TAG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). podman push --tls-verify=false $(IMG) localhost:5000/bootc-operator-e2e:latest helm dependency build ./chart/bootc-operator + kubectl --kubeconfig $(KUBECONFIG_BINK) create namespace $(E2E_PULL_SECRET_NAMESPACE) \ + --dry-run=client -o yaml | kubectl --kubeconfig $(KUBECONFIG_BINK) apply -f - img="$(IMG_BINK)" && \ helm upgrade --install bootc-operator ./chart/bootc-operator \ --kubeconfig $(abspath $(KUBECONFIG_BINK)) \ @@ -266,6 +279,7 @@ deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-re --set-string image.repository="$${img%:*}" \ --set-string image.tag="$${img##*:}" \ --set-string image.pullPolicy=Always \ + --set-json 'pullSecretNamespaces=["$(E2E_PULL_SECRET_NAMESPACE)"]' \ --set-json 'controller.extraArgs=["--allow-insecure-registry","--tag-resolution-interval=10s"]' kubectl --kubeconfig $(KUBECONFIG_BINK) -n bootc-operator rollout restart deployment/bootc-operator-controller kubectl --kubeconfig $(KUBECONFIG_BINK) -n bootc-operator rollout status deployment/bootc-operator-controller --timeout=3m @@ -294,12 +308,14 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOLINES_BIN = $(LOCALBIN)/golines GOVULNCHECK = $(LOCALBIN)/govulncheck YQ ?= $(LOCALBIN)/yq +HELM ?= $(LOCALBIN)/helm KUSTOMIZE_VERSION ?= v5.8.1 CONTROLLER_TOOLS_VERSION ?= v0.20.1 GOLANGCI_LINT_VERSION ?= v2.11.4 GOVULNCHECK_VERSION ?= v1.9.0 YQ_VERSION ?= v4.53.2 +HELM_VERSION ?= v3.19.0 # ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ @@ -353,6 +369,11 @@ yq: $(YQ) ## Download yq locally if necessary. $(YQ): $(LOCALBIN) $(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,$(YQ_VERSION)) +.PHONY: helm-tool +helm-tool: $(HELM) ## Download helm locally if necessary. +$(HELM): $(LOCALBIN) + $(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm,$(HELM_VERSION)) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/chart/bootc-operator/templates/_helpers.tpl b/chart/bootc-operator/templates/_helpers.tpl index e16553e1..090b6cc8 100644 --- a/chart/bootc-operator/templates/_helpers.tpl +++ b/chart/bootc-operator/templates/_helpers.tpl @@ -23,6 +23,10 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} +{{- define "bootc-operator.pullSecretNamespaces" -}} +{{- prepend .Values.pullSecretNamespaces .Release.Namespace | uniq | join "," }} +{{- end }} + {{- define "bootc-operator.image" -}} {{- $tag := default .Chart.AppVersion .Values.image.tag -}} {{ printf "%s:%s" .Values.image.repository $tag }} diff --git a/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml b/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml new file mode 100644 index 00000000..8b3fc74a --- /dev/null +++ b/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml @@ -0,0 +1,37 @@ +{{- range without (.Values.pullSecretNamespaces | uniq) $.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets + namespace: {{ . }} + labels: + {{- include "bootc-operator.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets + namespace: {{ . }} + labels: + {{- include "bootc-operator.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets +subjects: +- kind: ServiceAccount + name: {{ include "bootc-operator.fullname" $ }}-controller + namespace: {{ $.Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/values.schema.json b/chart/bootc-operator/values.schema.json new file mode 100644 index 00000000..7c4847ef --- /dev/null +++ b/chart/bootc-operator/values.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "pullSecretNamespaces": { + "type": "array", + "items": { + "type": "string", + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + } + } + } +} diff --git a/chart/bootc-operator/values.yaml b/chart/bootc-operator/values.yaml index 383c3705..a91f91eb 100644 --- a/chart/bootc-operator/values.yaml +++ b/chart/bootc-operator/values.yaml @@ -10,5 +10,9 @@ image: imagePullSecrets: [] +# Extra namespaces that BootcNodePool pullSecretRefs may point to. The release +# namespace is always allowed. The namespaces must already exist. +pullSecretNamespaces: [] + controller: extraArgs: [] diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f15223d3..6fb2971f 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -49,6 +50,7 @@ func main() { var probeAddr string var tagResolutionInterval time.Duration var allowInsecureRegistry bool + var pullSecretNamespacesFlag string flag.StringVar( &probeAddr, "health-probe-bind-address", @@ -67,6 +69,13 @@ func main() { false, "Allow falling back to HTTP when resolving tag-based image refs against registries that do not serve TLS.", ) + flag.StringVar( + &pullSecretNamespacesFlag, + "pull-secret-namespaces", + "", + "Comma-separated extra namespaces that pools may reference pull secrets from. "+ + "The operator namespace is always allowed.", + ) flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -88,11 +97,30 @@ func main() { version.GitCommit, ) + operatorNamespace := detectNamespace() + pullSecretNamespaces, err := controller.ParsePullSecretNamespaces( + pullSecretNamespacesFlag, + operatorNamespace, + ) + if err != nil { + setupLog.Error(err, "Invalid --pull-secret-namespaces") + os.Exit(1) + } + if len(pullSecretNamespaces) == 0 { + setupLog.Error(nil, "No pull secret namespaces configured and operator namespace unknown; "+ + "set --pull-secret-namespaces or POD_NAMESPACE") + os.Exit(1) + } + setupLog.Info("Pull secret namespaces", "namespaces", pullSecretNamespaces) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "bootc-operator.bootc.dev", + Cache: cache.Options{ + ByObject: controller.SecretCacheByObject(pullSecretNamespaces), + }, }) if err != nil { setupLog.Error(err, "Failed to start manager") @@ -109,9 +137,10 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), KubeClient: kubeClient, - EventNamespace: detectNamespace(), + EventNamespace: operatorNamespace, TagResolver: ®istry.GGCRResolver{AllowInsecure: allowInsecureRegistry}, TagResolutionInterval: tagResolutionInterval, + PullSecretNamespaces: pullSecretNamespaces, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "bootcnodepool") os.Exit(1) diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 809aa028..066d954c 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -2,6 +2,7 @@ resources: - controller_service_account.yaml - role.yaml - role_binding.yaml +- namespaced_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml - daemon_service_account.yaml diff --git a/config/rbac/namespaced_role_binding.yaml b/config/rbac/namespaced_role_binding.yaml new file mode 100644 index 00000000..a4b2f20b --- /dev/null +++ b/config/rbac/namespaced_role_binding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: bootc-operator + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c25de146..a1e0f340 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,14 +27,6 @@ rules: - pods/eviction verbs: - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - apiGroups: - "" - events.k8s.io @@ -82,3 +74,18 @@ rules: - bootcnodes/status verbs: - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: manager-role + namespace: system +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch diff --git a/hack/sync-helm.sh b/hack/sync-helm.sh index f55df976..434b3102 100755 --- a/hack/sync-helm.sh +++ b/hack/sync-helm.sh @@ -20,10 +20,13 @@ echo "Synced CRDs" # --- RBAC --- # gen_rbac OUTPUT_FILE SOURCE_FILE KIND NAME_SUFFIX COMPONENT [NAMESPACE] +# Takes the rules from the KIND document in SOURCE_FILE, which may hold +# several (controller-gen emits namespaced rules as a separate Role). gen_rbac() { local output="$1" source="$2" kind="$3" suffix="$4" component="$5" ns="${6:-}" local rules - rules=$("$YQ" -I2 '.rules' "$source") + rules=$("$YQ" -I2 "select(.kind == \"${kind}\") | .rules" "$source") + [ -n "$rules" ] || { echo "error: no ${kind} rules in ${source}" >&2; exit 1; } { echo "apiVersion: rbac.authorization.k8s.io/v1" echo "kind: ${kind}" @@ -40,6 +43,8 @@ gen_rbac() { gen_rbac "$CHART_DIR/templates/controller-clusterrole.yaml" "$CONFIG_DIR/rbac/role.yaml" \ ClusterRole controller controller +gen_rbac "$CHART_DIR/templates/controller-role.yaml" "$CONFIG_DIR/rbac/role.yaml" \ + Role controller controller "{{ .Release.Namespace }}" gen_rbac "$CHART_DIR/templates/daemon-clusterrole.yaml" "$CONFIG_DIR/rbac/daemon_role.yaml" \ ClusterRole daemon daemon gen_rbac "$CHART_DIR/templates/leader-election-role.yaml" "$CONFIG_DIR/rbac/leader_election_role.yaml" \ @@ -84,8 +89,9 @@ gen_workload() { gen_workload "$CONFIG_DIR/manager/manager.yaml" Deployment controller \ "$CHART_DIR/templates/controller-deployment.yaml" -# Inject controller.extraArgs into the args list +# Inject --pull-secret-namespaces and controller.extraArgs into the args list sed -i '/^ image: {{ include "bootc-operator\.image" \. }}/i\ + - {{ printf "--pull-secret-namespaces=%s" (include "bootc-operator.pullSecretNamespaces" .) | quote }}\ {{- range .Values.controller.extraArgs }}\ - {{ . | quote }}\ {{- end }}' "$CHART_DIR/templates/controller-deployment.yaml" @@ -112,7 +118,8 @@ gen_binding() { .metadata.labels.HELM_LABELS_PLACEHOLDER = true | .roleRef.name = \"HELM_FULLNAME-${role_suffix}\" | .subjects[0].name = \"HELM_FULLNAME-${sa_suffix}\" | - .subjects[0].namespace = \"HELM_NAMESPACE\" + .subjects[0].namespace = \"HELM_NAMESPACE\" | + with(select(.metadata.namespace != null); .metadata.namespace = \"HELM_NAMESPACE\") " "$source" | helm_sed | sed \ -e '/HELM_LABELS_PLACEHOLDER/c\ {{- include "bootc-operator.labels" . | nindent 4 }}' \ > "$output" @@ -121,6 +128,9 @@ gen_binding() { gen_binding "$CONFIG_DIR/rbac/role_binding.yaml" \ "$CHART_DIR/templates/controller-clusterrolebinding.yaml" \ controller ClusterRole controller controller +gen_binding "$CONFIG_DIR/rbac/namespaced_role_binding.yaml" \ + "$CHART_DIR/templates/controller-rolebinding.yaml" \ + controller Role controller controller gen_binding "$CONFIG_DIR/rbac/daemon_role_binding.yaml" \ "$CHART_DIR/templates/daemon-clusterrolebinding.yaml" \ daemon ClusterRole daemon daemon diff --git a/hack/test-helm.sh b/hack/test-helm.sh new file mode 100755 index 00000000..c77d6e87 --- /dev/null +++ b/hack/test-helm.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Render the Helm chart with representative values and check the output. +# Run: make helm-test + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CHART_DIR="$REPO_ROOT/chart/bootc-operator" +HELM="${HELM:-helm}" +YQ="${YQ:-yq}" +NS=bootc-operator +SA=test-bootc-operator-controller + +fail() { echo "FAIL: $*" >&2; exit 1; } + +# A go-installed helm lacks build-time version info and assumes Kubernetes +# v1.20, which the chart's kubeVersion constraint rejects. +render() { + "$HELM" template test "$CHART_DIR" --namespace "$NS" --kube-version 1.34.0 "$@" +} + +# check MANIFESTS FLAG_VALUE NAMESPACES... +# Asserts that the controller flag is FLAG_VALUE, that each namespace has +# exactly one secret-reading Role bound to the controller ServiceAccount, +# and that the controller ClusterRole does not grant access to secrets. +check() { + local manifests="$1" flag="$2"; shift 2 + local want got + want=$(printf '%s\n' "$@" | sort) + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "Deployment") | + .spec.template.spec.containers[0].args[] | select(test("^--pull-secret-namespaces="))') + [ "$got" = "--pull-secret-namespaces=$flag" ] || fail "controller flag is [$got]" + + roles=$(echo "$manifests" | "$YQ" -N 'select(.kind == "Role") | + select(.rules | any_c(.resources | any_c(. == "secrets"))) | + .metadata.namespace + "/" + .metadata.name' | sort) + [ "$(echo "$roles" | cut -d/ -f1)" = "$want" ] || fail "secret Roles: [$roles]" + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "RoleBinding") | + select(.subjects | any_c(.name == "'"$SA"'" and .namespace == "'"$NS"'")) | + .metadata.namespace + "/" + .roleRef.name' | sort) + # The leader election binding has no namespace in the template. + got=$(echo "$got" | grep -v '^/') + [ "$got" = "$roles" ] || fail "controller RoleBindings [$got], want [$roles]" + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "ClusterRole") | + select(.metadata.name == "'"$SA"'") | + select(.rules | any_c(.resources | any_c(. == "secrets"))) | .metadata.name') + [ -z "$got" ] || fail "controller ClusterRole grants secrets access" +} + +"$HELM" dependency build "$CHART_DIR" >/dev/null + +check "$(render)" "$NS" "$NS" +# Duplicates and the release namespace itself must not produce extra RBAC. +check "$(render --set "pullSecretNamespaces={team-a,$NS,team-b,team-a}")" \ + "$NS,team-a,team-b" "$NS" team-a team-b + +for bad in "pullSecretNamespaces=team-a" "pullSecretNamespaces={team-a,}" \ + "pullSecretNamespaces={Team-A}"; do + if render --set "$bad" >/dev/null 2>&1; then fail "accepted --set $bad"; fi +done +echo "ok" diff --git a/internal/controller/bootcnodepool_controller.go b/internal/controller/bootcnodepool_controller.go index 6f9dee32..e23be865 100644 --- a/internal/controller/bootcnodepool_controller.go +++ b/internal/controller/bootcnodepool_controller.go @@ -67,6 +67,11 @@ type BootcNodePoolReconciler struct { TagResolver TagResolver TagResolutionInterval time.Duration + // PullSecretNamespaces is the allowlist of namespaces pools may + // reference pull secrets from. The manager's Secret cache must be + // restricted to the same namespaces. + PullSecretNamespaces []string + // drainCh receives events from drain goroutines to re-enqueue the // owning pool after a drain completes. drainCh chan event.GenericEvent @@ -85,7 +90,7 @@ type BootcNodePoolReconciler struct { // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes/status,verbs=get // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",namespace=system,resources=secrets,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list // +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get @@ -701,7 +706,8 @@ func (r *BootcNodePoolReconciler) listAllBootcNodes( } // fetchPullSecretData fetches the .dockerconfigjson data from the pull -// secret referenced by the pool. Returns nil if no pull secret is set. +// secret referenced by the pool. Returns nil if no pull secret is set, and +// an error if the secret's namespace is not in PullSecretNamespaces. func (r *BootcNodePoolReconciler) fetchPullSecretData( ctx context.Context, pool *bootcv1alpha1.BootcNodePool, @@ -710,6 +716,14 @@ func (r *BootcNodePoolReconciler) fetchPullSecretData( return nil, nil } + if !slices.Contains(r.PullSecretNamespaces, pool.Spec.PullSecretRef.Namespace) { + return nil, fmt.Errorf( + "pull secret namespace %q is not allowed (allowed: %s)", + pool.Spec.PullSecretRef.Namespace, + strings.Join(r.PullSecretNamespaces, ", "), + ) + } + key := types.NamespacedName{ Name: pool.Spec.PullSecretRef.Name, Namespace: pool.Spec.PullSecretRef.Namespace, diff --git a/internal/controller/pullsecret.go b/internal/controller/pullsecret.go new file mode 100644 index 00000000..795d82a1 --- /dev/null +++ b/internal/controller/pullsecret.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ParsePullSecretNamespaces parses a comma-separated namespace list and +// prepends operatorNamespace (when non-empty), which is always allowed. +// Empty entries and surrounding whitespace are dropped and duplicates +// removed. Invalid namespace names are rejected, since the Secret informer +// for them could never sync. +func ParsePullSecretNamespaces(value, operatorNamespace string) ([]string, error) { + var namespaces []string + for _, ns := range append([]string{operatorNamespace}, strings.Split(value, ",")...) { + ns = strings.TrimSpace(ns) + if ns == "" || slices.Contains(namespaces, ns) { + continue + } + if errs := validation.IsDNS1123Label(ns); len(errs) > 0 { + return nil, fmt.Errorf("invalid namespace %q: %s", ns, strings.Join(errs, "; ")) + } + namespaces = append(namespaces, ns) + } + return namespaces, nil +} + +// SecretCacheByObject returns manager cache settings that restrict the +// Secret informer to the given namespaces, so the controller never caches +// (or needs RBAC for) Secrets elsewhere in the cluster. +func SecretCacheByObject(namespaces []string) map[client.Object]cache.ByObject { + config := make(map[string]cache.Config, len(namespaces)) + for _, ns := range namespaces { + config[ns] = cache.Config{} + } + return map[client.Object]cache.ByObject{ + &corev1.Secret{}: {Namespaces: config}, + } +} diff --git a/internal/controller/pullsecret_test.go b/internal/controller/pullsecret_test.go index 27b6e1f3..c117ca08 100644 --- a/internal/controller/pullsecret_test.go +++ b/internal/controller/pullsecret_test.go @@ -36,7 +36,7 @@ func TestPullSecretRefPropagation(t *testing.T) { secretData := []byte(`{"auths":{"registry.example.com":{}}}`) - secret := newDockerConfigSecret("ps-prop-secret", "default", secretData) + secret := newDockerConfigSecret("ps-prop-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -46,7 +46,7 @@ func TestPullSecretRefPropagation(t *testing.T) { pool := testutil.NewPool("ps-prop-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-prop-secret", "default"), + testutil.WithPullSecret("ps-prop-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -56,7 +56,7 @@ func TestPullSecretRefPropagation(t *testing.T) { err := k8sClient.Get(ctx, client.ObjectKey{Name: "ps-prop-node"}, &bn) return bn.Spec.PullSecretRef, err }).Should(Equal(&bootcv1alpha1.PullSecretRef{ - Name: "ps-prop-secret", Namespace: "default", + Name: "ps-prop-secret", Namespace: testutil.OperatorNamespaceName, })) } @@ -72,7 +72,7 @@ func TestPullSecretMissingDegradedAndRecovery(t *testing.T) { pool := testutil.NewPool("ps-missing-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-missing-secret", "default"), + testutil.WithPullSecret("ps-missing-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -99,7 +99,7 @@ func TestPullSecretMissingDegradedAndRecovery(t *testing.T) { // Create the missing secret and verify recovery. secretData := []byte(`{"auths":{"recovered":{}}}`) - secret := newDockerConfigSecret("ps-missing-secret", "default", secretData) + secret := newDockerConfigSecret("ps-missing-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -121,7 +121,7 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { ctx := context.Background() secretData := []byte(`{"auths":{"remove":{}}}`) - secret := newDockerConfigSecret("ps-remove-secret", "default", secretData) + secret := newDockerConfigSecret("ps-remove-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -131,7 +131,7 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { pool := testutil.NewPool("ps-remove-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-remove-secret", "default"), + testutil.WithPullSecret("ps-remove-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -156,3 +156,115 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { return bn.Spec.PullSecretRef, err }).Should(BeNil()) } + +func TestPullSecretNamespaceDeniedAndRecovery(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + deniedNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ps-denied-ns"}} + g.Expect(k8sClient.Create(ctx, deniedNS)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, deniedNS) }) + + secretData := []byte(`{"auths":{"denied":{}}}`) + deniedSecret := newDockerConfigSecret("ps-denied-secret", deniedNS.Name, secretData) + g.Expect(k8sClient.Create(ctx, deniedSecret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, deniedSecret) }) + + node := testutil.NewK8sNode("ps-denied-node", testutil.WorkerLabels()) + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + pool := testutil.NewPool("ps-denied-pool", testImageDigestRefA, + testutil.WithWorkerSelector(), + testutil.WithPullSecret(deniedSecret.Name, deniedSecret.Namespace), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + // The secret exists, but its namespace is not in the allowlist. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolSecretError), + HaveField("Message", ContainSubstring(`"ps-denied-ns"`)), + ))) + + g.Consistently(func() error { + return k8sClient.Get( + ctx, + client.ObjectKey{Name: "ps-denied-node"}, + &bootcv1alpha1.BootcNode{}, + ) + }, "2s", pollInterval).ShouldNot(Succeed()) + + // Move the secret to the operator namespace and point the pool at it. + allowedSecret := newDockerConfigSecret( + "ps-denied-secret", + testutil.OperatorNamespaceName, + secretData, + ) + g.Expect(k8sClient.Create(ctx, allowedSecret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, allowedSecret) }) + + g.Eventually(func() error { + var p bootcv1alpha1.BootcNodePool + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p); err != nil { + return err + } + p.Spec.PullSecretRef.Namespace = testutil.OperatorNamespaceName + return k8sClient.Update(ctx, &p) + }).Should(Succeed()) + + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolHealthy), + ))) + + g.Eventually(func() (*bootcv1alpha1.PullSecretRef, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "ps-denied-node"}, &bn) + return bn.Spec.PullSecretRef, err + }).Should(Equal(&bootcv1alpha1.PullSecretRef{ + Name: "ps-denied-secret", Namespace: testutil.OperatorNamespaceName, + })) +} + +func TestParsePullSecretNamespaces(t *testing.T) { + tests := []struct { + name string + value string + opNS string + want []string + wantErr bool + }{ + {"empty", "", "op", []string{"op"}, false}, + {"only separators", " , ,", "op", []string{"op"}, false}, + {"unknown operator namespace", "", "", nil, false}, + {"operator namespace prepended", "team-a", "op", []string{"op", "team-a"}, false}, + {"trims and dedupes", " team-a ,,op,team-a", "op", []string{"op", "team-a"}, false}, + {"invalid name", "Team-A", "op", nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := ParsePullSecretNamespaces(tt.value, tt.opNS) + if tt.wantErr { + g.Expect(err).To(MatchError(ContainSubstring(`invalid namespace "Team-A"`))) + return + } + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index a85cb868..4dd5ec35 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -60,11 +61,15 @@ func TestMain(m *testing.M) { // Start a manager with the reconciler so controller logic runs // against the envtest API server. + pullSecretNamespaces := []string{testutil.OperatorNamespaceName} mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, Metrics: metricsserver.Options{ BindAddress: "0", // disable metrics server in tests }, + Cache: cache.Options{ + ByObject: SecretCacheByObject(pullSecretNamespaces), + }, }) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create manager: %v\n", err) @@ -84,10 +89,11 @@ func TestMain(m *testing.M) { } testReconciler = &BootcNodePoolReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - KubeClient: kubeClient, - EventNamespace: testutil.OperatorNamespaceName, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + KubeClient: kubeClient, + EventNamespace: testutil.OperatorNamespaceName, + PullSecretNamespaces: pullSecretNamespaces, } if err := testReconciler.SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "Failed to setup reconciler: %v\n", err) diff --git a/test/e2e/pullsecret_namespace_test.go b/test/e2e/pullsecret_namespace_test.go new file mode 100644 index 00000000..eae7c901 --- /dev/null +++ b/test/e2e/pullsecret_namespace_test.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "slices" + "testing" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// TestPullSecretNamespaces verifies that pools can only use pull secrets +// from the namespaces the operator is configured to read from. The +// operator must already allow E2E_PULL_SECRET_NAMESPACE (make deploy-bink +// does). A pool referencing a secret in a namespace the test creates must +// be Degraded without claiming its node; switched to a secret in the +// allowed namespace, its node must update through the authenticated +// registry. +func TestPullSecretNamespaces(t *testing.T) { + e2eutil.Providers(t, "bink", "eks") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + if env.RegistryUser() == "" || env.RegistryPassword() == "" { + t.Skip("E2E_REGISTRY_USER / E2E_REGISTRY_PASSWORD not set") + } + allowedNS := os.Getenv("E2E_PULL_SECRET_NAMESPACE") + if allowedNS == "" { + t.Skip("E2E_PULL_SECRET_NAMESPACE not set") + } + + ctx := context.Background() + if !slices.Contains(controllerPullSecretNamespaces(t, g, ctx, env.Client), allowedNS) { + t.Skipf("operator does not allow pull secrets from %q", allowedNS) + } + + deniedNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: env.TestID() + "-denied"}} + g.Expect(env.Client.Create(ctx, deniedNS)).To(Succeed()) + t.Cleanup(func() { _ = env.Client.Delete(context.Background(), deniedNS) }) + + nodeName := env.AddNode(t) + + deniedSecret := registrySecret(env, env.TestID()+"-pull-secret", deniedNS.Name) + g.Expect(env.Client.Create(ctx, deniedSecret)).To(Succeed()) + allowedSecret := registrySecret(env, env.TestID()+"-pull-secret", allowedNS) + g.Expect(env.Client.Create(ctx, allowedSecret)).To(Succeed()) + t.Cleanup(func() { _ = env.Client.Delete(context.Background(), allowedSecret) }) + + authImageRef := env.AuthImageRef() + pool := env.NewPool("psns", authImageRef, + testutil.WithPullSecret(deniedSecret.Name, deniedSecret.Namespace), + ) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + // Phase 1: the secret exists, but its namespace is not allowed. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolSecretError), + HaveField("Message", ContainSubstring(fmt.Sprintf("%q", deniedNS.Name))), + ))) + g.Consistently(func() bool { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return apierrors.IsNotFound(err) + }).WithTimeout(10*time.Second).Should(BeTrue(), + "no BootcNode should be created for a pool with a denied secret") + + t.Logf("Pool %q is Degraded for secret in namespace %q", pool.Name, deniedNS.Name) + + // Phase 2: a secret in the allowed namespace is used to pull the image. + modified := pool.DeepCopy() + modified.Spec.PullSecretRef = &bootcv1alpha1.PullSecretRef{ + Name: allowedSecret.Name, Namespace: allowedSecret.Namespace, + } + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + HaveField("Booted", HaveField("Image", Equal(authImageRef))), + ) + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) + + t.Logf("Node %q updated with a pull secret from namespace %q", nodeName, allowedNS) +} + +// registrySecret returns a dockerconfigjson Secret with the credentials of +// the authenticated e2e registry. +func registrySecret(env *e2eutil.Env, name, namespace string) *corev1.Secret { + auth := base64.StdEncoding.EncodeToString( + []byte(env.RegistryUser() + ":" + env.RegistryPassword()), + ) + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: fmt.Appendf(nil, + `{"auths":{%q:{"auth":%q}}}`, env.RegistryHost(), auth), + }, + } +} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index da7ff675..95f94b60 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -32,9 +32,9 @@ var operatorCRDNames = []string{ } // TestOperatorUpgrade installs the released version of the operator -// via Helm, verifies basic functionality, then upgrades by running -// helm upgrade with the current image and verifies the operator still -// works. +// from its published Helm chart, verifies basic functionality, then +// upgrades by running helm upgrade with the current chart and image and +// verifies the operator still works. func TestOperatorUpgrade(t *testing.T) { e2eutil.Providers(t, "bink", "eks") @@ -42,10 +42,14 @@ func TestOperatorUpgrade(t *testing.T) { if releaseTag == "" { t.Skip("E2E_OPERATOR_RELEASE_TAG not set") } + // Optional: overrides the image the released chart deploys, e.g. + // with a copy in a local registry. releasedImg := os.Getenv("E2E_OPERATOR_RELEASED_IMG") - if releasedImg == "" { - t.Skip("E2E_OPERATOR_RELEASED_IMG not set") + releasedChart := os.Getenv("E2E_OPERATOR_RELEASED_CHART") + if releasedChart == "" { + t.Skip("E2E_OPERATOR_RELEASED_CHART not set") } + releasedChartVersion := strings.TrimPrefix(releaseTag, "v") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -63,24 +67,32 @@ func TestOperatorUpgrade(t *testing.T) { g.Expect(env.Client.Get(ctx, operatorDeployKey(), &origDeploy)).To(Succeed()) currentImg := containerImage(t, origDeploy.Spec.Template.Spec.Containers, "manager") extraArgs := controllerExtraArgs(t, origDeploy.Spec.Template.Spec.Containers) + pullSecretNamespaces := controllerPullSecretNamespaces(t, g, ctx, env.Client) t.Logf("Current operator image: %s", currentImg) - t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) + t.Logf("Released operator tag: %s, image: %s, chart: %s", + releaseTag, releasedImg, releasedChart) - // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // Phase 1: Uninstall the current operator and re-install the + // released version from its published chart, the way users + // upgrading from it have it deployed. The current chart may not work + // with the released image. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) - waitForOperatorReady(t, g, ctx, env.Client) - - t.Logf("Released operator %s installed via Helm", releaseTag) + // Restore the current operator even if the released one fails to + // install, so the tests that run after this one have an operator. t.Cleanup(func() { t.Logf("Restoring operator to current version...") - helmUpgradeOperator(t, chartPath, currentImg, extraArgs) + helmUpgradeOperator(t, chartPath, currentImg, extraArgs, pullSecretNamespaces) waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Operator restored") }) + helmInstallOperator(t, releasedChart, releasedChartVersion, + releasedImg, extraArgs) + waitForOperatorReady(t, g, ctx, env.Client) + + t.Logf("Released operator %s installed via Helm", releaseTag) + // Phase 2: Verify the released version works. nodeName := env.AddNode(t) @@ -92,7 +104,7 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Node %q is Idle with released operator", nodeName) // Phase 3: Upgrade via helm upgrade to the current version. - helmUpgradeOperator(t, chartPath, currentImg, extraArgs) + helmUpgradeOperator(t, chartPath, currentImg, extraArgs, pullSecretNamespaces) waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Upgraded operator to current version via Helm") @@ -181,12 +193,17 @@ func uninstallOperator( t.Logf("Operator uninstalled") } -// helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmInstallOperator installs the given version of a published chart +// via helm install. +func helmInstallOperator( + t *testing.T, + chartRef, chartVersion, img string, + extraArgs []string, +) { t.Helper() - helmRun(t, "dependency", "build", chartPath) args := []string{ - "install", helmReleaseName, chartPath, + "install", helmReleaseName, chartRef, + "--version", chartVersion, "--namespace", testutil.OperatorNamespaceName, "--create-namespace", } @@ -194,25 +211,42 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string helmRun(t, args...) } -// helmUpgradeOperator upgrades the operator via helm upgrade. -func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmUpgradeOperator upgrades the operator to the local chart via helm +// upgrade. The release namespace is always allowed, so including it in +// pullSecretNamespaces is harmless. +func helmUpgradeOperator( + t *testing.T, + chartPath, img string, + extraArgs, pullSecretNamespaces []string, +) { t.Helper() + helmRun(t, "dependency", "build", chartPath) args := []string{ "upgrade", "--install", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, } args = append(args, helmImageArgs(t, img, extraArgs)...) + if len(pullSecretNamespaces) > 0 { + j, err := json.Marshal(pullSecretNamespaces) + if err != nil { + t.Fatalf("marshaling pullSecretNamespaces: %v", err) + } + args = append(args, "--set-json", "pullSecretNamespaces="+string(j)) + } helmRun(t, args...) } // helmImageArgs returns the --set flags for image and controller args. +// An empty img keeps the image of the chart. func helmImageArgs(t *testing.T, img string, extraArgs []string) []string { t.Helper() - repo, tag := splitImage(img) - args := []string{ - "--set-string", "image.repository=" + repo, - "--set-string", "image.tag=" + tag, - "--set-string", "image.pullPolicy=Always", + args := []string{"--set-string", "image.pullPolicy=Always"} + if img != "" { + repo, tag := splitImage(img) + args = append(args, + "--set-string", "image.repository="+repo, + "--set-string", "image.tag="+tag, + ) } if secret := os.Getenv("E2E_IMAGE_PULL_SECRET"); secret != "" { args = append(args, "--set", "imagePullSecrets[0].name="+secret) @@ -288,7 +322,8 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { for _, arg := range c.Args { switch { case arg == "--leader-elect", - strings.HasPrefix(arg, "--health-probe-bind-address="): + strings.HasPrefix(arg, "--health-probe-bind-address="), + strings.HasPrefix(arg, "--pull-secret-namespaces="): default: extra = append(extra, arg) } @@ -299,6 +334,31 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { return nil } +// controllerPullSecretNamespaces returns the namespaces the running +// controller allows pull secrets from, as given by its +// --pull-secret-namespaces flag. +func controllerPullSecretNamespaces( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, +) []string { + t.Helper() + var d appsv1.Deployment + g.Expect(c.Get(ctx, operatorDeployKey(), &d)).To(Succeed()) + for _, cont := range d.Spec.Template.Spec.Containers { + if cont.Name != "manager" { + continue + } + for _, arg := range cont.Args { + if v, ok := strings.CutPrefix(arg, "--pull-secret-namespaces="); ok { + return strings.Split(v, ",") + } + } + } + return nil +} + // waitForOperatorReady polls until the operator deployment has one available and up-to-date replica. func waitForOperatorReady( t *testing.T,