🌱 Ensure COS phase immutability for referenced object approach

api/v1/clusterobjectset_types.go

@@ -510,6 +510,31 @@ type ClusterObjectSetStatus struct {
 	// +listMapKey=type
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// observedPhases records the content hashes of resolved phases
+	// at first successful reconciliation. This is used to detect if
+	// referenced object sources were deleted and recreated with
+	// different content. Each entry covers all fully-resolved object
+	// manifests within a phase, making it source-agnostic.
+	//
+	// +listType=map
+	// +listMapKey=name
+	// +optional
+	ObservedPhases []ObservedPhase `json:"observedPhases,omitempty"`
+}
+
+// ObservedPhase records the observed content digest of a resolved phase.
+type ObservedPhase struct {
+	// name is the phase name matching a phase in spec.phases.
+	//
+	// +required
+	Name string `json:"name"`
+
+	// digest is the hex-encoded SHA-256 digest of the phase's resolved
+	// object content at first successful resolution.
+	//
+	// +required
+	Digest string `json:"digest"`
 }
 
 // +genclient

docs/api-reference/crd-ref-docs-gen-config.yaml

@@ -1,5 +1,5 @@
 processor:
-  ignoreTypes: [ClusterObjectSet, ClusterObjectSetList]
+  ignoreTypes: [ClusterObjectSet, ClusterObjectSetList, ObservedPhase]
   ignoreFields: []
 
 render:

docs/draft/concepts/large-bundle-support.md

@@ -142,14 +142,18 @@ Recommended conventions:
 1. **Secret type**: Secrets should use the dedicated type
    `olm.operatorframework.io/object-data` to distinguish them from user-created
    Secrets and enable easy identification. The system always sets this type on
-   Secrets it creates. The reconciler does not enforce the type when resolving
-   refs — Secrets with any type are accepted — but producers should set it for
-   consistency.
-
-2. **Immutability**: Secrets should set `immutable: true`. Because COS phases
-   are immutable, the content backing a ref should not change after creation.
-   Mutable referenced Secrets are not rejected, but modifying them after the
-   COS is created leads to undefined behavior.
+   Secrets it creates. The reconciler does not enforce the Secret type when
+   resolving refs, but it does enforce that referenced Secrets have
+   `immutable: true` set and that their content has not changed since first
+   resolution.
+
+2. **Immutability**: Secrets must set `immutable: true`. The reconciler verifies
+   that all referenced Secrets have `immutable: true` set before proceeding.
+   Mutable referenced Secrets are rejected and reconciliation is blocked with
+   `Progressing=False, Reason=Blocked`. Additionally, the reconciler records
+   content hashes of the resolved phases on first successful reconciliation
+   and blocks reconciliation if the content changes (e.g., if a Secret is
+   deleted and recreated with the same name but different data).
 
 3. **Owner references**: Referenced Secrets should carry an ownerReference to
    the COS so that Kubernetes garbage collection removes them when the COS is
@@ -388,6 +392,14 @@ Key properties:
 
 ### COS reconciler behavior
 
+Before resolving individual object refs, the reconciler verifies that all
+referenced Secrets have `immutable: true` set. After successfully building
+the phases (resolving all refs), the reconciler computes a per-phase content
+digest and compares it against the digests recorded in `.status.observedPhases`
+(if present). If any phase's content has changed, reconciliation is blocked
+with `Progressing=False, Reason=Blocked`. On first successful build, phase
+content digests are persisted to status for future comparisons.
+
 When processing a COS phase:
 - For each object entry in the phase:
   - If `object` is set, use it directly (current behavior, unchanged).

helm/olmv1/base/operator-controller/crd/experimental/olm.operatorframework.io_clusterobjectsets.yaml

@@ -621,6 +621,33 @@ spec:
                 x-kubernetes-list-map-keys:
                 - type
                 x-kubernetes-list-type: map
+              observedPhases:
+                description: |-
+                  observedPhases records the content hashes of resolved phases
+                  at first successful reconciliation. This is used to detect if
+                  referenced object sources were deleted and recreated with
+                  different content. Each entry covers all fully-resolved object
+                  manifests within a phase, making it source-agnostic.
+                items:
+                  description: ObservedPhase records the observed content digest of
+                    a resolved phase.
+                  properties:
+                    digest:
+                      description: |-
+                        digest is the hex-encoded SHA-256 digest of the phase's resolved
+                        object content at first successful resolution.
+                      type: string
+                    name:
+                      description: name is the phase name matching a phase in spec.phases.
+                      type: string
+                  required:
+                  - digest
+                  - name
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
             type: object
         type: object
     served: true

internal/operator-controller/controllers/clusterobjectset_controller.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -15,6 +16,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -144,12 +146,36 @@ func (c *ClusterObjectSetReconciler) reconcile(ctx context.Context, cos *ocv1.Cl
 		return c.delete(ctx, cos)
 	}
 
+	if err := c.verifyReferencedSecretsImmutable(ctx, cos); err != nil {
+		l.Error(err, "referenced Secret verification failed, blocking reconciliation")
+		markAsNotProgressing(cos, ocv1.ClusterObjectSetReasonBlocked, err.Error())
+		return ctrl.Result{}, nil
+	}
+
 	phases, opts, err := c.buildBoxcutterPhases(ctx, cos)
 	if err != nil {
 		setRetryingConditions(cos, err.Error())
 		return ctrl.Result{}, fmt.Errorf("converting to boxcutter revision: %v", err)
 	}
 
+	currentPhases := make([]ocv1.ObservedPhase, 0, len(phases))
+	for _, phase := range phases {
+		hash, err := computePhaseDigest(phase)
+		if err != nil {
+			setRetryingConditions(cos, err.Error())
+			return ctrl.Result{}, fmt.Errorf("computing phase digest: %v", err)
+		}
+		currentPhases = append(currentPhases, ocv1.ObservedPhase{Name: phase.GetName(), Digest: hash})
+	}
+
+	if len(cos.Status.ObservedPhases) == 0 {
+		cos.Status.ObservedPhases = currentPhases
+	} else if err := verifyObservedPhases(cos.Status.ObservedPhases, currentPhases); err != nil {
+		l.Error(err, "resolved phases content changed, blocking reconciliation")
+		markAsNotProgressing(cos, ocv1.ClusterObjectSetReasonBlocked, err.Error())
+		return ctrl.Result{}, nil
+	}
+
 	revisionEngine, err := c.RevisionEngineFactory.CreateRevisionEngine(ctx, cos)
 	if err != nil {
 		setRetryingConditions(cos, err.Error())
@@ -343,8 +369,10 @@ func (c *ClusterObjectSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			if !rev.DeletionTimestamp.IsZero() {
 				return true
 			}
-			if cnd := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeProgressing); cnd != nil && cnd.Status == metav1.ConditionFalse && cnd.Reason == ocv1.ReasonProgressDeadlineExceeded {
-				return false
+			if cnd := meta.FindStatusCondition(rev.Status.Conditions, ocv1.ClusterObjectSetTypeProgressing); cnd != nil && cnd.Status == metav1.ConditionFalse {
+				if cnd.Reason == ocv1.ReasonProgressDeadlineExceeded {
+					return false
+				}
 			}
 			return true
 		},
@@ -693,3 +721,82 @@ func markAsArchived(cos *ocv1.ClusterObjectSet) bool {
 	updated := markAsNotProgressing(cos, ocv1.ClusterObjectSetReasonArchived, msg)
 	return markAsAvailableUnknown(cos, ocv1.ClusterObjectSetReasonArchived, msg) || updated
 }
+
+// computePhaseDigest computes a deterministic SHA-256 digest of a phase's
+// resolved content (name + objects). JSON serialization of each object
+// produces a canonical encoding with sorted map keys.
+func computePhaseDigest(phase boxcutter.Phase) (string, error) {
+	h := sha256.New()
+	fmt.Fprintf(h, "phase:%s\n", phase.GetName())
+	for _, obj := range phase.GetObjects() {
+		data, err := json.Marshal(obj)
+		if err != nil {
+			return "", fmt.Errorf("marshaling object in phase %q: %w", phase.GetName(), err)
+		}
+		h.Write(data)
+		h.Write([]byte{'\n'})
+	}
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// verifyObservedPhases compares current per-phase digests against stored
+// digests. Returns an error naming the first mismatched phase.
+func verifyObservedPhases(stored, current []ocv1.ObservedPhase) error {
+	storedMap := make(map[string]string, len(stored))
+	for _, s := range stored {
+		storedMap[s.Name] = s.Digest
+	}
+	for _, c := range current {
+		if prev, ok := storedMap[c.Name]; ok && prev != c.Digest {
+			return fmt.Errorf(
+				"resolved content of phase %q has changed (expected digest %s, got %s): "+
+					"a referenced object source may have been deleted and recreated with different content",
+				c.Name, prev, c.Digest)
+		}
+	}
+	return nil
+}
+
+// verifyReferencedSecretsImmutable checks that all referenced Secrets
+// have Immutable set to true. This is a fast-fail validation that
+// provides a clear error message for misconfigured Secrets.
+func (c *ClusterObjectSetReconciler) verifyReferencedSecretsImmutable(ctx context.Context, cos *ocv1.ClusterObjectSet) error {
+	type secretRef struct {
+		name      string
+		namespace string
+	}
+	seen := make(map[secretRef]struct{})
+	var refs []secretRef
+
+	for _, phase := range cos.Spec.Phases {
+		for _, obj := range phase.Objects {
+			if obj.Ref.Name == "" {
+				continue
+			}
+			sr := secretRef{name: obj.Ref.Name, namespace: obj.Ref.Namespace}
+			if _, ok := seen[sr]; !ok {
+				seen[sr] = struct{}{}
+				refs = append(refs, sr)
+			}
+		}
+	}
+
+	for _, ref := range refs {
+		secret := &corev1.Secret{}
+		key := client.ObjectKey{Name: ref.name, Namespace: ref.namespace}
+		if err := c.Client.Get(ctx, key, secret); err != nil {
+			if apierrors.IsNotFound(err) {
+				// Secret not yet available — skip verification.
+				// resolveObjectRef will handle the not-found with a retryable error.
+				continue
+			}
+			return fmt.Errorf("getting Secret %s/%s: %w", ref.namespace, ref.name, err)
+		}
+
+		if secret.Immutable == nil || !*secret.Immutable {
+			return fmt.Errorf("secret %s/%s is not immutable: referenced secrets must have immutable set to true", ref.namespace, ref.name)
+		}
+	}
+
+	return nil
+}