diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 011385d..fc45a52 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -153,15 +153,6 @@ jobs: - name: Set up KVM run: sudo chmod 666 /dev/kvm - - name: Configure kernel - run: | - # Unload AppArmor profiles — the passt profile blocks remount - # operations needed for passt's self-sandboxing inside containers. - sudo aa-teardown 2>/dev/null || true - # Allow unprivileged user namespace creation (needed by passt - # inside containers). - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Enable KSM run: | sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' @@ -172,6 +163,17 @@ jobs: sudo apt-get update sudo apt-get install -y podman + - name: Configure kernel + run: | + # Unload AppArmor profiles — the host-loaded passt profile + # blocks passt's self-sandboxing inside bink node containers. + # Must run AFTER installing podman, whose passt dependency + # loads the profile during package installation. + sudo aa-teardown 2>/dev/null || true + # Allow unprivileged user namespace creation (needed by passt + # inside containers). + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Start podman socket run: systemctl --user start podman.socket diff --git a/Makefile b/Makefile index 5fec160..b3beaad 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= ARTIFACTS=$(ARTIFACTS) \ BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \ BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \ - go test -timeout 20m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) . + go test -timeout 30m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) . ##@ Build diff --git a/api/v1alpha1/bootcnode_types.go b/api/v1alpha1/bootcnode_types.go index 88079fb..1e26e44 100644 --- a/api/v1alpha1/bootcnode_types.go +++ b/api/v1alpha1/bootcnode_types.go @@ -158,7 +158,7 @@ type BootcNodeStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Cluster,shortName=bn // BootcNode represents a single node managed by the bootc operator. // BootcNode objects are auto-created by the controller (one per managed diff --git a/api/v1alpha1/bootcnodepool_types.go b/api/v1alpha1/bootcnodepool_types.go index e750ff3..6fc5f6f 100644 --- a/api/v1alpha1/bootcnodepool_types.go +++ b/api/v1alpha1/bootcnodepool_types.go @@ -211,7 +211,11 @@ type BootcNodePoolStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Cluster,shortName=bnp +// +kubebuilder:printcolumn:name="Nodes",type=integer,JSONPath=`.status.nodeCount` +// +kubebuilder:printcolumn:name="Updated",type=integer,JSONPath=`.status.updatedCount` +// +kubebuilder:printcolumn:name="Updating",type=integer,JSONPath=`.status.updatingCount` +// +kubebuilder:printcolumn:name="Degraded",type=integer,JSONPath=`.status.degradedCount` // BootcNodePool defines a group of nodes and their desired OS image state. // Users create BootcNodePool resources to register nodes with the bootc diff --git a/config/crd/bases/node.bootc.dev_bootcnodepools.yaml b/config/crd/bases/node.bootc.dev_bootcnodepools.yaml index 7e8da3e..9ce421a 100644 --- a/config/crd/bases/node.bootc.dev_bootcnodepools.yaml +++ b/config/crd/bases/node.bootc.dev_bootcnodepools.yaml @@ -11,10 +11,25 @@ spec: kind: BootcNodePool listKind: BootcNodePoolList plural: bootcnodepools + shortNames: + - bnp singular: bootcnodepool scope: Cluster versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.nodeCount + name: Nodes + type: integer + - jsonPath: .status.updatedCount + name: Updated + type: integer + - jsonPath: .status.updatingCount + name: Updating + type: integer + - jsonPath: .status.degradedCount + name: Degraded + type: integer + name: v1alpha1 schema: openAPIV3Schema: description: |- diff --git a/config/crd/bases/node.bootc.dev_bootcnodes.yaml b/config/crd/bases/node.bootc.dev_bootcnodes.yaml index 88538fc..a769bf1 100644 --- a/config/crd/bases/node.bootc.dev_bootcnodes.yaml +++ b/config/crd/bases/node.bootc.dev_bootcnodes.yaml @@ -11,6 +11,8 @@ spec: kind: BootcNode listKind: BootcNodeList plural: bootcnodes + shortNames: + - bn singular: bootcnode scope: Cluster versions: diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 6ffe67e..e9fc387 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -172,7 +172,7 @@ the full controller+daemon loop can be tested end-to-end. NotReady), verify the controller stops assigning new slots even when `maxUnavailable` has capacity. -### 3c. Pool status aggregation +### 3c. Pool status aggregation ✅ - Compute `nodeCount`, `updatedCount`, `updatingCount`, `degradedCount` - `UpToDate` condition with reasons: `AllUpdated`, `RolloutInProgress`, diff --git a/internal/controller/bootcnodepool_controller.go b/internal/controller/bootcnodepool_controller.go index a859bd3..59eb7fe 100644 --- a/internal/controller/bootcnodepool_controller.go +++ b/internal/controller/bootcnodepool_controller.go @@ -287,13 +287,19 @@ func (r *BootcNodePoolReconciler) Reconcile(ctx context.Context, req ctrl.Reques // reconciliation run. // Drive the rollout state machine. - if err := r.driveRollout(ctx, &pool, ownedBootcNodes); err != nil { + rs, err := r.driveRollout(ctx, &pool, ownedBootcNodes) + if err != nil { if isInvalidSpecError(err) { return r.setInvalidSpecCondition(ctx, &pool, err) } return ctrl.Result{}, fmt.Errorf("driving rollout: %w", err) } + // Early-return paths above (TargetDigest empty, InvalidSpec) skip + // aggregation. In-flight updates may complete during error conditions + // but counts catch up on the next successful reconcile. + syncPoolStatus(&pool, rs) + return complete(resolveResult) } diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index 0ccbb58..17c9d1e 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -57,13 +57,13 @@ func (rs *rolloutState) nodeCount() int { } // driveRollout is the main function that advances the rollout state machine. -func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv1alpha1.BootcNodePool, ownedBootcNodes map[string]*bootcv1alpha1.BootcNode) error { +func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv1alpha1.BootcNodePool, ownedBootcNodes map[string]*bootcv1alpha1.BootcNode) (*rolloutState, error) { log := logf.FromContext(ctx) // Process drain results first. This isn't really ordering dependent, // but it feels natural to do this upfront before classifying. if err := r.collectDrainResults(ctx, ownedBootcNodes); err != nil { - return fmt.Errorf("collecting drain results: %w", err) + return nil, fmt.Errorf("collecting drain results: %w", err) } rs := buildRolloutState(log, ownedBootcNodes) @@ -80,7 +80,7 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv // the desired image. This runs before computing available slots so // that freed capacity is immediately usable for new candidates. if err := r.freeCompletedSlots(ctx, rs); err != nil { - return fmt.Errorf("freeing completed slots: %w", err) + return nil, fmt.Errorf("freeing completed slots: %w", err) } // Check for unhealthy nodes on the target digest in reboot slots. If @@ -107,12 +107,12 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv // reconciles will still have their results collected and // desiredImageState set to Booted. Trying to "un-drain" and // uncordon fully drained nodes is out of scope for now. - return nil + return rs, nil } maxUnavail, err := resolveMaxUnavailable(pool, rs.nodeCount()) if err != nil { - return err + return nil, err } availableSlots := max(0, maxUnavail-rs.occupiedSlots) @@ -136,10 +136,10 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv for _, bn := range candidates { var node corev1.Node if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err != nil { - return fmt.Errorf("fetching node %s: %w", bn.Name, err) + return nil, fmt.Errorf("fetching node %s: %w", bn.Name, err) } if err := r.assignRebootSlot(ctx, bn, &node); err != nil { - return fmt.Errorf("assigning reboot slot to %s: %w", bn.Name, err) + return nil, fmt.Errorf("assigning reboot slot to %s: %w", bn.Name, err) } } @@ -155,7 +155,7 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv r.ensureDrain(ctx, pool, bn) } - return nil + return rs, nil } // assignRebootSlot marks a BootcNode as occupying a reboot slot and diff --git a/internal/controller/status.go b/internal/controller/status.go new file mode 100644 index 0000000..62eec56 --- /dev/null +++ b/internal/controller/status.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "strings" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" +) + +// syncPoolStatus populates the pool's status counts, digest tracking, +// and UpToDate condition from the current rollout state. +func syncPoolStatus(pool *bootcv1alpha1.BootcNodePool, rs *rolloutState) { + if pool == nil || rs == nil { + return + } + pool.Status.ObservedGeneration = pool.Generation + pool.Status.NodeCount = int32(rs.nodeCount()) + pool.Status.UpdatedCount = int32(len(rs.upToDate)) + pool.Status.UpdatingCount = int32(len(rs.pending) + len(rs.staging) + len(rs.staged) + len(rs.rebooting)) + pool.Status.DegradedCount = int32(len(rs.degraded)) + + if pool.Status.NodeCount == pool.Status.UpdatedCount { + pool.Status.DeployedDigest = pool.Status.TargetDigest + } + pool.Status.UpdateAvailable = pool.Status.TargetDigest != pool.Status.DeployedDigest + + syncUpToDateCondition(pool, rs) +} + +func syncUpToDateCondition(pool *bootcv1alpha1.BootcNodePool, rs *rolloutState) { + if pool == nil || rs == nil { + return + } + switch { + case pool.Spec.Rollout != nil && pool.Spec.Rollout.Paused && pool.Status.NodeCount != pool.Status.UpdatedCount: + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.PoolUpToDate, + Status: metav1.ConditionFalse, + Reason: bootcv1alpha1.PoolPaused, + Message: rolloutBreakdown(pool, rs), + }) + case pool.Status.NodeCount != pool.Status.UpdatedCount: + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.PoolUpToDate, + Status: metav1.ConditionFalse, + Reason: bootcv1alpha1.PoolRolloutInProgress, + Message: rolloutBreakdown(pool, rs), + }) + default: + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.PoolUpToDate, + Status: metav1.ConditionTrue, + Reason: bootcv1alpha1.PoolAllUpdated, + }) + } +} + +func rolloutBreakdown(pool *bootcv1alpha1.BootcNodePool, rs *rolloutState) string { + var b strings.Builder + fmt.Fprintf(&b, "%d/%d updated", len(rs.upToDate), rs.nodeCount()) + + type bucket struct { + name string + count int + } + buckets := []bucket{ + {"pending", len(rs.pending)}, + {"staging", len(rs.staging)}, + {"staged", len(rs.staged)}, + {"rebooting", len(rs.rebooting)}, + {"degraded", len(rs.degraded)}, + } + + b.WriteString("; ") + for i, bk := range buckets { + if i > 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "%d %s", bk.count, bk.name) + } + return b.String() +} diff --git a/internal/controller/status_test.go b/internal/controller/status_test.go new file mode 100644 index 0000000..4402401 --- /dev/null +++ b/internal/controller/status_test.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// Verifies that when all nodes are running the target digest, the counts +// are correct, UpToDate is True/AllUpdated, and deployedDigest is set. +func TestSyncPoolStatusAllUpdated(t *testing.T) { + g := NewWithT(t) + + pool := testutil.NewPool("test", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + rs := &rolloutState{ + upToDate: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n1", testImageDigestRefA, testutil.WithBootedDigest(testDigestA)), + testutil.NewNode("n2", testImageDigestRefA, testutil.WithBootedDigest(testDigestA)), + testutil.NewNode("n3", testImageDigestRefA, testutil.WithBootedDigest(testDigestA)), + }, + } + + syncPoolStatus(pool, rs) + + g.Expect(pool.Status.NodeCount).To(Equal(int32(3))) + g.Expect(pool.Status.UpdatedCount).To(Equal(int32(3))) + g.Expect(pool.Status.UpdatingCount).To(Equal(int32(0))) + g.Expect(pool.Status.DegradedCount).To(Equal(int32(0))) + g.Expect(pool.Status.DeployedDigest).To(Equal(testDigestA)) + g.Expect(pool.Status.UpdateAvailable).To(BeFalse()) + g.Expect(pool.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolAllUpdated), + ))) +} + +// Verifies that a rollout in progress with nodes in every bucket produces +// the correct counts, UpToDate False/RolloutInProgress, and a breakdown message. +func TestSyncPoolStatusRolloutInProgress(t *testing.T) { + g := NewWithT(t) + + pool := testutil.NewPool("test", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + rs := &rolloutState{ + upToDate: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n1", testImageDigestRefA, testutil.WithBootedDigest(testDigestA)), + }, + pending: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n2", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + }, + staging: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n3", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + }, + staged: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n4", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + }, + rebooting: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n5", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + }, + } + + syncPoolStatus(pool, rs) + + g.Expect(pool.Status.NodeCount).To(Equal(int32(5))) + g.Expect(pool.Status.UpdatedCount).To(Equal(int32(1))) + g.Expect(pool.Status.UpdatingCount).To(Equal(int32(4))) + g.Expect(pool.Status.DegradedCount).To(Equal(int32(0))) + g.Expect(pool.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolRolloutInProgress), + HaveField("Message", Equal("1/5 updated; 1 pending, 1 staging, 1 staged, 1 rebooting, 0 degraded")), + ))) +} + +// Verifies that a paused pool with pending nodes reports UpToDate +// False/Paused and includes the breakdown message. +func TestSyncPoolStatusPaused(t *testing.T) { + g := NewWithT(t) + + pool := testutil.NewPool("test", testImageDigestRefA, testutil.WithWorkerSelector(), testutil.WithPaused(true)) + pool.Status.TargetDigest = testDigestA + + rs := &rolloutState{ + upToDate: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n1", testImageDigestRefA, testutil.WithBootedDigest(testDigestA)), + }, + pending: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n2", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + testutil.NewNode("n3", testImageDigestRefA, testutil.WithBootedDigest(testDigestB)), + }, + } + + syncPoolStatus(pool, rs) + + g.Expect(pool.Status.NodeCount).To(Equal(int32(3))) + g.Expect(pool.Status.UpdatedCount).To(Equal(int32(1))) + g.Expect(pool.Status.UpdatingCount).To(Equal(int32(2))) + g.Expect(pool.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolPaused), + HaveField("Message", Equal("1/3 updated; 2 pending, 0 staging, 0 staged, 0 rebooting, 0 degraded")), + ))) +} + +// Verifies that an empty pool with no nodes is vacuously considered +// up-to-date, with all counts at zero and deployedDigest set. +func TestSyncPoolStatusEmptyPool(t *testing.T) { + g := NewWithT(t) + + pool := testutil.NewPool("test", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + rs := &rolloutState{} + + syncPoolStatus(pool, rs) + + g.Expect(pool.Status.NodeCount).To(Equal(int32(0))) + g.Expect(pool.Status.UpdatedCount).To(Equal(int32(0))) + g.Expect(pool.Status.UpdatingCount).To(Equal(int32(0))) + g.Expect(pool.Status.DegradedCount).To(Equal(int32(0))) + g.Expect(pool.Status.DeployedDigest).To(Equal(testDigestA)) + g.Expect(pool.Status.UpdateAvailable).To(BeFalse()) + g.Expect(pool.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolAllUpdated), + ))) +} + +// Verifies that deployedDigest retains its previous value while a rollout +// is in progress, and updateAvailable is true. +func TestSyncPoolStatusDeployedDigestPreserved(t *testing.T) { + g := NewWithT(t) + + pool := testutil.NewPool("test", testImageDigestRefB, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestB + pool.Status.DeployedDigest = testDigestA + + rs := &rolloutState{ + pending: []*bootcv1alpha1.BootcNode{ + testutil.NewNode("n1", testImageDigestRefB, testutil.WithBootedDigest(testDigestA)), + }, + } + + syncPoolStatus(pool, rs) + + g.Expect(pool.Status.DeployedDigest).To(Equal(testDigestA)) + g.Expect(pool.Status.UpdateAvailable).To(BeTrue()) +} diff --git a/internal/daemon/reconciler.go b/internal/daemon/reconciler.go index d5222b6..d670e20 100644 --- a/internal/daemon/reconciler.go +++ b/internal/daemon/reconciler.go @@ -16,10 +16,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" @@ -66,7 +68,7 @@ func (r *BootcNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { r.stageDone = make(chan event.GenericEvent, 1) return ctrl.NewControllerManagedBy(mgr). - For(&bootcv1alpha1.BootcNode{}). + For(&bootcv1alpha1.BootcNode{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WatchesRawSource(source.Channel(r.stageDone, &handler.EnqueueRequestForObject{})). WatchesRawSource(source.Channel(r.StatusWatcher.Events, &handler.EnqueueRequestForObject{})). Named("bootcnode"). diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 7ddf51e..120b755 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -11,6 +11,7 @@ import ( "time" . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -92,6 +93,9 @@ func TestControllerMembership(t *testing.T) { HaveField("Reason", bootcv1alpha1.NodeReasonIdle), ))), )) + + // Verify pool status reflects steady state. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(poolAllUpdated(1, env.NodeImageDigest())) } // TestUpdateReboot provisions a worker node, creates a pool with the @@ -150,6 +154,19 @@ func TestUpdateReboot(t *testing.T) { t.Logf("Node %q is Rebooting", nodeName) + // Verify pool status during rollout. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(And( + HaveField("NodeCount", BeEquivalentTo(1)), + HaveField("UpdatedCount", BeEquivalentTo(0)), + HaveField("UpdatingCount", BeEquivalentTo(1)), + HaveField("UpdateAvailable", BeTrue()), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolRolloutInProgress), + ))), + )) + // Phase 4: Wait for Idle with the update digest — proves the full // update lifecycle completed (staging, reboot, boot into new image). g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { @@ -170,6 +187,9 @@ func TestUpdateReboot(t *testing.T) { t.Logf("Node %q is Idle with update image", nodeName) + // Verify pool status after rollout completes. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) + // Phase 5: Verify node is schedulable (uncordoned after reboot). g.Eventually(func() (bool, error) { var node corev1.Node @@ -256,6 +276,9 @@ func TestUpdateReboot(t *testing.T) { ), "expected node to reach Idle with original image after rollback") t.Logf("Node %q successfully rolled back to original image", nodeName) + + // Verify pool status after rollback completes. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(poolAllUpdated(1, env.NodeImageDigest())) } // TestTagResolution creates a pool with a tag-based image ref, verifies @@ -416,6 +439,18 @@ func TestPauseResume(t *testing.T) { t.Logf("Node %q staged update but did not reboot (paused)", nodeName) + // Verify pool status while paused. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(And( + HaveField("NodeCount", BeEquivalentTo(1)), + HaveField("UpdatedCount", BeEquivalentTo(0)), + HaveField("UpdateAvailable", BeTrue()), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolPaused), + ))), + )) + // Verify the node stays Staged and does not proceed to reboot. g.Consistently(func() ([]metav1.Condition, error) { var bn2 bootcv1alpha1.BootcNode @@ -453,6 +488,9 @@ func TestPauseResume(t *testing.T) { ), "expected node to reach Idle with update image after resume") t.Logf("Node %q completed update after resume", nodeName) + + // Verify pool status after resume completes. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) } // TestNonExistingImage provisions a worker node, creates a pool with the @@ -517,6 +555,20 @@ func TestNonExistingImage(t *testing.T) { t.Logf("Node %q entered degraded state as expected", nodeName) + // Verify pool status reflects degraded node. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).Should(And( + HaveField("NodeCount", BeEquivalentTo(1)), + HaveField("UpdatedCount", BeEquivalentTo(0)), + HaveField("UpdatingCount", BeEquivalentTo(0)), + HaveField("DegradedCount", BeEquivalentTo(1)), + HaveField("UpdateAvailable", BeTrue()), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolNodeDegraded), + ))), + )) + // Phase 4: Verify the node did not stage the non-existing image. g.Expect(env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) g.Expect(bn.Status.Staged).To(BeNil(), @@ -524,3 +576,27 @@ func TestNonExistingImage(t *testing.T) { t.Logf("Verified node %q did not stage non-existing image", nodeName) } + +func fetchPoolStatus(ctx context.Context, c client.Client, pool *bootcv1alpha1.BootcNodePool) func() (bootcv1alpha1.BootcNodePoolStatus, error) { + return func() (bootcv1alpha1.BootcNodePoolStatus, error) { + var p bootcv1alpha1.BootcNodePool + err := c.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status, err + } +} + +func poolAllUpdated(nodeCount int32, deployedDigest string) types.GomegaMatcher { + return And( + HaveField("NodeCount", BeEquivalentTo(nodeCount)), + HaveField("UpdatedCount", BeEquivalentTo(nodeCount)), + HaveField("UpdatingCount", BeEquivalentTo(0)), + HaveField("DegradedCount", BeEquivalentTo(0)), + HaveField("DeployedDigest", Equal(deployedDigest)), + HaveField("UpdateAvailable", BeFalse()), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolAllUpdated), + ))), + ) +}