diff --git a/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml b/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml index 5eead8763..d11e5b6cf 100644 --- a/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml +++ b/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml @@ -1973,6 +1973,14 @@ spec: items: type: string type: array + finalizerHash: + description: |- + FinalizerHash is a short, deterministic hash derived from the nodeset name. + Used to create unique, collision-free finalizer names for RabbitMQ users. + Format: first 8 characters of SHA256(nodeset.metadata.name) + Example: "a3f2b5c8" + This allows easy lookup of which nodeset owns a specific finalizer. + type: string inventorySecretName: description: InventorySecretName Name of a secret containing the ansible inventory diff --git a/api/dataplane/v1beta1/openstackdataplanenodeset_types.go b/api/dataplane/v1beta1/openstackdataplanenodeset_types.go index 6731593ae..a06e297f6 100644 --- a/api/dataplane/v1beta1/openstackdataplanenodeset_types.go +++ b/api/dataplane/v1beta1/openstackdataplanenodeset_types.go @@ -160,6 +160,13 @@ type OpenStackDataPlaneNodeSetStatus struct { //DeployedBmhHash - Hash of BMHs deployed DeployedBmhHash string `json:"deployedBmhHash,omitempty"` + + // FinalizerHash is a short, deterministic hash derived from the nodeset name. + // Used to create unique, collision-free finalizer names for RabbitMQ users. + // Format: first 8 characters of SHA256(nodeset.metadata.name) + // Example: "a3f2b5c8" + // This allows easy lookup of which nodeset owns a specific finalizer. + FinalizerHash string `json:"finalizerHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/dataplane/v1beta1/openstackdataplanenodeset_webhook.go b/api/dataplane/v1beta1/openstackdataplanenodeset_webhook.go index 4a3db4537..e4cf5e172 100644 --- a/api/dataplane/v1beta1/openstackdataplanenodeset_webhook.go +++ b/api/dataplane/v1beta1/openstackdataplanenodeset_webhook.go @@ -27,6 +27,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" apimachineryvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" @@ -165,6 +166,27 @@ func (r *OpenStackDataPlaneNodeSet) ValidateUpdate(ctx context.Context, old runt for deployName, deployConditions := range oldNodeSet.Status.DeploymentStatuses { deployCondition := deployConditions.Get(NodeSetDeploymentReadyCondition) if !deployConditions.IsTrue(NodeSetDeploymentReadyCondition) && !condition.IsError(deployCondition) { + // Check if the deployment is being deleted - if so, allow the NodeSet update + deployment := &OpenStackDataPlaneDeployment{} + err := c.Get(ctx, types.NamespacedName{Name: deployName, Namespace: r.Namespace}, deployment) + if err != nil { + if apierrors.IsNotFound(err) { + // Deployment no longer exists, allow the update + continue + } + // If we can't check the deployment, log but don't block + openstackdataplanenodesetlog.Info("could not check deployment status during validation", + "deployment", deployName, "error", err) + continue + } + + // If deployment is being deleted, allow the NodeSet update + if deployment.DeletionTimestamp != nil { + openstackdataplanenodesetlog.Info("allowing NodeSet update because deployment is being deleted", + "deployment", deployName) + continue + } + return nil, apierrors.NewConflict( schema.GroupResource{Group: "dataplane.openstack.org", Resource: "OpenStackDataPlaneNodeSet"}, r.Name, diff --git a/bindata/crds/crds.yaml b/bindata/crds/crds.yaml index a3a1c0971..c2890f5dc 100644 --- a/bindata/crds/crds.yaml +++ b/bindata/crds/crds.yaml @@ -20543,6 +20543,14 @@ spec: items: type: string type: array + finalizerHash: + description: |- + FinalizerHash is a short, deterministic hash derived from the nodeset name. + Used to create unique, collision-free finalizer names for RabbitMQ users. + Format: first 8 characters of SHA256(nodeset.metadata.name) + Example: "a3f2b5c8" + This allows easy lookup of which nodeset owns a specific finalizer. + type: string inventorySecretName: description: InventorySecretName Name of a secret containing the ansible inventory diff --git a/bindata/rbac/rbac.yaml b/bindata/rbac/rbac.yaml index 20a134624..3a1e5a746 100644 --- a/bindata/rbac/rbac.yaml +++ b/bindata/rbac/rbac.yaml @@ -655,6 +655,23 @@ rules: - patch - update - watch +- apiGroups: + - rabbitmq.openstack.org + resources: + - rabbitmqusers + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - rabbitmq.openstack.org + resources: + - rabbitmqusers/finalizers + verbs: + - patch + - update - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml index 5eead8763..d11e5b6cf 100644 --- a/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml +++ b/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml @@ -1973,6 +1973,14 @@ spec: items: type: string type: array + finalizerHash: + description: |- + FinalizerHash is a short, deterministic hash derived from the nodeset name. + Used to create unique, collision-free finalizer names for RabbitMQ users. + Format: first 8 characters of SHA256(nodeset.metadata.name) + Example: "a3f2b5c8" + This allows easy lookup of which nodeset owns a specific finalizer. + type: string inventorySecretName: description: InventorySecretName Name of a secret containing the ansible inventory diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 92ee8f170..061a96ee8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -606,6 +606,23 @@ rules: - patch - update - watch +- apiGroups: + - rabbitmq.openstack.org + resources: + - rabbitmqusers + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - rabbitmq.openstack.org + resources: + - rabbitmqusers/finalizers + verbs: + - patch + - update - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/internal/controller/dataplane/manage_service_finalizers.go b/internal/controller/dataplane/manage_service_finalizers.go new file mode 100644 index 000000000..23b021206 --- /dev/null +++ b/internal/controller/dataplane/manage_service_finalizers.go @@ -0,0 +1,302 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dataplane + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "time" + + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/api/dataplane/v1beta1" + deployment "github.com/openstack-k8s-operators/openstack-operator/internal/dataplane" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // finalizerPrefix is the domain prefix for our finalizers + finalizerPrefix = "nodeset.os/" +) + +// computeFinalizerHash computes a deterministic 8-character hash from a nodeset name. +// Uses SHA256 and returns the first 8 hex characters. +// This hash is stored in the nodeset status and used to build unique finalizer names. +func computeFinalizerHash(nodesetName string) string { + hash := sha256.Sum256([]byte(nodesetName)) + return hex.EncodeToString(hash[:])[:8] +} + +// buildFinalizerName creates a unique, deterministic finalizer name using a hash. +// +// Format: nodeset.os/{8-char-hash}-{service} +// +// The hash is derived from SHA256(nodeset.metadata.name) and stored in the nodeset status +// for easy lookup of which nodeset owns a specific finalizer. +// +// Examples: +// - nodeset.os/a3f2b5c8-nova (28 chars) +// - nodeset.os/7e9d1234-neutron (30 chars) +// - nodeset.os/5a6b7c8d-ironic (29 chars) +// +// Benefits: +// 1. Guaranteed collision-free (SHA256 hash uniqueness) +// 2. Always fits within 63-char Kubernetes limit (max ~30 chars) +// 3. Deterministic (same nodeset → same hash → same finalizer) +// 4. Easy debugging via nodeset status (finalizerHash field) +func buildFinalizerName(finalizerHash, serviceName string) string { + return fmt.Sprintf("%s%s-%s", finalizerPrefix, finalizerHash, serviceName) +} + +// manageServiceFinalizers manages finalizers on RabbitMqUser CRs for a specific service +// This function: +// 1. ALWAYS adds finalizers to users currently in use (even during partial deployments) +// 2. Checks if all nodes for this service have been updated +// 3. Checks if all nodesets using the same RabbitMQ cluster for this service are updated +// 4. Only removes finalizers from old users when ALL nodes and ALL nodesets are updated +// +// This ensures credentials are protected as soon as they're in use, preventing accidental +// deletion during rolling updates or partial deployments. +func (r *OpenStackDataPlaneNodeSetReconciler) manageServiceFinalizers( + ctx context.Context, + helper *helper.Helper, + instance *dataplanev1.OpenStackDataPlaneNodeSet, + serviceName string, + serviceDisplayName string, + secretsLastModified map[string]time.Time, +) { + Log := r.GetLogger(ctx) + + // Get the service tracking data + tracking, err := deployment.GetServiceTracking(ctx, helper, instance.Name, instance.Namespace, serviceName) + if err != nil { + Log.Error(err, "Failed to get service tracking", "service", serviceName) + return + } + + // Safety check: Ensure we have a secret hash (meaning we're tracking secret changes) + if tracking.SecretHash == "" { + Log.Info("No secret hash set yet for service, skipping finalizer management", + "service", serviceName) + return + } + + // Check if at least some nodes have been updated (we need at least one to add finalizers) + allNodeNames := r.getAllNodeNamesFromNodeset(instance) + if len(tracking.UpdatedNodes) == 0 { + Log.Info("No nodes updated yet for service, skipping finalizer management", + "service", serviceName) + return + } + + // Check if all nodes in this nodeset are updated + allNodesInNodesetUpdated := len(tracking.UpdatedNodes) == len(allNodeNames) + + // Check if all nodesets using the same RabbitMQ cluster have been updated + // This includes tracking node coverage across multiple AnsibleLimit deployments + allNodesetsUpdated, err := r.allNodesetsUsingClusterUpdated(ctx, helper, instance, serviceName, secretsLastModified) + if err != nil { + Log.Error(err, "Failed to check if all nodesets are updated", "service", serviceName) + return + } + + // Determine if we should remove old finalizers + // Only safe to remove when ALL nodes across ALL nodesets are updated + shouldRemoveOldFinalizers := allNodesInNodesetUpdated && allNodesetsUpdated + + if shouldRemoveOldFinalizers { + Log.Info("Service deployed successfully and all nodes updated, managing RabbitMQ user finalizers", + "service", serviceDisplayName, + "updatedNodes", len(tracking.UpdatedNodes), + "totalNodes", len(allNodeNames)) + } else { + Log.Info("Adding finalizers to current RabbitMQ users (partial deployment in progress)", + "service", serviceDisplayName, + "updatedNodes", len(tracking.UpdatedNodes), + "totalNodes", len(allNodeNames), + "allNodesInNodesetUpdated", allNodesInNodesetUpdated, + "allNodesetsUpdated", allNodesetsUpdated) + } + + // Get the finalizer hash from nodeset status + // This hash is a deterministic SHA256-based identifier used to create unique finalizer names + // The hash is computed and stored during reconciliation + finalizerHash := instance.Status.FinalizerHash + if finalizerHash == "" { + Log.Error(fmt.Errorf("finalizerHash not set in nodeset status"), "Cannot manage finalizers without hash", + "nodeset", instance.Name) + return + } + + // Build the finalizer name: nodeset.os/{hash}-{service} + // This format is guaranteed unique, collision-free, and always fits in 63 chars + finalizerName := buildFinalizerName(finalizerHash, serviceName) + Log.Info("Using finalizer", "name", finalizerName, "hash", finalizerHash, "length", len(finalizerName)) + + // Track current usernames that should have our finalizer + currentUsernames := make(map[string]bool) + + // Get usernames based on service type + if serviceName == "nova" { + // Get Nova cell usernames + cellNames, err := deployment.GetNovaComputeConfigCellNames(ctx, helper, instance.Namespace) + if err != nil { + Log.Error(err, "Failed to get Nova cell names") + return + } + + for _, cellName := range cellNames { + username, err := deployment.GetNovaCellRabbitMqUserFromSecret(ctx, helper, instance.Namespace, cellName) + if err != nil { + Log.Info("Failed to get RabbitMQ username for Nova cell", "cell", cellName, "error", err) + continue + } + + if username != "" { + currentUsernames[username] = true + Log.Info("Found current RabbitMQ username for Nova cell", "cell", cellName, "username", username) + } + } + } else if serviceName == "neutron" { + // Get Neutron username (shared across DHCP and SRIOV agents) + neutronUsername, err := deployment.GetNeutronRabbitMqUserFromSecret(ctx, helper, instance.Namespace) + if err != nil { + Log.Error(err, "Failed to get RabbitMQ username for Neutron") + return + } + + if neutronUsername != "" { + currentUsernames[neutronUsername] = true + Log.Info("Found current RabbitMQ username for Neutron", "username", neutronUsername) + } + } else if serviceName == "ironic" { + // Get Ironic Neutron Agent username + ironicUsername, err := deployment.GetIronicRabbitMqUserFromSecret(ctx, helper, instance.Namespace) + if err != nil { + Log.Error(err, "Failed to get RabbitMQ username for Ironic Neutron Agent") + return + } + + if ironicUsername != "" { + currentUsernames[ironicUsername] = true + Log.Info("Found current RabbitMQ username for Ironic Neutron Agent", "username", ironicUsername) + } + } + + // If we found no RabbitMQ users for this service, skip finalizer management + if len(currentUsernames) == 0 { + Log.Info("No RabbitMQ users found in secrets for service, skipping finalizer management", + "service", serviceName) + return + } + + // List all RabbitMqUsers in the namespace + rabbitmqUserList := &rabbitmqv1.RabbitMQUserList{} + err = r.List(ctx, rabbitmqUserList, client.InNamespace(instance.Namespace)) + if err != nil { + Log.Error(err, "Failed to list RabbitMQUsers", "service", serviceName) + return + } + + // Process each RabbitMqUser + for i := range rabbitmqUserList.Items { + rabbitmqUser := &rabbitmqUserList.Items[i] + + // Check if this user is currently in use by this nodeset for this service + // Match by either CR name or status username + isCurrentlyInUse := currentUsernames[rabbitmqUser.Name] || currentUsernames[rabbitmqUser.Status.Username] + + hasFinalizer := slices.Contains(rabbitmqUser.Finalizers, finalizerName) + hasCleanupBlockedFinalizer := slices.Contains(rabbitmqUser.Finalizers, rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer) + + needsUpdate := false + newFinalizers := make([]string, 0, len(rabbitmqUser.Finalizers)) + + if isCurrentlyInUse && !hasFinalizer { + // Add finalizer to this RabbitMqUser (this is the current user) + // We add immediately when ANY node starts using this user to protect it + Log.Info("Adding finalizer to RabbitMqUser", + "service", serviceName, + "user", rabbitmqUser.Name, + "finalizer", finalizerName) + newFinalizers = append(newFinalizers, rabbitmqUser.Finalizers...) + newFinalizers = append(newFinalizers, finalizerName) + needsUpdate = true + } else if !isCurrentlyInUse && hasFinalizer && shouldRemoveOldFinalizers { + // Remove finalizer from this RabbitMqUser (no longer in use) + // Safe to remove because we only reach here when: + // 1. The deployment was created AFTER the secret was modified + // 2. No deployments are currently running + // 3. This user is not in the current secret configuration for this service + // 4. All nodes for this service have been updated + // 5. All nodesets using the same cluster for this service have been updated + Log.Info("Removing finalizer from RabbitMqUser (no longer in use)", + "service", serviceName, + "user", rabbitmqUser.Name, + "finalizer", finalizerName) + for _, f := range rabbitmqUser.Finalizers { + if f != finalizerName { + newFinalizers = append(newFinalizers, f) + } + } + needsUpdate = true + } else if !isCurrentlyInUse && hasFinalizer && !shouldRemoveOldFinalizers { + // This user is no longer in use but we can't remove the finalizer yet + // because not all nodes/nodesets have been updated + Log.Info("RabbitMqUser has finalizer but is no longer in use - waiting for all nodes to update before removing", + "service", serviceName, + "user", rabbitmqUser.Name, + "finalizer", finalizerName, + "updatedNodes", len(tracking.UpdatedNodes), + "totalNodes", len(allNodeNames)) + newFinalizers = append(newFinalizers, rabbitmqUser.Finalizers...) + } else { + // No change to nodeset finalizers + newFinalizers = append(newFinalizers, rabbitmqUser.Finalizers...) + } + + // Remove the temporary cleanup-blocked finalizer if present + // This finalizer was added by infra-operator as a temporary safeguard during + // credential rotation, but should be removed now that proper finalizer management is in place + if hasCleanupBlockedFinalizer { + Log.Info("Removing temporary cleanup-blocked finalizer from RabbitMqUser", + "user", rabbitmqUser.Name, + "finalizer", rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer) + filteredFinalizers := make([]string, 0, len(newFinalizers)) + for _, f := range newFinalizers { + if f != rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer { + filteredFinalizers = append(filteredFinalizers, f) + } + } + newFinalizers = filteredFinalizers + needsUpdate = true + } + + // Apply all finalizer changes in a single update + if needsUpdate { + rabbitmqUser.Finalizers = newFinalizers + err = r.Update(ctx, rabbitmqUser) + if err != nil { + Log.Error(err, "Failed to update finalizers on RabbitMQUser", + "service", serviceName, + "user", rabbitmqUser.Name) + // Don't fail reconciliation, just log the error + } + } + } +} diff --git a/internal/controller/dataplane/manage_service_finalizers_test.go b/internal/controller/dataplane/manage_service_finalizers_test.go new file mode 100644 index 000000000..430d2f686 --- /dev/null +++ b/internal/controller/dataplane/manage_service_finalizers_test.go @@ -0,0 +1,253 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dataplane + +import ( + "strings" + "testing" +) + +func TestComputeFinalizerHash(t *testing.T) { + tests := []struct { + name string + nodesetName string + expectedHash string // First 8 chars of SHA256 + }{ + { + name: "Short nodeset name", + nodesetName: "compute", + expectedHash: "b04a12f6", // First 8 chars of SHA256("compute") + }, + { + name: "Medium length name", + nodesetName: "edpm-compute-nodes", + expectedHash: "32188c96", // First 8 chars of SHA256("edpm-compute-nodes") + }, + { + name: "Long nodeset name", + nodesetName: "my-very-long-openstack-dataplane-nodeset-name-for-production", + expectedHash: "ae0c35f8", // First 8 chars of SHA256(...) + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := computeFinalizerHash(tt.nodesetName) + + // Check length is exactly 8 characters + if len(result) != 8 { + t.Errorf("computeFinalizerHash() length = %v, want 8", len(result)) + } + + // Check it's a valid hex string + for _, c := range result { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + t.Errorf("computeFinalizerHash() contains non-hex char %c, result = %v", c, result) + } + } + + // Check it matches expected hash + if result != tt.expectedHash { + t.Errorf("computeFinalizerHash() = %v, want %v", result, tt.expectedHash) + } + + t.Logf("Hash for %q: %s", tt.nodesetName, result) + }) + } +} + +func TestComputeFinalizerHashDeterministic(t *testing.T) { + // The same input should always produce the same hash + nodesetName := "my-very-long-openstack-dataplane-nodeset-name" + + hash1 := computeFinalizerHash(nodesetName) + hash2 := computeFinalizerHash(nodesetName) + + if hash1 != hash2 { + t.Errorf("computeFinalizerHash() not deterministic: first=%v, second=%v", hash1, hash2) + } +} + +func TestComputeFinalizerHashUniqueness(t *testing.T) { + // Different inputs should produce different hashes + tests := []struct { + nodeset1, nodeset2 string + }{ + {"compute-zone1", "compute-zone2"}, + {"edpm-compute-nodes-a", "edpm-compute-nodes-b"}, + {"my-very-long-name-production-zone1", "my-very-long-name-staging-zone1"}, + } + + for _, tt := range tests { + hash1 := computeFinalizerHash(tt.nodeset1) + hash2 := computeFinalizerHash(tt.nodeset2) + + if hash1 == hash2 { + t.Errorf("computeFinalizerHash() collision detected:\n %s => %s\n %s => %s", + tt.nodeset1, hash1, + tt.nodeset2, hash2) + } + } +} + +func TestBuildFinalizerName(t *testing.T) { + tests := []struct { + name string + finalizerHash string + serviceName string + expectedResult string + expectedMaxLen int + }{ + { + name: "Nova service", + finalizerHash: "a3f2b5c8", + serviceName: "nova", + expectedResult: "nodeset.os/a3f2b5c8-nova", + expectedMaxLen: 63, + }, + { + name: "Neutron service", + finalizerHash: "7e9d1234", + serviceName: "neutron", + expectedResult: "nodeset.os/7e9d1234-neutron", + expectedMaxLen: 63, + }, + { + name: "Ironic service", + finalizerHash: "5a6b7c8d", + serviceName: "ironic", + expectedResult: "nodeset.os/5a6b7c8d-ironic", + expectedMaxLen: 63, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := buildFinalizerName(tt.finalizerHash, tt.serviceName) + + // Check exact result + if result != tt.expectedResult { + t.Errorf("buildFinalizerName() = %v, want %v", result, tt.expectedResult) + } + + // Check length is within Kubernetes limit + if len(result) > tt.expectedMaxLen { + t.Errorf("buildFinalizerName() length = %v, exceeds max %v", len(result), tt.expectedMaxLen) + } + + // Check format + if !strings.HasPrefix(result, finalizerPrefix) { + t.Errorf("buildFinalizerName() missing prefix %q, got %v", finalizerPrefix, result) + } + + if !strings.HasSuffix(result, "-"+tt.serviceName) { + t.Errorf("buildFinalizerName() missing service suffix %q, got %v", tt.serviceName, result) + } + + t.Logf("Result: %s (length: %d)", result, len(result)) + }) + } +} + +func TestBuildFinalizerNameDeterministic(t *testing.T) { + // The same inputs should always produce the same output + finalizerHash := "a3f2b5c8" + serviceName := "nova" + + result1 := buildFinalizerName(finalizerHash, serviceName) + result2 := buildFinalizerName(finalizerHash, serviceName) + + if result1 != result2 { + t.Errorf("buildFinalizerName() not deterministic: first=%v, second=%v", result1, result2) + } +} + +func TestBuildFinalizerNameUniqueness(t *testing.T) { + // Different inputs should produce different outputs + tests := []struct { + hash1, service1 string + hash2, service2 string + }{ + { + hash1: "a3f2b5c8", service1: "nova", + hash2: "7e9d1234", service2: "nova", + }, + { + hash1: "a3f2b5c8", service1: "nova", + hash2: "a3f2b5c8", service2: "neutron", + }, + } + + for _, tt := range tests { + result1 := buildFinalizerName(tt.hash1, tt.service1) + result2 := buildFinalizerName(tt.hash2, tt.service2) + + if result1 == result2 { + t.Errorf("buildFinalizerName() not unique:\n input1: %s-%s = %s\n input2: %s-%s = %s", + tt.hash1, tt.service1, result1, + tt.hash2, tt.service2, result2) + } + } +} + +func TestEndToEndFinalizerGeneration(t *testing.T) { + // Test complete workflow: nodeset name -> hash -> finalizer + nodesets := []string{ + "compute", + "edpm-compute-nodes", + "production-compute-cluster-zone1", + "my-extremely-long-dataplane-nodeset-name-for-testing", + "edpm-very-long-name-production-zone1", + "edpm-very-long-name-staging-zone1", // Should have different hash than above + } + + services := []string{"nova", "neutron", "ironic"} + + t.Log("End-to-end finalizer generation examples:") + + // Track all generated finalizers to ensure uniqueness + finalizers := make(map[string]string) + + for _, nodeset := range nodesets { + hash := computeFinalizerHash(nodeset) + + for _, service := range services { + finalizer := buildFinalizerName(hash, service) + + // Check for collisions + if existingNodeset, exists := finalizers[finalizer]; exists { + t.Errorf("Collision detected! Finalizer %q used by both %q and %q", + finalizer, existingNodeset, nodeset+"-"+service) + } + finalizers[finalizer] = nodeset + "-" + service + + // Verify length limit + if len(finalizer) > 63 { + t.Errorf("INVALID: finalizer exceeds 63 chars: %s", finalizer) + } + + // Verify format + if !strings.HasPrefix(finalizer, finalizerPrefix) { + t.Errorf("INVALID: finalizer missing prefix: %s", finalizer) + } + + t.Logf(" %s + %s => %s (hash=%s, len=%d)", + nodeset, service, finalizer, hash, len(finalizer)) + } + } + + t.Logf("Generated %d unique finalizers across %d nodesets and %d services", + len(finalizers), len(nodesets), len(services)) +} diff --git a/internal/controller/dataplane/openstackdataplanenodeset_controller.go b/internal/controller/dataplane/openstackdataplanenodeset_controller.go index b0f77ebac..cbb8e7b9f 100644 --- a/internal/controller/dataplane/openstackdataplanenodeset_controller.go +++ b/internal/controller/dataplane/openstackdataplanenodeset_controller.go @@ -24,6 +24,7 @@ import ( "time" "github.com/go-playground/validator/v10" + "github.com/iancoleman/strcase" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -35,11 +36,14 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -48,6 +52,7 @@ import ( "github.com/go-logr/logr" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/rolebinding" @@ -108,6 +113,8 @@ func (r *OpenStackDataPlaneNodeSetReconciler) GetLogger(ctx context.Context) log // +kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; // +kubebuilder:rbac:groups=core.openstack.org,resources=openstackversions,verbs=get;list;watch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=rabbitmqusers,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=rabbitmqusers/finalizers,verbs=update;patch // RBAC for the ServiceAccount for the internal image registry // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch @@ -231,6 +238,35 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req instance.Status.ContainerImages = make(map[string]string) } + // Compute and store finalizer hash if not already set + // This hash is used to create unique, collision-free finalizer names for RabbitMQ users + // The hash is deterministic (same nodeset name -> same hash) and allows easy lookup + if instance.Status.FinalizerHash == "" { + instance.Status.FinalizerHash = computeFinalizerHash(instance.Name) + Log.Info("Computed finalizer hash for nodeset", + "nodeset", instance.Name, + "hash", instance.Status.FinalizerHash) + } + + // Add finalizer to the nodeset if it's not being deleted + if instance.DeletionTimestamp.IsZero() { + if controllerutil.AddFinalizer(instance, helper.GetFinalizer()) { + // Finalizer was added, update the instance to persist it + if err := r.Update(ctx, instance); err != nil { + Log.Error(err, "Failed to add finalizer to NodeSet") + return ctrl.Result{}, err + } + Log.Info("Added finalizer to NodeSet", "finalizer", helper.GetFinalizer()) + // Don't return early - continue with normal reconcile logic + // The deferred patch will persist status changes + } + } + + // Handle nodeset deletion - clean up RabbitMQ user finalizers before allowing deletion + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + instance.Status.Conditions.MarkFalse(dataplanev1.SetupReadyCondition, condition.RequestedReason, condition.SeverityInfo, condition.ReadyInitMessage) // Detect config changes and set Status ConfigHash @@ -404,7 +440,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req } isDeploymentReady, isDeploymentRunning, isDeploymentFailed, failedDeployment, err := checkDeployment( - ctx, helper, instance) + ctx, helper, instance, r) if !isDeploymentFailed && err != nil { instance.Status.Conditions.MarkFalse( condition.DeploymentReadyCondition, @@ -489,7 +525,8 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req } func checkDeployment(ctx context.Context, helper *helper.Helper, - instance *dataplanev1.OpenStackDataPlaneNodeSet) ( + instance *dataplanev1.OpenStackDataPlaneNodeSet, + r *OpenStackDataPlaneNodeSetReconciler) ( isNodeSetDeploymentReady bool, isNodeSetDeploymentRunning bool, isNodeSetDeploymentFailed bool, failedDeploymentName string, err error) { @@ -516,9 +553,13 @@ func checkDeployment(ctx context.Context, helper *helper.Helper, } } + // If there are no active deployments, we should not manage RabbitMQ finalizers + // because we can't reliably verify the current state + hasActiveDeployments := len(relevantDeployments) > 0 + // Sort relevant deployments from oldest to newest, then take the last one var latestRelevantDeployment *dataplanev1.OpenStackDataPlaneDeployment - if len(relevantDeployments) > 0 { + if hasActiveDeployments { slices.SortFunc(relevantDeployments, func(a, b *dataplanev1.OpenStackDataPlaneDeployment) int { aReady := a.Status.Conditions.Get(condition.DeploymentReadyCondition) bReady := b.Status.Conditions.Get(condition.DeploymentReadyCondition) @@ -532,6 +573,83 @@ func checkDeployment(ctx context.Context, helper *helper.Helper, latestRelevantDeployment = relevantDeployments[len(relevantDeployments)-1] } + // Get Nova, Neutron, and Ironic secrets with their modification times + // Do this before the loop to avoid variable shadowing of 'deployment' package + novaSecretsLastModified, errNovaSecrets := deployment.GetNovaCellSecretsLastModified(ctx, helper, instance.Namespace) + neutronSecretsLastModified, errNeutronSecrets := deployment.GetNeutronSecretsLastModified(ctx, helper, instance.Namespace) + ironicSecretsLastModified, errIronicSecrets := deployment.GetIronicSecretsLastModified(ctx, helper, instance.Namespace) + + // Get owner references for the tracking ConfigMap + ownerRefs := []v1.OwnerReference{ + { + APIVersion: instance.APIVersion, + Kind: instance.Kind, + Name: instance.Name, + UID: instance.UID, + Controller: ptr.To(true), + }, + } + + // Check Nova credential changes and update tracking ConfigMap + currentNovaHash, errNovaHash := deployment.ComputeNovaCellSecretsHash(ctx, helper, instance.Namespace) + if errNovaHash != nil { + helper.GetLogger().Error(errNovaHash, "Failed to compute Nova cell secrets hash") + } else if currentNovaHash != "" { + novaTracking, err := deployment.GetServiceTracking(ctx, helper, instance.Name, instance.Namespace, "nova") + if err != nil { + helper.GetLogger().Error(err, "Failed to get Nova service tracking") + } else if novaTracking.SecretHash != currentNovaHash { + // Nova secret hash changed - reset Nova node update tracking + helper.GetLogger().Info("Nova secret hash changed, resetting Nova node update tracking", + "oldHash", novaTracking.SecretHash, + "newHash", currentNovaHash) + err := deployment.ResetServiceNodeTracking(ctx, helper, instance.Name, instance.Namespace, "nova", currentNovaHash, ownerRefs) + if err != nil { + helper.GetLogger().Error(err, "Failed to reset Nova service tracking") + } + } + } + + // Check Neutron credential changes and update tracking ConfigMap + currentNeutronHash, errNeutronHash := deployment.ComputeNeutronSecretsHash(ctx, helper, instance.Namespace) + if errNeutronHash != nil { + helper.GetLogger().Error(errNeutronHash, "Failed to compute Neutron secrets hash") + } else if currentNeutronHash != "" { + neutronTracking, err := deployment.GetServiceTracking(ctx, helper, instance.Name, instance.Namespace, "neutron") + if err != nil { + helper.GetLogger().Error(err, "Failed to get Neutron service tracking") + } else if neutronTracking.SecretHash != currentNeutronHash { + // Neutron secret hash changed - reset Neutron node update tracking + helper.GetLogger().Info("Neutron secret hash changed, resetting Neutron node update tracking", + "oldHash", neutronTracking.SecretHash, + "newHash", currentNeutronHash) + err := deployment.ResetServiceNodeTracking(ctx, helper, instance.Name, instance.Namespace, "neutron", currentNeutronHash, ownerRefs) + if err != nil { + helper.GetLogger().Error(err, "Failed to reset Neutron service tracking") + } + } + } + + // Check Ironic credential changes and update tracking ConfigMap + currentIronicHash, errIronicHash := deployment.ComputeIronicSecretsHash(ctx, helper, instance.Namespace) + if errIronicHash != nil { + helper.GetLogger().Error(errIronicHash, "Failed to compute Ironic secrets hash") + } else if currentIronicHash != "" { + ironicTracking, err := deployment.GetServiceTracking(ctx, helper, instance.Name, instance.Namespace, "ironic") + if err != nil { + helper.GetLogger().Error(err, "Failed to get Ironic service tracking") + } else if ironicTracking.SecretHash != currentIronicHash { + // Ironic secret hash changed - reset Ironic node update tracking + helper.GetLogger().Info("Ironic secret hash changed, resetting Ironic node update tracking", + "oldHash", ironicTracking.SecretHash, + "newHash", currentIronicHash) + err := deployment.ResetServiceNodeTracking(ctx, helper, instance.Name, instance.Namespace, "ironic", currentIronicHash, ownerRefs) + if err != nil { + helper.GetLogger().Error(err, "Failed to reset Ironic service tracking") + } + } + } + for _, deployment := range relevantDeployments { // Always add to DeploymentStatuses (for visibility) deploymentConditions := deployment.Status.NodeSetConditions[instance.Name] @@ -609,6 +727,124 @@ func checkDeployment(ctx context.Context, helper *helper.Helper, services = instance.Spec.Services } + // Check each service's edpmServiceType to detect which RabbitMQ services were deployed + // Track Nova, Neutron, and Ironic separately for independent credential rotation + novaServiceDeployed := false + neutronServiceDeployed := false + ironicServiceDeployed := false + var novaServiceName string + var neutronServiceName string + var ironicServiceName string + + for _, serviceName := range services { + service := &dataplanev1.OpenStackDataPlaneService{} + name := types.NamespacedName{ + Namespace: instance.Namespace, + Name: serviceName, + } + err := helper.GetClient().Get(ctx, name, service) + if err != nil { + helper.GetLogger().Error(err, "Unable to retrieve OpenStackDataPlaneService", "service", serviceName) + continue + } + + // Check if this is a RabbitMQ-using service based on edpmServiceType + serviceType := service.Spec.EDPMServiceType + if serviceType == "" { + // If not set, defaults to the service name + serviceType = serviceName + } + + // Check Nova service + if serviceType == "nova" { + novaServiceName = serviceName + serviceCondition := condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(serviceName))) + if deploymentConditions.IsTrue(serviceCondition) { + novaServiceDeployed = true + } + } + + // Check Neutron services (DHCP and SRIOV) + // neutron-ovn and neutron-metadata don't use RabbitMQ (only OVN connections) + if serviceType == "neutron-dhcp" || serviceType == "neutron-sriov" { + neutronServiceName = serviceName + serviceCondition := condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(serviceName))) + if deploymentConditions.IsTrue(serviceCondition) { + neutronServiceDeployed = true + } + } + + // Check Ironic Neutron Agent service + if serviceType == "ironic-neutron-agent" { + ironicServiceName = serviceName + serviceCondition := condition.Type(fmt.Sprintf("Service%sDeploymentReady", strcase.ToCamel(serviceName))) + if deploymentConditions.IsTrue(serviceCondition) { + ironicServiceDeployed = true + } + } + } + + // If Nova service was deployed successfully, track which nodes were updated for Nova + if novaServiceDeployed && isCurrentDeploymentReady { + // Update the status to track which nodes were covered by this deployment + // This persists the state so it survives pod restarts and deployment deletions + r.updateNodeCoverageForService(ctx, helper, instance, deployment, "nova", novaSecretsLastModified) + } + + // If Neutron service was deployed successfully, track which nodes were updated for Neutron + if neutronServiceDeployed && isCurrentDeploymentReady { + // Update the status to track which nodes were covered by this deployment + r.updateNodeCoverageForService(ctx, helper, instance, deployment, "neutron", neutronSecretsLastModified) + } + + // If Ironic service was deployed successfully, track which nodes were updated for Ironic + if ironicServiceDeployed && isCurrentDeploymentReady { + r.updateNodeCoverageForService(ctx, helper, instance, deployment, "ironic", ironicSecretsLastModified) + } + + // Manage finalizers separately for Nova, Neutron, and Ironic + // This allows independent credential rotation for each service + // IMPORTANT: Check that no other deployment is running to avoid premature cleanup + // IMPORTANT: Don't manage finalizers if the deployment is being deleted + // IMPORTANT: Require at least one active deployment to ensure reliable state + isDeploymentBeingDeleted := !deployment.DeletionTimestamp.IsZero() + + // Manage Nova user finalizers if Nova service was deployed + if novaServiceDeployed && isLatestDeployment && !isNodeSetDeploymentRunning && !isDeploymentBeingDeleted && hasActiveDeployments { + helper.GetLogger().Info("Checking if Nova RabbitMQ finalizers should be managed", + "deployment", deployment.Name, + "service", novaServiceName) + if errNovaSecrets != nil { + helper.GetLogger().Error(errNovaSecrets, "Failed to get Nova secrets last modified times") + } else { + r.manageServiceFinalizers(ctx, helper, instance, "nova", novaServiceName, novaSecretsLastModified) + } + } + + // Manage Neutron user finalizers if Neutron service was deployed + if neutronServiceDeployed && isLatestDeployment && !isNodeSetDeploymentRunning && !isDeploymentBeingDeleted && hasActiveDeployments { + helper.GetLogger().Info("Checking if Neutron RabbitMQ finalizers should be managed", + "deployment", deployment.Name, + "service", neutronServiceName) + if errNeutronSecrets != nil { + helper.GetLogger().Error(errNeutronSecrets, "Failed to get Neutron secrets last modified times") + } else { + r.manageServiceFinalizers(ctx, helper, instance, "neutron", neutronServiceName, neutronSecretsLastModified) + } + } + + // Manage Ironic user finalizers if Ironic service was deployed + if ironicServiceDeployed && isLatestDeployment && !isNodeSetDeploymentRunning && !isDeploymentBeingDeleted && hasActiveDeployments { + helper.GetLogger().Info("Checking if Ironic RabbitMQ finalizers should be managed", + "deployment", deployment.Name, + "service", ironicServiceName) + if errIronicSecrets != nil { + helper.GetLogger().Error(errIronicSecrets, "Failed to get Ironic secrets last modified times") + } else { + r.manageServiceFinalizers(ctx, helper, instance, "ironic", ironicServiceName, ironicSecretsLastModified) + } + } + // For each service, check if EDPMServiceType is "update" or "update-services", and // if so, copy Deployment.Status.DeployedVersion to // NodeSet.Status.DeployedVersion @@ -639,6 +875,437 @@ func checkDeployment(ctx context.Context, helper *helper.Helper, return isNodeSetDeploymentReady, isNodeSetDeploymentRunning, isNodeSetDeploymentFailed, failedDeploymentName, err } +// updateNodeCoverageForService updates the service tracking ConfigMap to track which nodes +// have been covered by a successful deployment after the secret change for a specific service +func (r *OpenStackDataPlaneNodeSetReconciler) updateNodeCoverageForService( + ctx context.Context, + helper *helper.Helper, + instance *dataplanev1.OpenStackDataPlaneNodeSet, + deploymentObj *dataplanev1.OpenStackDataPlaneDeployment, + serviceName string, + secretsLastModified map[string]time.Time, +) { + Log := r.GetLogger(ctx) + + // Check if this deployment completed after the secret change + // We check the deployment's ready condition timestamp, not creation timestamp, + // because deployments can be created before they run. + deploymentConditions := deploymentObj.Status.NodeSetConditions[instance.Name] + readyCondition := deploymentConditions.Get(dataplanev1.NodeSetDeploymentReadyCondition) + if readyCondition == nil { + return + } + + deploymentCompletedTime := readyCondition.LastTransitionTime.Time + for _, secretModTime := range secretsLastModified { + if deploymentCompletedTime.Before(secretModTime) { + // This deployment completed before the secret change, don't track it + Log.Info("Deployment completed before secret change, skipping node coverage tracking", + "service", serviceName, + "deployment", deploymentObj.Name, + "deploymentCompletedTime", deploymentCompletedTime, + "secretModTime", secretModTime) + return + } + } + + // Get all nodes in the nodeset + allNodeNames := r.getAllNodeNamesFromNodeset(instance) + if len(allNodeNames) == 0 { + return + } + + // Determine which nodes were covered by this deployment + coveredNodes := r.getNodesCoveredByDeployment(deploymentObj, allNodeNames) + if len(coveredNodes) == 0 { + return + } + + // Get owner references for the ConfigMap + ownerRefs := []v1.OwnerReference{ + { + APIVersion: instance.APIVersion, + Kind: instance.Kind, + Name: instance.Name, + UID: instance.UID, + Controller: ptr.To(true), + }, + } + + // Add newly covered nodes to the ConfigMap with retry on conflict + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return deployment.AddUpdatedNodes(ctx, helper, instance.Name, instance.Namespace, serviceName, coveredNodes, ownerRefs) + }) + if err != nil { + Log.Error(err, "Failed to add updated nodes to service tracking", + "service", serviceName, + "deployment", deploymentObj.Name) + return + } + + // Get updated tracking to log + tracking, err := deployment.GetServiceTracking(ctx, helper, instance.Name, instance.Namespace, serviceName) + if err != nil { + Log.Error(err, "Failed to get updated service tracking", "service", serviceName) + } else { + Log.Info("Updated node coverage tracking in ConfigMap", + "service", serviceName, + "deployment", deploymentObj.Name, + "coveredByThisDeployment", len(coveredNodes), + "totalCovered", len(tracking.UpdatedNodes), + "totalNodes", len(allNodeNames)) + } +} + +// allNodesetsUsingClusterUpdated checks if all nodesets using the same RabbitMQ cluster +// for a specific service have been fully updated after the secret was modified. +// This ensures we don't remove the finalizer from the old RabbitMQUser until ALL nodesets +// that might be using it for that service have been updated to the new configuration. +func (r *OpenStackDataPlaneNodeSetReconciler) allNodesetsUsingClusterUpdated( + ctx context.Context, + helper *helper.Helper, + currentNodeset *dataplanev1.OpenStackDataPlaneNodeSet, + serviceName string, + secretsLastModified map[string]time.Time, +) (bool, error) { + Log := r.GetLogger(ctx) + + // Collect all RabbitMQ clusters used by this nodeset for the specified service + clustersUsedByCurrentNodeset := make(map[string]bool) + + if serviceName == "nova" { + // Get Nova clusters + cellNames, err := deployment.GetNovaComputeConfigCellNames(ctx, helper, currentNodeset.Namespace) + if err != nil { + Log.Info("Failed to get Nova cell names", "error", err) + } else { + for _, cellName := range cellNames { + cluster, err := deployment.GetRabbitMQClusterForCell(ctx, helper, currentNodeset.Namespace, cellName) + if err != nil { + Log.Info("Failed to get RabbitMQ cluster for Nova cell", "cell", cellName, "error", err) + continue + } + if cluster != "" { + clustersUsedByCurrentNodeset[cluster] = true + } + } + } + } else if serviceName == "neutron" { + // Get Neutron cluster + neutronCluster, err := deployment.GetRabbitMQClusterForNeutron(ctx, helper, currentNodeset.Namespace) + if err != nil { + Log.Info("Failed to get RabbitMQ cluster for Neutron", "error", err) + } else if neutronCluster != "" { + clustersUsedByCurrentNodeset[neutronCluster] = true + } + } else if serviceName == "ironic" { + // Get Ironic cluster + ironicCluster, err := deployment.GetRabbitMQClusterForIronic(ctx, helper, currentNodeset.Namespace) + if err != nil { + Log.Info("Failed to get RabbitMQ cluster for Ironic", "error", err) + } else if ironicCluster != "" { + clustersUsedByCurrentNodeset[ironicCluster] = true + } + } + + if len(clustersUsedByCurrentNodeset) == 0 { + // No clusters identified - this likely means the RabbitMQ cluster info + // is not available yet. We should NOT proceed with finalizer removal + // until we can properly verify all nodes are updated. + Log.Info("No RabbitMQ clusters identified, cannot verify node coverage - skipping finalizer management", + "service", serviceName, + "nodeset", currentNodeset.Name) + return false, nil + } + + // Get all nodesets in the namespace + nodesetList := &dataplanev1.OpenStackDataPlaneNodeSetList{} + err := r.List(ctx, nodesetList, client.InNamespace(currentNodeset.Namespace)) + if err != nil { + return false, fmt.Errorf("failed to list nodesets: %w", err) + } + + // Check each nodeset that might be using the same cluster for this service + for _, nodeset := range nodesetList.Items { + // Check if this nodeset uses any of the same clusters for this service + usesSharedCluster := false + + if serviceName == "nova" { + // Check Nova cells + nodesetCellNames, err := deployment.GetNovaComputeConfigCellNames(ctx, helper, nodeset.Namespace) + if err == nil { + for _, cellName := range nodesetCellNames { + cluster, err := deployment.GetRabbitMQClusterForCell(ctx, helper, nodeset.Namespace, cellName) + if err != nil { + continue + } + if clustersUsedByCurrentNodeset[cluster] { + usesSharedCluster = true + break + } + } + } + } else if serviceName == "neutron" { + // Check Neutron cluster + nodesetNeutronCluster, err := deployment.GetRabbitMQClusterForNeutron(ctx, helper, nodeset.Namespace) + if err == nil && nodesetNeutronCluster != "" && clustersUsedByCurrentNodeset[nodesetNeutronCluster] { + usesSharedCluster = true + } + } else if serviceName == "ironic" { + // Check Ironic cluster + nodesetIronicCluster, err := deployment.GetRabbitMQClusterForIronic(ctx, helper, nodeset.Namespace) + if err == nil && nodesetIronicCluster != "" && clustersUsedByCurrentNodeset[nodesetIronicCluster] { + usesSharedCluster = true + } + } + + if !usesSharedCluster { + // This nodeset doesn't use the same cluster for this service, skip it + continue + } + + // This nodeset uses the same cluster for this service, check if it's been updated + // IMPORTANT: If this is the current nodeset, use the in-memory version + // to avoid stale data (ConfigMap may have been updated but not refreshed yet) + nodesetToCheck := &nodeset + if nodeset.Name == currentNodeset.Name && nodeset.Namespace == currentNodeset.Namespace { + nodesetToCheck = currentNodeset + } + + nodesetUpdated, err := r.isNodesetFullyUpdated(ctx, helper, nodesetToCheck, serviceName, secretsLastModified) + if err != nil { + Log.Error(err, "Failed to check if nodeset is fully updated", "service", serviceName, "nodeset", nodeset.Name) + return false, err + } + + if !nodesetUpdated { + Log.Info("Nodeset using same RabbitMQ cluster has not been fully updated yet for service", + "service", serviceName, + "nodeset", nodeset.Name, + "currentNodeset", currentNodeset.Name) + return false, nil + } + } + + // All nodesets using the same cluster for this service have been updated + return true, nil +} + +// isNodesetFullyUpdated checks if all nodes in a nodeset have been successfully deployed +// after the secrets were last modified for a specific service. This uses the ConfigMap +// tracking as the source of truth, which survives pod restarts and deployment deletions. +func (r *OpenStackDataPlaneNodeSetReconciler) isNodesetFullyUpdated( + ctx context.Context, + helper *helper.Helper, + nodeset *dataplanev1.OpenStackDataPlaneNodeSet, + serviceName string, + _ map[string]time.Time, +) (bool, error) { + Log := r.GetLogger(ctx) + + // Get all node names/hostnames from the nodeset + allNodeNames := r.getAllNodeNamesFromNodeset(nodeset) + if len(allNodeNames) == 0 { + // No nodes defined in nodeset, consider it updated + return true, nil + } + + // Get the tracking data from the ConfigMap + tracking, err := deployment.GetServiceTracking(ctx, helper, nodeset.Name, nodeset.Namespace, serviceName) + if err != nil { + return false, fmt.Errorf("failed to get service tracking for %s: %w", serviceName, err) + } + + // Use the persisted ConfigMap as the source of truth for node coverage + // This survives pod restarts and deployment deletions + coveredNodes := make(map[string]bool) + for _, nodeName := range tracking.UpdatedNodes { + coveredNodes[nodeName] = true + } + + // Check if all nodes are covered + uncoveredNodes := make([]string, 0) + for _, nodeName := range allNodeNames { + if !coveredNodes[nodeName] { + uncoveredNodes = append(uncoveredNodes, nodeName) + } + } + + if len(uncoveredNodes) > 0 { + Log.Info("Not all nodes have been updated yet for service", + "service", serviceName, + "nodeset", nodeset.Name, + "allNodeNames", allNodeNames, + "coveredNodesList", tracking.UpdatedNodes, + "coveredNodes", len(coveredNodes), + "totalNodes", len(allNodeNames), + "uncoveredCount", len(uncoveredNodes), + "uncoveredNodes", uncoveredNodes) + return false, nil + } + + Log.Info("All nodes in nodeset have been successfully updated for service", + "service", serviceName, + "nodeset", nodeset.Name, + "allNodeNames", allNodeNames, + "coveredNodesList", tracking.UpdatedNodes, + "totalNodes", len(allNodeNames), + "coveredNodes", len(coveredNodes)) + return true, nil +} + +// getAllNodeNamesFromNodeset extracts all node names from a nodeset +// Only returns the node names (map keys), not hostnames or ansible_host values, +// since those are just aliases for the same node +func (r *OpenStackDataPlaneNodeSetReconciler) getAllNodeNamesFromNodeset( + nodeset *dataplanev1.OpenStackDataPlaneNodeSet, +) []string { + nodeNames := make([]string, 0, len(nodeset.Spec.Nodes)) + + for nodeName := range nodeset.Spec.Nodes { + // Only add the node name from the map key + // Don't add hostname or ansible_host as they're just aliases for the same node + nodeNames = append(nodeNames, nodeName) + } + + return nodeNames +} + +// getNodesCoveredByDeployment determines which nodes were covered by a deployment +// based on its AnsibleLimit setting +func (r *OpenStackDataPlaneNodeSetReconciler) getNodesCoveredByDeployment( + deployment *dataplanev1.OpenStackDataPlaneDeployment, + allNodes []string, +) []string { + ansibleLimit := strings.TrimSpace(deployment.Spec.AnsibleLimit) + + // If no AnsibleLimit or it's "*", all nodes are covered + if ansibleLimit == "" || ansibleLimit == "*" { + return allNodes + } + + // Parse AnsibleLimit to find covered nodes + // AnsibleLimit can be a comma-separated list of node names, patterns, or groups + limitParts := strings.Split(ansibleLimit, ",") + + coveredNodes := make([]string, 0) + for _, node := range allNodes { + if r.nodeMatchesAnsibleLimit(node, limitParts) { + coveredNodes = append(coveredNodes, node) + } + } + + return coveredNodes +} + +// nodeMatchesAnsibleLimit checks if a node matches any pattern in the AnsibleLimit +func (r *OpenStackDataPlaneNodeSetReconciler) nodeMatchesAnsibleLimit( + nodeName string, + limitParts []string, +) bool { + for _, part := range limitParts { + part = strings.TrimSpace(part) + + // Exact match + if part == nodeName { + return true + } + + // Simple wildcard matching (* at the end) + // e.g., "compute-*" matches "compute-0", "compute-1", etc. + if strings.HasSuffix(part, "*") { + prefix := strings.TrimSuffix(part, "*") + if strings.HasPrefix(nodeName, prefix) { + return true + } + } + + // Simple wildcard matching (* at the beginning) + // e.g., "*-0" matches "compute-0", "controller-0", etc. + if strings.HasPrefix(part, "*") { + suffix := strings.TrimPrefix(part, "*") + if strings.HasSuffix(nodeName, suffix) { + return true + } + } + + // TODO: More complex Ansible patterns could be supported here + // For now, we handle the most common cases (exact match and simple wildcards) + } + + return false +} + +// reconcileDelete handles the deletion of a NodeSet by cleaning up RabbitMQ user finalizers +func (r *OpenStackDataPlaneNodeSetReconciler) reconcileDelete( + ctx context.Context, + instance *dataplanev1.OpenStackDataPlaneNodeSet, + helper *helper.Helper, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info("Reconciling NodeSet deletion") + + // Get all RabbitMQ users in the namespace to clean up our finalizers + rabbitmqUsers := &unstructured.UnstructuredList{} + rabbitmqUsers.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "rabbitmq.openstack.org", + Version: "v1beta1", + Kind: "RabbitMQUserList", + }) + + listOpts := &client.ListOptions{ + Namespace: instance.Namespace, + } + + err := r.List(ctx, rabbitmqUsers, listOpts) + if err != nil { + Log.Error(err, "Failed to list RabbitMQ users during deletion") + return ctrl.Result{}, err + } + + // Remove our finalizers from all RabbitMQ users + // Our finalizers follow the pattern: nodeset.os/{finalizerHash}-{service} + finalizerPrefix := fmt.Sprintf("nodeset.os/%s-", instance.Status.FinalizerHash) + + for _, userObj := range rabbitmqUsers.Items { + user := userObj.DeepCopy() + originalFinalizers := user.GetFinalizers() + updatedFinalizers := []string{} + + // Keep only finalizers that don't belong to this nodeset + // Also remove the temporary cleanup-blocked finalizer if present + for _, f := range originalFinalizers { + if strings.HasPrefix(f, finalizerPrefix) { + Log.Info("Removing finalizer from RabbitMQ user during nodeset deletion", + "user", user.GetName(), + "finalizer", f) + } else if f == rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer { + Log.Info("Removing temporary cleanup-blocked finalizer from RabbitMQ user during nodeset deletion", + "user", user.GetName(), + "finalizer", f) + } else { + updatedFinalizers = append(updatedFinalizers, f) + } + } + + // Update the user if we removed any finalizers + if len(updatedFinalizers) != len(originalFinalizers) { + user.SetFinalizers(updatedFinalizers) + if err := r.Update(ctx, user); err != nil { + Log.Error(err, "Failed to remove finalizer from RabbitMQ user", + "user", user.GetName()) + return ctrl.Result{}, err + } + } + } + + // Remove the nodeset's own finalizer to allow deletion + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info("Finalizer removed from NodeSet, allowing deletion", "nodeset", instance.Name) + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager( ctx context.Context, mgr ctrl.Manager, diff --git a/internal/dataplane/rabbitmq.go b/internal/dataplane/rabbitmq.go new file mode 100644 index 000000000..5a0192294 --- /dev/null +++ b/internal/dataplane/rabbitmq.go @@ -0,0 +1,702 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployment + +import ( + "context" + "fmt" + "maps" + "net/url" + "regexp" + "slices" + "strings" + "time" + + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetNovaCellRabbitMqUserFromSecret extracts the RabbitMQ username from a nova-cellX-compute-config secret +// Returns the username extracted from rabbitmq_user_name field (preferred) or transport_url (fallback) +// As of nova-operator PR #1066, the RabbitMQUser CR name is propagated directly in the +// rabbitmq_user_name and notification_rabbitmq_user_name fields for easier tracking. +func GetNovaCellRabbitMqUserFromSecret( + ctx context.Context, + h *helper.Helper, + namespace string, + cellName string, +) (string, error) { + // List all secrets in the namespace + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } + + // Pattern to match nova-cellX-compute-config secrets + // Supports both split secrets (nova-cell1-compute-config-0) and non-split (nova-cell1-compute-config) + secretPattern := regexp.MustCompile(`^nova-(` + cellName + `)-compute-config(-\d+)?$`) + + for _, secret := range secretList.Items { + matches := secretPattern.FindStringSubmatch(secret.Name) + if matches == nil { + continue + } + + // Preferred: Use the rabbitmq_user_name field directly if available + // This field is populated by nova-operator PR #1066 with the RabbitMQUser CR name + if rabbitmqUserName, ok := secret.Data["rabbitmq_user_name"]; ok && len(rabbitmqUserName) > 0 { + return string(rabbitmqUserName), nil + } + + // Fallback: Extract transport_url from secret data for backwards compatibility + transportURLBytes, ok := secret.Data["transport_url"] + if !ok { + // Try to extract from config files as fallback (in case it's embedded) + // Check both custom.conf and 01-nova.conf + for _, configKey := range []string{"custom.conf", "01-nova.conf"} { + customConfig, hasCustom := secret.Data[configKey] + if !hasCustom { + continue + } + // Try to extract from custom config + username := extractUsernameFromCustomConfig(string(customConfig)) + if username != "" { + return username, nil + } + } + continue + } + + // Parse transport_url to extract username + username, err := parseUsernameFromTransportURL(string(transportURLBytes)) + if err != nil { + h.GetLogger().Info("Failed to parse transport_url", "secret", secret.Name, "error", err) + continue + } + + if username != "" { + return username, nil + } + } + + return "", fmt.Errorf("no RabbitMQ username found for cell %s", cellName) +} + +// GetNovaCellNotificationRabbitMqUserFromSecret extracts the notification RabbitMQ username +// from a nova-cellX-compute-config secret. This is used for tracking the RabbitMQUser CR +// used for notifications (separate from the messaging/RPC bus). +// Returns the username extracted from notification_rabbitmq_user_name field (preferred) +// or attempts to extract from notification transport_url (fallback). +func GetNovaCellNotificationRabbitMqUserFromSecret( + ctx context.Context, + h *helper.Helper, + namespace string, + cellName string, +) (string, error) { + // List all secrets in the namespace + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } + + // Pattern to match nova-cellX-compute-config secrets + secretPattern := regexp.MustCompile(`^nova-(` + cellName + `)-compute-config(-\d+)?$`) + + for _, secret := range secretList.Items { + matches := secretPattern.FindStringSubmatch(secret.Name) + if matches == nil { + continue + } + + // Preferred: Use the notification_rabbitmq_user_name field directly if available + // This field is populated by nova-operator PR #1066 with the RabbitMQUser CR name + if notificationRabbitmqUserName, ok := secret.Data["notification_rabbitmq_user_name"]; ok && len(notificationRabbitmqUserName) > 0 { + return string(notificationRabbitmqUserName), nil + } + + // If notification_rabbitmq_user_name is not available, this likely means: + // 1. Nova-operator hasn't been updated to PR #1066 yet, or + // 2. Notifications are not configured for this cell + // Return empty string to indicate no notification user (not an error) + return "", nil + } + + return "", fmt.Errorf("no compute-config secret found for cell %s", cellName) +} + +// parseUsernameFromTransportURL extracts the username from a RabbitMQ transport URL +// Format: rabbit://username:password@host:port/vhost +// Also supports: rabbit+tls://username:password@host1:port1,host2:port2/vhost +func parseUsernameFromTransportURL(transportURL string) (string, error) { + // Handle empty URLs + if transportURL == "" { + return "", fmt.Errorf("empty transport URL") + } + + // Parse the URL + // First, replace rabbit:// or rabbit+tls:// with http:// for URL parsing + tempURL := strings.Replace(transportURL, "rabbit://", "http://", 1) + tempURL = strings.Replace(tempURL, "rabbit+tls://", "http://", 1) + + parsedURL, err := url.Parse(tempURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + // Extract username from UserInfo + if parsedURL.User == nil { + return "", fmt.Errorf("no user info in transport URL") + } + + username := parsedURL.User.Username() + if username == "" { + return "", fmt.Errorf("empty username in transport URL") + } + + return username, nil +} + +// extractUsernameFromCustomConfig attempts to extract RabbitMQ username from custom config +// This is a fallback for cases where transport_url is embedded in the config file +func extractUsernameFromCustomConfig(customConfig string) string { + // Look for transport_url in the config + // Format: transport_url = rabbit://username:password@... + transportURLPattern := regexp.MustCompile(`transport_url\s*=\s*rabbit[^:]*://([^:]+):`) + matches := transportURLPattern.FindStringSubmatch(customConfig) + if len(matches) > 1 { + return matches[1] + } + return "" +} + +// extractTransportURLFromConfig attempts to extract the full transport_url from a config file +// This is used to get the RabbitMQ cluster information when transport_url is embedded in the config +func extractTransportURLFromConfig(customConfig string) string { + // Look for transport_url in the config + // Format: transport_url=rabbit://username:password@host:port/vhost?options + // or: transport_url = rabbit://username:password@host:port/vhost?options + transportURLPattern := regexp.MustCompile(`transport_url\s*=\s*(rabbit[^\s\n]+)`) + matches := transportURLPattern.FindStringSubmatch(customConfig) + if len(matches) > 1 { + return matches[1] + } + return "" +} + +// GetNovaComputeConfigCellNames returns a list of cell names from nova-cellX-compute-config secrets +// referenced in the NodeSet's dataSources +func GetNovaComputeConfigCellNames( + ctx context.Context, + h *helper.Helper, + namespace string, +) ([]string, error) { + // List all secrets in the namespace + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list secrets: %w", err) + } + + cellNames := []string{} + // Pattern to match nova-cellX-compute-config secrets + secretPattern := regexp.MustCompile(`^nova-(cell\d+)-compute-config(-\d+)?$`) + + for _, secret := range secretList.Items { + matches := secretPattern.FindStringSubmatch(secret.Name) + if matches == nil { + continue + } + + cellName := matches[1] // Extract cell name (e.g., "cell1") + // Avoid duplicates + found := false + for _, cn := range cellNames { + if cn == cellName { + found = true + break + } + } + if !found { + cellNames = append(cellNames, cellName) + } + } + + return cellNames, nil +} + +// ExtractCellNameFromSecretName extracts the cell name from a secret name +// Example: "nova-cell1-compute-config" -> "cell1" +// Example: "nova-cell1-compute-config-0" -> "cell1" +func ExtractCellNameFromSecretName(secretName string) string { + pattern := regexp.MustCompile(`nova-(cell\d+)-compute-config`) + matches := pattern.FindStringSubmatch(secretName) + if len(matches) > 1 { + return matches[1] + } + return "" +} + +// ComputeNovaCellSecretsHash calculates a hash of all nova-cellX-compute-config secrets +// This is used to detect when the secrets change and reset node update tracking +func ComputeNovaCellSecretsHash( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + secretsLastModified, err := GetNovaCellSecretsLastModified(ctx, h, namespace) + if err != nil { + return "", err + } + + if len(secretsLastModified) == 0 { + return "", nil + } + + // Build a stable string representation of all secrets and their modification times + var secretNames []string + for name := range secretsLastModified { + secretNames = append(secretNames, name) + } + // Sort for stable hash + slices.Sort(secretNames) + + hashData := "" + for _, name := range secretNames { + modTime := secretsLastModified[name] + hashData += fmt.Sprintf("%s:%d;", name, modTime.Unix()) + } + + // Use a simple hash + return fmt.Sprintf("%x", hashData), nil +} + +// GetNovaCellSecretsLastModified returns a map of nova-cellX-compute-config secret names +// to their last modification timestamps +func GetNovaCellSecretsLastModified( + ctx context.Context, + h *helper.Helper, + namespace string, +) (map[string]time.Time, error) { + // List all secrets in the namespace + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list secrets: %w", err) + } + + secretTimes := make(map[string]time.Time) + // Pattern to match nova-cellX-compute-config secrets + secretPattern := regexp.MustCompile(`^nova-(cell\d+)-compute-config(-\d+)?$`) + + for _, secret := range secretList.Items { + matches := secretPattern.FindStringSubmatch(secret.Name) + if matches == nil { + continue + } + + // Use the resource version change time if available, otherwise creation time + modTime := secret.CreationTimestamp.Time + if secret.ManagedFields != nil { + for _, field := range secret.ManagedFields { + if field.Time != nil && field.Time.After(modTime) { + modTime = field.Time.Time + } + } + } + + secretTimes[secret.Name] = modTime + } + + return secretTimes, nil +} + +// GetRabbitMQClusterForCell returns the RabbitMQ cluster name used by a specific nova cell +// by extracting it from the transport_url in the nova-cellX-compute-config secret +func GetRabbitMQClusterForCell( + ctx context.Context, + h *helper.Helper, + namespace string, + cellName string, +) (string, error) { + // List all secrets in the namespace + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } + + // Pattern to match nova-cellX-compute-config secrets + secretPattern := regexp.MustCompile(`^nova-(` + cellName + `)-compute-config(-\d+)?$`) + + for _, secret := range secretList.Items { + matches := secretPattern.FindStringSubmatch(secret.Name) + if matches == nil { + continue + } + + // Extract transport_url from config files (01-nova.conf or custom.conf) + // The transport_url is embedded in the config, not as a separate field + var transportURL string + for _, configKey := range []string{"custom.conf", "01-nova.conf"} { + configData, hasConfig := secret.Data[configKey] + if !hasConfig { + continue + } + // Try to extract transport_url from config + transportURL = extractTransportURLFromConfig(string(configData)) + if transportURL != "" { + break + } + } + + if transportURL == "" { + continue + } + + // Parse transport_url to extract hostname (which typically includes cluster info) + // Format: rabbit://username:password@host:port/vhost + // The host part often contains the cluster name + cluster, err := extractClusterFromTransportURL(transportURL) + if err == nil && cluster != "" { + return cluster, nil + } + } + + return "", fmt.Errorf("no RabbitMQ cluster found for cell %s", cellName) +} + +// extractClusterFromTransportURL extracts the cluster identifier from a RabbitMQ transport URL +// This is a heuristic approach - the cluster name is often part of the hostname +func extractClusterFromTransportURL(transportURL string) (string, error) { + if transportURL == "" { + return "", fmt.Errorf("empty transport URL") + } + + // Replace rabbit:// or rabbit+tls:// with http:// for URL parsing + tempURL := strings.Replace(transportURL, "rabbit://", "http://", 1) + tempURL = strings.Replace(tempURL, "rabbit+tls://", "http://", 1) + + parsedURL, err := url.Parse(tempURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + // Extract the hostname (may contain multiple hosts separated by commas) + host := parsedURL.Host + if host == "" { + return "", fmt.Errorf("no host in transport URL") + } + + // If multiple hosts, take the first one + hosts := strings.Split(host, ",") + if len(hosts) > 0 { + // Parse the first host to get just the hostname (strip port) + firstHost := strings.Split(hosts[0], ":")[0] + + // Extract cluster name - typically the first part of the hostname + // e.g., "rabbitmq-cell1.openstack.svc" -> "rabbitmq-cell1" + // or "rabbitmq.openstack.svc" -> "rabbitmq" + parts := strings.Split(firstHost, ".") + if len(parts) > 0 { + return parts[0], nil + } + return firstHost, nil + } + + return "", fmt.Errorf("could not extract cluster from transport URL") +} + +// ServiceSecretConfig defines the secret and config file patterns for a service +type ServiceSecretConfig struct { + SecretNames []string + ConfigKeys []string +} + +// GetRabbitMqUserFromServiceSecrets extracts the RabbitMQ username from service config secrets +// This is a generic function that works with any service that uses RabbitMQ +func GetRabbitMqUserFromServiceSecrets( + ctx context.Context, + h *helper.Helper, + namespace string, + config ServiceSecretConfig, + serviceName string, +) (string, error) { + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } + + for _, secret := range secretList.Items { + for _, pattern := range config.SecretNames { + if secret.Name == pattern { + for _, configKey := range config.ConfigKeys { + configData, hasConfig := secret.Data[configKey] + if !hasConfig { + continue + } + username := extractUsernameFromCustomConfig(string(configData)) + if username != "" { + return username, nil + } + } + } + } + } + + return "", fmt.Errorf("no RabbitMQ username found in %s secrets", serviceName) +} + +// GetNeutronRabbitMqUserFromSecret extracts the RabbitMQ username from Neutron agent config secrets +func GetNeutronRabbitMqUserFromSecret( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + config := ServiceSecretConfig{ + SecretNames: []string{ + "neutron-dhcp-agent-neutron-config", + "neutron-sriov-agent-neutron-config", + }, + ConfigKeys: []string{"10-neutron-dhcp.conf", "10-neutron-sriov.conf"}, + } + return GetRabbitMqUserFromServiceSecrets(ctx, h, namespace, config, "Neutron") +} + +// GetServiceSecretsLastModified returns a map of service secret names to their last modification timestamps +func GetServiceSecretsLastModified( + ctx context.Context, + h *helper.Helper, + namespace string, + secretNames []string, +) (map[string]time.Time, error) { + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return nil, fmt.Errorf("failed to list secrets: %w", err) + } + + secretTimes := make(map[string]time.Time) + for _, secret := range secretList.Items { + for _, pattern := range secretNames { + if secret.Name == pattern { + modTime := secret.CreationTimestamp.Time + if secret.ManagedFields != nil { + for _, field := range secret.ManagedFields { + if field.Time != nil && field.Time.After(modTime) { + modTime = field.Time.Time + } + } + } + secretTimes[secret.Name] = modTime + } + } + } + + return secretTimes, nil +} + +// GetNeutronSecretsLastModified returns a map of Neutron agent config secret names to their last modification timestamps +func GetNeutronSecretsLastModified( + ctx context.Context, + h *helper.Helper, + namespace string, +) (map[string]time.Time, error) { + secretNames := []string{ + "neutron-dhcp-agent-neutron-config", + "neutron-sriov-agent-neutron-config", + } + return GetServiceSecretsLastModified(ctx, h, namespace, secretNames) +} + +// GetRabbitMQClusterForService returns the RabbitMQ cluster name by extracting from transport_url +func GetRabbitMQClusterForService( + ctx context.Context, + h *helper.Helper, + namespace string, + config ServiceSecretConfig, + serviceName string, +) (string, error) { + secretList := &corev1.SecretList{} + err := h.GetClient().List(ctx, secretList, client.InNamespace(namespace)) + if err != nil { + return "", fmt.Errorf("failed to list secrets: %w", err) + } + + for _, secret := range secretList.Items { + for _, pattern := range config.SecretNames { + if secret.Name == pattern { + for _, configKey := range config.ConfigKeys { + configData, hasConfig := secret.Data[configKey] + if !hasConfig { + continue + } + transportURL := extractTransportURLFromConfig(string(configData)) + if transportURL == "" { + continue + } + cluster, err := extractClusterFromTransportURL(transportURL) + if err == nil && cluster != "" { + return cluster, nil + } + } + } + } + } + + return "", fmt.Errorf("no RabbitMQ cluster found in %s secrets", serviceName) +} + +// GetRabbitMQClusterForNeutron returns the RabbitMQ cluster name used by Neutron agents +func GetRabbitMQClusterForNeutron( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + config := ServiceSecretConfig{ + SecretNames: []string{ + "neutron-dhcp-agent-neutron-config", + "neutron-sriov-agent-neutron-config", + }, + ConfigKeys: []string{"10-neutron-dhcp.conf", "10-neutron-sriov.conf"}, + } + return GetRabbitMQClusterForService(ctx, h, namespace, config, "Neutron") +} + +// ComputeServiceSecretsHash calculates a hash of service secrets to detect changes +func ComputeServiceSecretsHash( + ctx context.Context, + h *helper.Helper, + namespace string, + secretNames []string, +) (string, error) { + secretsLastModified, err := GetServiceSecretsLastModified(ctx, h, namespace, secretNames) + if err != nil { + return "", err + } + + if len(secretsLastModified) == 0 { + return "", nil + } + + var names []string + for name := range secretsLastModified { + names = append(names, name) + } + slices.Sort(names) + + hashData := "" + for _, name := range names { + modTime := secretsLastModified[name] + hashData += fmt.Sprintf("%s:%d;", name, modTime.Unix()) + } + + return fmt.Sprintf("%x", hashData), nil +} + +// ComputeNeutronSecretsHash calculates a hash of Neutron agent config secrets +func ComputeNeutronSecretsHash( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + secretNames := []string{ + "neutron-dhcp-agent-neutron-config", + "neutron-sriov-agent-neutron-config", + } + return ComputeServiceSecretsHash(ctx, h, namespace, secretNames) +} + +// GetIronicRabbitMqUserFromSecret extracts the RabbitMQ username from Ironic Neutron Agent config secrets +func GetIronicRabbitMqUserFromSecret( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + config := ServiceSecretConfig{ + SecretNames: []string{"ironic-neutron-agent-config-data"}, + ConfigKeys: []string{"01-ironic_neutron_agent.conf"}, + } + return GetRabbitMqUserFromServiceSecrets(ctx, h, namespace, config, "Ironic Neutron Agent") +} + +// GetIronicSecretsLastModified returns a map of Ironic Neutron Agent config secret names to their last modification timestamps +func GetIronicSecretsLastModified( + ctx context.Context, + h *helper.Helper, + namespace string, +) (map[string]time.Time, error) { + secretNames := []string{"ironic-neutron-agent-config-data"} + return GetServiceSecretsLastModified(ctx, h, namespace, secretNames) +} + +// GetRabbitMQClusterForIronic returns the RabbitMQ cluster name used by Ironic Neutron Agent +func GetRabbitMQClusterForIronic( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + config := ServiceSecretConfig{ + SecretNames: []string{"ironic-neutron-agent-config-data"}, + ConfigKeys: []string{"01-ironic_neutron_agent.conf"}, + } + return GetRabbitMQClusterForService(ctx, h, namespace, config, "Ironic Neutron Agent") +} + +// ComputeIronicSecretsHash calculates a hash of Ironic Neutron Agent config secrets +func ComputeIronicSecretsHash( + ctx context.Context, + h *helper.Helper, + namespace string, +) (string, error) { + secretNames := []string{"ironic-neutron-agent-config-data"} + return ComputeServiceSecretsHash(ctx, h, namespace, secretNames) +} + +// GetRabbitMQSecretsLastModified returns a combined map of all RabbitMQ-related secret names +// (Nova, Neutron, and Ironic) to their last modification timestamps +func GetRabbitMQSecretsLastModified( + ctx context.Context, + h *helper.Helper, + namespace string, +) (map[string]time.Time, error) { + allSecrets := make(map[string]time.Time) + + // Get and merge Nova cell secrets + novaSecretsLastModified, err := GetNovaCellSecretsLastModified(ctx, h, namespace) + if err != nil { + return nil, err + } + maps.Copy(allSecrets, novaSecretsLastModified) + + // Get and merge Neutron agent secrets + neutronSecretsLastModified, err := GetNeutronSecretsLastModified(ctx, h, namespace) + if err != nil { + return nil, err + } + maps.Copy(allSecrets, neutronSecretsLastModified) + + // Get and merge Ironic Neutron Agent secrets + ironicSecretsLastModified, err := GetIronicSecretsLastModified(ctx, h, namespace) + if err != nil { + return nil, err + } + maps.Copy(allSecrets, ironicSecretsLastModified) + + return allSecrets, nil +} diff --git a/internal/dataplane/rabbitmq_test.go b/internal/dataplane/rabbitmq_test.go new file mode 100644 index 000000000..1ad7f5433 --- /dev/null +++ b/internal/dataplane/rabbitmq_test.go @@ -0,0 +1,464 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployment + +import ( + "context" + "testing" + + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// setupTestHelper creates a fake client and helper for testing +func setupTestHelper(objects ...client.Object) *helper.Helper { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + fakeClient := fakeclient.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + Build() + + // Create a fake kubernetes clientset + fakeKubeClient := fake.NewSimpleClientset() + + // Create a mock object for the helper (minimal valid object) + mockObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-config", + Namespace: "test-namespace", + }, + } + + h, _ := helper.NewHelper( + mockObj, + fakeClient, + fakeKubeClient, + scheme, + ctrl.Log.WithName("test"), + ) + return h +} + +func TestGetNovaCellRabbitMqUserFromSecret(t *testing.T) { + ctx := context.Background() + namespace := "openstack" + cellName := "cell1" + + tests := []struct { + name string + secrets []runtime.Object + cellName string + expectedUser string + expectedError bool + errorContains string + }{ + { + name: "New format: rabbitmq_user_name field present (preferred path)", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("nova-cell1-user"), + "01-nova.conf": []byte("[DEFAULT]\ntransport_url = rabbit://old-user:pass@host:5672/\n"), + }, + }, + }, + expectedUser: "nova-cell1-user", + expectedError: false, + }, + { + name: "Old format: transport_url in config file (fallback path)", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "01-nova.conf": []byte("[DEFAULT]\ntransport_url = rabbit://fallback-user:password@rabbitmq.openstack.svc:5672/\n"), + }, + }, + }, + expectedUser: "fallback-user", + expectedError: false, + }, + { + name: "Both fields present: should prefer rabbitmq_user_name", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("preferred-user"), + "01-nova.conf": []byte("[DEFAULT]\ntransport_url = rabbit://fallback-user:pass@host:5672/\n"), + }, + }, + }, + expectedUser: "preferred-user", + expectedError: false, + }, + { + name: "Secret with versioned suffix (nova-cell1-compute-config-1)", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config-1", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("versioned-user"), + }, + }, + }, + expectedUser: "versioned-user", + expectedError: false, + }, + { + name: "Empty rabbitmq_user_name falls back to transport_url", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte(""), + "01-nova.conf": []byte("[DEFAULT]\ntransport_url = rabbit://empty-fallback:pass@host:5672/\n"), + }, + }, + }, + expectedUser: "empty-fallback", + expectedError: false, + }, + { + name: "No matching secret found", + cellName: "cell99", + secrets: []runtime.Object{}, + expectedUser: "", + expectedError: true, + errorContains: "no RabbitMQ username found for cell cell99", + }, + { + name: "Secret exists but no transport_url or rabbitmq_user_name", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "other-data": []byte("some-value"), + }, + }, + }, + expectedUser: "", + expectedError: true, + errorContains: "no RabbitMQ username found for cell cell1", + }, + { + name: "Transport URL with TLS", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "01-nova.conf": []byte("[DEFAULT]\ntransport_url = rabbit+tls://tls-user:password@host1:5671,host2:5671/vhost\n"), + }, + }, + }, + expectedUser: "tls-user", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Convert runtime.Object slice to client.Object slice + clientObjects := make([]client.Object, len(tt.secrets)) + for i, obj := range tt.secrets { + clientObjects[i] = obj.(client.Object) + } + + // Create helper with fake client + h := setupTestHelper(clientObjects...) + + // Call the function + username, err := GetNovaCellRabbitMqUserFromSecret(ctx, h, namespace, tt.cellName) + + // Verify results + if tt.expectedError { + assert.Error(t, err) + if tt.errorContains != "" { + assert.Contains(t, err.Error(), tt.errorContains) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedUser, username) + } + }) + } +} + +func TestGetNovaCellNotificationRabbitMqUserFromSecret(t *testing.T) { + ctx := context.Background() + namespace := "openstack" + cellName := "cell1" + + tests := []struct { + name string + secrets []runtime.Object + cellName string + expectedUser string + expectedError bool + errorContains string + }{ + { + name: "notification_rabbitmq_user_name field present", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "notification_rabbitmq_user_name": []byte("nova-cell1-notification-user"), + "rabbitmq_user_name": []byte("nova-cell1-user"), + }, + }, + }, + expectedUser: "nova-cell1-notification-user", + expectedError: false, + }, + { + name: "notification_rabbitmq_user_name not present (notifications not configured)", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("nova-cell1-user"), + }, + }, + }, + expectedUser: "", + expectedError: false, + }, + { + name: "Empty notification_rabbitmq_user_name", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "notification_rabbitmq_user_name": []byte(""), + "rabbitmq_user_name": []byte("nova-cell1-user"), + }, + }, + }, + expectedUser: "", + expectedError: false, + }, + { + name: "Secret with versioned suffix", + cellName: cellName, + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config-2", + Namespace: namespace, + }, + Data: map[string][]byte{ + "notification_rabbitmq_user_name": []byte("versioned-notification-user"), + }, + }, + }, + expectedUser: "versioned-notification-user", + expectedError: false, + }, + { + name: "No matching secret found", + cellName: "cell99", + secrets: []runtime.Object{}, + expectedUser: "", + expectedError: true, + errorContains: "no compute-config secret found for cell cell99", + }, + { + name: "Multiple cells - should match correct cell", + cellName: "cell2", + secrets: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "notification_rabbitmq_user_name": []byte("cell1-notification-user"), + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell2-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "notification_rabbitmq_user_name": []byte("cell2-notification-user"), + }, + }, + }, + expectedUser: "cell2-notification-user", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Convert runtime.Object slice to client.Object slice + clientObjects := make([]client.Object, len(tt.secrets)) + for i, obj := range tt.secrets { + clientObjects[i] = obj.(client.Object) + } + + // Create helper with fake client + h := setupTestHelper(clientObjects...) + + // Call the function + username, err := GetNovaCellNotificationRabbitMqUserFromSecret(ctx, h, namespace, tt.cellName) + + // Verify results + if tt.expectedError { + assert.Error(t, err) + if tt.errorContains != "" { + assert.Contains(t, err.Error(), tt.errorContains) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedUser, username) + } + }) + } +} + +func TestGetNovaCellRabbitMqUserFromSecret_EdgeCases(t *testing.T) { + ctx := context.Background() + namespace := "openstack" + + t.Run("Multiple secrets for same cell - should use first match", func(t *testing.T) { + secrets := []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("first-user"), + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config-1", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("second-user"), + }, + }, + } + + // Convert runtime.Object slice to client.Object slice + clientObjects := make([]client.Object, len(secrets)) + for i, obj := range secrets { + clientObjects[i] = obj.(client.Object) + } + + h := setupTestHelper(clientObjects...) + + username, err := GetNovaCellRabbitMqUserFromSecret(ctx, h, namespace, "cell1") + assert.NoError(t, err) + // Should get one of the users (order may vary, but shouldn't error) + assert.NotEmpty(t, username) + assert.Contains(t, []string{"first-user", "second-user"}, username) + }) + + t.Run("Complex transport URL parsing", func(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "01-nova.conf": []byte(`[DEFAULT] +debug = true +transport_url = rabbit://complex-user:p@ssw0rd@host1:5672,host2:5672,host3:5672/cell1 +log_dir = /var/log/nova +`), + }, + } + + h := setupTestHelper(secret) + + username, err := GetNovaCellRabbitMqUserFromSecret(ctx, h, namespace, "cell1") + assert.NoError(t, err) + assert.Equal(t, "complex-user", username) + }) + + t.Run("Cell name with special characters", func(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-cell-prod-az1-compute-config", + Namespace: namespace, + }, + Data: map[string][]byte{ + "rabbitmq_user_name": []byte("cell-prod-az1-user"), + }, + } + + h := setupTestHelper(secret) + + username, err := GetNovaCellRabbitMqUserFromSecret(ctx, h, namespace, "cell-prod-az1") + assert.NoError(t, err) + assert.Equal(t, "cell-prod-az1-user", username) + }) +} diff --git a/internal/dataplane/service_tracking.go b/internal/dataplane/service_tracking.go new file mode 100644 index 000000000..6ec1e9f07 --- /dev/null +++ b/internal/dataplane/service_tracking.go @@ -0,0 +1,259 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployment + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + // ServiceTrackingConfigMapSuffix is appended to nodeset name to create tracking ConfigMap + ServiceTrackingConfigMapSuffix = "-service-tracking" +) + +// ServiceTrackingData stores tracking information for a service's credential rotation +type ServiceTrackingData struct { + // SecretHash is the hash of the service's secrets to detect changes + SecretHash string `json:"secretHash"` + // UpdatedNodes is the list of nodes that have been updated after the secret change + UpdatedNodes []string `json:"updatedNodes"` +} + +// GetServiceTrackingConfigMapName returns the name of the tracking ConfigMap for a nodeset +func GetServiceTrackingConfigMapName(nodesetName string) string { + return nodesetName + ServiceTrackingConfigMapSuffix +} + +// EnsureServiceTrackingConfigMap ensures the tracking ConfigMap exists for a nodeset +func EnsureServiceTrackingConfigMap( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + ownerRefs []metav1.OwnerReference, +) (*corev1.ConfigMap, error) { + configMapName := GetServiceTrackingConfigMapName(nodesetName) + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + OwnerReferences: ownerRefs, + }, + Data: make(map[string]string), + } + + // Try to get existing ConfigMap + existing := &corev1.ConfigMap{} + err := h.GetClient().Get(ctx, client.ObjectKey{Name: configMapName, Namespace: namespace}, existing) + if err == nil { + // ConfigMap exists, return it + return existing, nil + } + + if !k8s_errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get service tracking ConfigMap: %w", err) + } + + // ConfigMap doesn't exist, create it + err = h.GetClient().Create(ctx, configMap) + if err != nil { + return nil, fmt.Errorf("failed to create service tracking ConfigMap: %w", err) + } + + return configMap, nil +} + +// GetServiceTracking retrieves tracking data for a specific service from the ConfigMap +func GetServiceTracking( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + serviceName string, +) (*ServiceTrackingData, error) { + configMapName := GetServiceTrackingConfigMapName(nodesetName) + + configMap := &corev1.ConfigMap{} + err := h.GetClient().Get(ctx, client.ObjectKey{Name: configMapName, Namespace: namespace}, configMap) + if err != nil { + if k8s_errors.IsNotFound(err) { + // ConfigMap doesn't exist yet, return empty tracking data + return &ServiceTrackingData{ + SecretHash: "", + UpdatedNodes: []string{}, + }, nil + } + return nil, fmt.Errorf("failed to get service tracking ConfigMap: %w", err) + } + + // Get the data for this service + secretHashKey := fmt.Sprintf("%s.secretHash", serviceName) + updatedNodesKey := fmt.Sprintf("%s.updatedNodes", serviceName) + + tracking := &ServiceTrackingData{ + SecretHash: configMap.Data[secretHashKey], + UpdatedNodes: []string{}, + } + + // Parse the updated nodes JSON array + if nodesJSON, ok := configMap.Data[updatedNodesKey]; ok && nodesJSON != "" { + err := json.Unmarshal([]byte(nodesJSON), &tracking.UpdatedNodes) + if err != nil { + return nil, fmt.Errorf("failed to parse updated nodes JSON: %w", err) + } + } + + return tracking, nil +} + +// UpdateServiceTracking updates tracking data for a specific service in the ConfigMap +func UpdateServiceTracking( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + serviceName string, + tracking *ServiceTrackingData, + ownerRefs []metav1.OwnerReference, +) error { + configMapName := GetServiceTrackingConfigMapName(nodesetName) + + // Marshal updated nodes to JSON + nodesJSON, err := json.Marshal(tracking.UpdatedNodes) + if err != nil { + return fmt.Errorf("failed to marshal updated nodes: %w", err) + } + + secretHashKey := fmt.Sprintf("%s.secretHash", serviceName) + updatedNodesKey := fmt.Sprintf("%s.updatedNodes", serviceName) + + // Use CreateOrUpdate to handle both creation and update + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, h.GetClient(), configMap, func() error { + if configMap.Data == nil { + configMap.Data = make(map[string]string) + } + configMap.Data[secretHashKey] = tracking.SecretHash + configMap.Data[updatedNodesKey] = string(nodesJSON) + + // Set owner references if provided and not already set + if len(ownerRefs) > 0 && len(configMap.OwnerReferences) == 0 { + configMap.OwnerReferences = ownerRefs + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to update service tracking ConfigMap: %w", err) + } + + return nil +} + +// ResetServiceNodeTracking resets the updated nodes list for a service (called when secret hash changes) +func ResetServiceNodeTracking( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + serviceName string, + newSecretHash string, + ownerRefs []metav1.OwnerReference, +) error { + tracking := &ServiceTrackingData{ + SecretHash: newSecretHash, + UpdatedNodes: []string{}, + } + + return UpdateServiceTracking(ctx, h, nodesetName, namespace, serviceName, tracking, ownerRefs) +} + +// AddUpdatedNode adds a node to the updated nodes list for a service +func AddUpdatedNode( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + serviceName string, + nodeName string, + ownerRefs []metav1.OwnerReference, +) error { + tracking, err := GetServiceTracking(ctx, h, nodesetName, namespace, serviceName) + if err != nil { + return err + } + + // Check if node is already in the list + for _, existingNode := range tracking.UpdatedNodes { + if existingNode == nodeName { + // Node already tracked + return nil + } + } + + // Add the node + tracking.UpdatedNodes = append(tracking.UpdatedNodes, nodeName) + + return UpdateServiceTracking(ctx, h, nodesetName, namespace, serviceName, tracking, ownerRefs) +} + +// AddUpdatedNodes adds multiple nodes to the updated nodes list for a service +func AddUpdatedNodes( + ctx context.Context, + h *helper.Helper, + nodesetName string, + namespace string, + serviceName string, + nodeNames []string, + ownerRefs []metav1.OwnerReference, +) error { + tracking, err := GetServiceTracking(ctx, h, nodesetName, namespace, serviceName) + if err != nil { + return err + } + + // Build a map of existing nodes for fast lookup + existingNodes := make(map[string]bool) + for _, node := range tracking.UpdatedNodes { + existingNodes[node] = true + } + + // Add new nodes + for _, nodeName := range nodeNames { + if !existingNodes[nodeName] { + tracking.UpdatedNodes = append(tracking.UpdatedNodes, nodeName) + existingNodes[nodeName] = true + } + } + + return UpdateServiceTracking(ctx, h, nodesetName, namespace, serviceName, tracking, ownerRefs) +} diff --git a/test/functional/dataplane/openstackdataplanenodeset_multicluster_test.go b/test/functional/dataplane/openstackdataplanenodeset_multicluster_test.go new file mode 100644 index 000000000..231d13113 --- /dev/null +++ b/test/functional/dataplane/openstackdataplanenodeset_multicluster_test.go @@ -0,0 +1,48 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package functional + +// NOTE: Comprehensive multi-cluster RabbitMQ finalizer tests have been implemented +// in openstackdataplanenodeset_rabbitmq_finalizer_test.go +// +// The tests cover: +// +// CORE FUNCTIONALITY: +// 1. Incremental Node Deployments +// - 3-node deployment with incremental updates using ansibleLimit +// - Finalizer added only after ALL nodes are updated +// +// 2. Multi-NodeSet Shared User Management +// - Independent finalizers per nodeset on shared RabbitMQ user +// - User protected until all nodesets remove their finalizers +// - Deletion of one nodeset doesn't affect others +// +// 3. RabbitMQ User Credential Rotation +// - Switch from old user to new user during rolling update +// - Finalizer moves from old to new user after all nodes updated +// - Safe credential rotation without service interruption +// +// ADVANCED SCENARIOS: +// 4. Multi-Service RabbitMQ Cluster Management +// - Multiple services (Nova, Neutron, Ironic) using different clusters +// - Service-specific finalizers (nodeset.os/{hash}-{service}) +// - Independent lifecycle management per service +// +// 5. Deployment Timing and Secret Changes +// - Deployment completion time vs creation time validation +// - Secret changes during active deployment (resets tracking) +// - Multiple deployment scenarios and timing edge cases +// +// See openstackdataplanenodeset_rabbitmq_finalizer_test.go for full implementation. diff --git a/test/functional/dataplane/openstackdataplanenodeset_rabbitmq_finalizer_test.go b/test/functional/dataplane/openstackdataplanenodeset_rabbitmq_finalizer_test.go new file mode 100644 index 000000000..9fc51196c --- /dev/null +++ b/test/functional/dataplane/openstackdataplanenodeset_rabbitmq_finalizer_test.go @@ -0,0 +1,1593 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package functional + +import ( + "fmt" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + + infrav1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/api/dataplane/v1beta1" + dataplaneutil "github.com/openstack-k8s-operators/openstack-operator/internal/dataplane/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("Dataplane NodeSet RabbitMQ Finalizer Management", func() { + + // Helper function to create Nova cell config secret + CreateNovaCellConfigSecret := func(cellName, username, cluster string) *corev1.Secret { + transportURL := fmt.Sprintf("rabbit://%s:password@%s.openstack.svc:5672/", username, cluster) + config := fmt.Sprintf("[DEFAULT]\ntransport_url = %s\n", transportURL) + name := types.NamespacedName{ + Namespace: namespace, + Name: fmt.Sprintf("nova-%s-compute-config", cellName), + } + return th.CreateSecret(name, map[string][]byte{ + "01-nova.conf": []byte(config), + }) + } + + // Helper function to update Nova cell config secret + UpdateNovaCellConfigSecret := func(cellName, username, cluster string) { + transportURL := fmt.Sprintf("rabbit://%s:password@%s.openstack.svc:5672/", username, cluster) + config := fmt.Sprintf("[DEFAULT]\ntransport_url = %s\n", transportURL) + name := types.NamespacedName{ + Namespace: namespace, + Name: fmt.Sprintf("nova-%s-compute-config", cellName), + } + secret := &corev1.Secret{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, name, secret) + g.Expect(err).NotTo(HaveOccurred()) + }, timeout, interval).Should(Succeed()) + + secret.Data["01-nova.conf"] = []byte(config) + Expect(k8sClient.Update(ctx, secret)).Should(Succeed()) + } + + // Helper function to create Neutron agent config secret + CreateNeutronAgentConfigSecret := func(agentType, username, cluster string) *corev1.Secret { + transportURL := fmt.Sprintf("rabbit://%s:password@%s.openstack.svc:5672/", username, cluster) + config := fmt.Sprintf("[DEFAULT]\ntransport_url = %s\n", transportURL) + name := types.NamespacedName{ + Namespace: namespace, + Name: fmt.Sprintf("neutron-%s-agent-neutron-config", agentType), + } + configKey := fmt.Sprintf("10-neutron-%s.conf", agentType) + return th.CreateSecret(name, map[string][]byte{ + configKey: []byte(config), + }) + } + + // Helper function to create Ironic Neutron Agent config secret + CreateIronicNeutronAgentConfigSecret := func(username, cluster string) *corev1.Secret { + transportURL := fmt.Sprintf("rabbit://%s:password@%s.openstack.svc:5672/", username, cluster) + config := fmt.Sprintf("[DEFAULT]\ntransport_url = %s\n", transportURL) + name := types.NamespacedName{ + Namespace: namespace, + Name: "ironic-neutron-agent-config-data", + } + return th.CreateSecret(name, map[string][]byte{ + "01-ironic_neutron_agent.conf": []byte(config), + }) + } + + // Helper function to create RabbitMQUser + CreateRabbitMQUser := func(username string) *rabbitmqv1.RabbitMQUser { + user := &rabbitmqv1.RabbitMQUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: username, + Namespace: namespace, + }, + Spec: rabbitmqv1.RabbitMQUserSpec{ + Username: username, + }, + } + Expect(k8sClient.Create(ctx, user)).Should(Succeed()) + // Set status username to match spec + user.Status.Username = username + Expect(k8sClient.Status().Update(ctx, user)).Should(Succeed()) + return user + } + + // Helper function to get RabbitMQUser + GetRabbitMQUser := func(username string) *rabbitmqv1.RabbitMQUser { + user := &rabbitmqv1.RabbitMQUser{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: username, + Namespace: namespace, + }, user) + g.Expect(err).NotTo(HaveOccurred()) + }, timeout, interval).Should(Succeed()) + return user + } + + // Helper to check if finalizer exists on RabbitMQ user + HasFinalizer := func(username, finalizer string) bool { + user := GetRabbitMQUser(username) + for _, f := range user.Finalizers { + if f == finalizer { + return true + } + } + return false + } + + // Helper to simulate deployment completion + SimulateDeploymentComplete := func(deploymentName types.NamespacedName, nodesetName string, _ []string) { + // First, complete the AnsibleEE jobs for each service + deployment := GetDataplaneDeployment(deploymentName) + nodeset := GetDataplaneNodeSet(types.NamespacedName{ + Namespace: deploymentName.Namespace, + Name: nodesetName, + }) + + // Get list of services from deployment or nodeset + var services []string + if len(deployment.Spec.ServicesOverride) != 0 { + services = deployment.Spec.ServicesOverride + } else { + services = nodeset.Spec.Services + } + + // Complete AnsibleEE job for each service + for _, serviceName := range services { + service := &dataplanev1.OpenStackDataPlaneService{} + g := NewWithT(GinkgoT()) + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: deploymentName.Namespace, + Name: serviceName, + }, service)).Should(Succeed()) + + aeeName, _ := dataplaneutil.GetAnsibleExecutionNameAndLabels( + service, deployment.GetName(), nodeset.GetName()) + Eventually(func(g Gomega) { + ansibleeeName := types.NamespacedName{ + Name: aeeName, + Namespace: deploymentName.Namespace, + } + ansibleEE := GetAnsibleee(ansibleeeName) + ansibleEE.Status.Succeeded = 1 + g.Expect(k8sClient.Status().Update(ctx, ansibleEE)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + } + + // Then set the deployment status to ready + Eventually(func(g Gomega) { + deployment := &dataplanev1.OpenStackDataPlaneDeployment{} + g.Expect(k8sClient.Get(ctx, deploymentName, deployment)).Should(Succeed()) + + // Get the nodeset to access its ConfigHash + nodeset := &dataplanev1.OpenStackDataPlaneNodeSet{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: deploymentName.Namespace, + Name: nodesetName, + }, nodeset)).Should(Succeed()) + + // Set the deployment to ready with matching hashes + deployment.Status.NodeSetConditions = map[string]condition.Conditions{ + nodesetName: { + { + Type: dataplanev1.NodeSetDeploymentReadyCondition, + Status: corev1.ConditionTrue, + Reason: condition.ReadyReason, + Message: condition.DeploymentReadyMessage, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }, + }, + } + // Set hashes to match nodeset + if deployment.Status.NodeSetHashes == nil { + deployment.Status.NodeSetHashes = make(map[string]string) + } + deployment.Status.NodeSetHashes[nodesetName] = nodeset.Status.ConfigHash + + g.Expect(k8sClient.Status().Update(ctx, deployment)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + } + + Context("Incremental Node Deployments", func() { + var dataplaneNodeSetName types.NamespacedName + var novaServiceName types.NamespacedName + var dataplaneSSHSecretName types.NamespacedName + var caBundleSecretName types.NamespacedName + var dataplaneNetConfigName types.NamespacedName + var dnsMasqName types.NamespacedName + var novaUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + dataplaneNodeSetName = types.NamespacedName{ + Name: "compute-rolling", + Namespace: namespace, + } + novaServiceName = types.NamespacedName{ + Name: "nova", + Namespace: namespace, + } + dataplaneSSHSecretName = types.NamespacedName{ + Namespace: namespace, + Name: "dataplane-ansible-ssh-private-key-secret", + } + caBundleSecretName = types.NamespacedName{ + Namespace: namespace, + Name: "combined-ca-bundle", + } + dataplaneNetConfigName = types.NamespacedName{ + Namespace: namespace, + Name: "dataplane-netconfig-rolling", + } + dnsMasqName = types.NamespacedName{ + Name: "dnsmasq-rolling", + Namespace: namespace, + } + + // Create Nova config secret with user1 + CreateNovaCellConfigSecret("cell1", "nova-user1", "rabbitmq-cell1") + novaUser = CreateRabbitMQUser("nova-user1") + + // Create Nova service + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{ + "edpmServiceType": "nova", + }) + DeferCleanup(th.DeleteService, novaServiceName) + + // Create network infrastructure + DeferCleanup(th.DeleteInstance, CreateNetConfig(dataplaneNetConfigName, DefaultNetConfigSpec())) + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create SSH and CA secrets + CreateSSHSecret(dataplaneSSHSecretName) + CreateCABundleSecret(caBundleSecretName) + + // Create nova-migration-ssh-key secret required by nova service + CreateSSHSecret(types.NamespacedName{ + Namespace: namespace, + Name: "nova-migration-ssh-key", + }) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, novaUser) + }) + + It("Should add finalizer only after ALL nodes are updated in rolling deployment", func() { + // Create nodeset with 3 nodes + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + "compute-1": { + HostName: "compute-1", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.101", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + "compute-2": { + HostName: "compute-2", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.102", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + DeferCleanup(th.DeleteInstance, CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec)) + + // Simulate IP sets + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-1"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-2"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Step 1: Deploy first node (ansibleLimit: compute-0) + deployment1Name := types.NamespacedName{ + Name: "deploy-compute-0", + Namespace: namespace, + } + deploymentSpec1 := map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-0", + } + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployment1Name, deploymentSpec1)) + SimulateDeploymentComplete(deployment1Name, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // After first deployment, finalizer should NOT be added (only 1/3 nodes) + Consistently(func(g Gomega) { + g.Expect(HasFinalizer("nova-user1", "nodeset.os/")).Should(BeFalse()) + }, time.Second*5, interval).Should(Succeed()) + + // Step 2: Deploy second node (ansibleLimit: compute-1) + deployment2Name := types.NamespacedName{ + Name: "deploy-compute-1", + Namespace: namespace, + } + deploymentSpec2 := map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-1", + } + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployment2Name, deploymentSpec2)) + SimulateDeploymentComplete(deployment2Name, dataplaneNodeSetName.Name, []string{"compute-1"}) + + // After second deployment, finalizer should still NOT be added (only 2/3 nodes) + Consistently(func(g Gomega) { + g.Expect(HasFinalizer("nova-user1", "nodeset.os/")).Should(BeFalse()) + }, time.Second*5, interval).Should(Succeed()) + + // Step 3: Deploy third node (ansibleLimit: compute-2) + deployment3Name := types.NamespacedName{ + Name: "deploy-compute-2", + Namespace: namespace, + } + deploymentSpec3 := map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-2", + } + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployment3Name, deploymentSpec3)) + SimulateDeploymentComplete(deployment3Name, dataplaneNodeSetName.Name, []string{"compute-2"}) + + // After ALL nodes deployed, finalizer SHOULD be added + // First verify all deployments are actually marked as Ready + Eventually(func(g Gomega) { + for _, deployName := range []string{"deploy-compute-0", "deploy-compute-1", "deploy-compute-2"} { + deploy := &dataplanev1.OpenStackDataPlaneDeployment{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: deployName, + }, deploy)).Should(Succeed()) + conds := deploy.Status.NodeSetConditions[dataplaneNodeSetName.Name] + g.Expect(conds).ShouldNot(BeNil(), fmt.Sprintf("%s should have conditions", deployName)) + readyCond := conds.Get(dataplanev1.NodeSetDeploymentReadyCondition) + g.Expect(readyCond).ShouldNot(BeNil(), fmt.Sprintf("%s should have ready condition", deployName)) + g.Expect(readyCond.Status).Should(Equal(corev1.ConditionTrue), fmt.Sprintf("%s should be ready", deployName)) + } + }, timeout, interval).Should(Succeed()) + + // Now check for finalizer + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + hasFinalizerPrefix := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasFinalizerPrefix = true + break + } + } + g.Expect(hasFinalizerPrefix).Should(BeTrue(), "Finalizer should be added after all nodes updated") + }, timeout, interval).Should(Succeed()) + }) + }) + + Context("Multi-NodeSet Shared User Management", func() { + var nodeset1Name, nodeset2Name types.NamespacedName + var novaServiceName types.NamespacedName + var novaUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + nodeset1Name = types.NamespacedName{ + Name: "compute-zone1", + Namespace: namespace, + } + nodeset2Name = types.NamespacedName{ + Name: "compute-zone2", + Namespace: namespace, + } + novaServiceName = types.NamespacedName{ + Name: "nova", + Namespace: namespace, + } + + // Both nodesets use the same Nova cluster and user + CreateNovaCellConfigSecret("cell1", "nova-cell1", "rabbitmq-cell1") + novaUser = CreateRabbitMQUser("nova-cell1") + + // Create Nova service + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{ + "edpmServiceType": "nova", + }) + DeferCleanup(th.DeleteService, novaServiceName) + + // Create nova-migration-ssh-key secret required by nova service + CreateSSHSecret(types.NamespacedName{ + Namespace: namespace, + Name: "nova-migration-ssh-key", + }) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, novaUser) + }) + + It("Should add independent finalizers from each nodeset to shared user", func() { + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-multi"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-multi"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create first nodeset + nodeSet1Spec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "zone1-compute-0": { + HostName: "zone1-compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.110", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(nodeset1Name, nodeSet1Spec) + // Note: nodeset1 is explicitly deleted as part of the test, so no DeferCleanup needed + + // Setup for zone1 + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "zone1-compute-0"}) + SimulateDNSDataComplete(nodeset1Name) + + // Deploy zone1 + deploy1Name := types.NamespacedName{Name: "deploy-zone1", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy1Name, map[string]interface{}{ + "nodeSets": []string{nodeset1Name.Name}, + })) + SimulateDeploymentComplete(deploy1Name, nodeset1Name.Name, []string{"zone1-compute-0"}) + + // Verify zone1 finalizer added + var zone1Finalizer string + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-cell1") + found := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + zone1Finalizer = f + found = true + break + } + } + g.Expect(found).Should(BeTrue(), "Zone1 should add its finalizer") + }, timeout, interval).Should(Succeed()) + + // Create second nodeset + nodeSet2Spec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "zone2-compute-0": { + HostName: "zone2-compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.120", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(nodeset2Name, nodeSet2Spec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(nodeset2Name)) }) + + // Setup for zone2 + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "zone2-compute-0"}) + SimulateDNSDataComplete(nodeset2Name) + + // Deploy zone2 + deploy2Name := types.NamespacedName{Name: "deploy-zone2", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy2Name, map[string]interface{}{ + "nodeSets": []string{nodeset2Name.Name}, + })) + SimulateDeploymentComplete(deploy2Name, nodeset2Name.Name, []string{"zone2-compute-0"}) + + // Verify BOTH finalizers exist (zone1 and zone2) + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-cell1") + finalizerCount := 0 + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + finalizerCount++ + } + } + g.Expect(finalizerCount).Should(Equal(2), "Both nodesets should have independent finalizers") + }, timeout, interval).Should(Succeed()) + + // Delete zone1 nodeset + Expect(k8sClient.Delete(ctx, GetDataplaneNodeSet(nodeset1Name))).Should(Succeed()) + + // Verify zone1 finalizer removed but zone2 finalizer remains + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-cell1") + hasZone1 := false + hasZone2 := false + for _, f := range user.Finalizers { + if f == zone1Finalizer { + hasZone1 = true + } + if len(f) > 11 && f[:11] == "nodeset.os/" && f != zone1Finalizer { + hasZone2 = true + } + } + g.Expect(hasZone1).Should(BeFalse(), "Zone1 finalizer should be removed after deletion") + g.Expect(hasZone2).Should(BeTrue(), "Zone2 finalizer should remain") + }, timeout, interval).Should(Succeed()) + }) + }) + + Context("RabbitMQ User Credential Rotation", func() { + var dataplaneNodeSetName types.NamespacedName + var novaServiceName types.NamespacedName + var oldUser, newUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + dataplaneNodeSetName = types.NamespacedName{ + Name: "compute-rotation", + Namespace: namespace, + } + novaServiceName = types.NamespacedName{ + Name: "nova", + Namespace: namespace, + } + + // Create initial config with old user + CreateNovaCellConfigSecret("cell1", "nova-old", "rabbitmq-cell1") + oldUser = CreateRabbitMQUser("nova-old") + newUser = CreateRabbitMQUser("nova-new") + + // Create Nova service + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{ + "edpmServiceType": "nova", + }) + DeferCleanup(th.DeleteService, novaServiceName) + + // Create nova-migration-ssh-key secret required by nova service + CreateSSHSecret(types.NamespacedName{ + Namespace: namespace, + Name: "nova-migration-ssh-key", + }) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, oldUser) + _ = k8sClient.Delete(ctx, newUser) + }) + + It("Should switch finalizer from old user to new user after rotation completes", func() { + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-rotation"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-rotation"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset with 2 nodes + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + "compute-1": { + HostName: "compute-1", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.101", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(dataplaneNodeSetName)) }) + + // Setup + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-1"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Initial deployment with old user + deploy1Name := types.NamespacedName{Name: "deploy-initial", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy1Name, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deploy1Name, dataplaneNodeSetName.Name, []string{"compute-0", "compute-1"}) + + // Verify old user has finalizer + var oldUserFinalizer string + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-old") + found := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + oldUserFinalizer = f + found = true + break + } + } + g.Expect(found).Should(BeTrue(), "Old user should have finalizer after initial deployment") + }, timeout, interval).Should(Succeed()) + + // Rotate credentials: update secret to use new user + UpdateNovaCellConfigSecret("cell1", "nova-new", "rabbitmq-cell1") + + // Wait a moment for secret to propagate + time.Sleep(time.Second * 2) + + // Rolling update with new credentials - first node + deploy2Name := types.NamespacedName{Name: "deploy-rotate-node0", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy2Name, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-0", + })) + SimulateDeploymentComplete(deploy2Name, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // After first node, old user finalizer should remain (not all nodes updated) + Consistently(func(g Gomega) { + g.Expect(HasFinalizer("nova-old", oldUserFinalizer)).Should(BeTrue()) + }, time.Second*5, interval).Should(Succeed()) + + // Rolling update - second node + deploy3Name := types.NamespacedName{Name: "deploy-rotate-node1", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy3Name, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-1", + })) + SimulateDeploymentComplete(deploy3Name, dataplaneNodeSetName.Name, []string{"compute-1"}) + + // After all nodes updated with new credentials: + // 1. New user should have finalizer + // 2. Old user finalizer should be removed + Eventually(func(g Gomega) { + newUserObj := GetRabbitMQUser("nova-new") + newUserHasFinalizer := false + for _, f := range newUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + newUserHasFinalizer = true + break + } + } + g.Expect(newUserHasFinalizer).Should(BeTrue(), "New user should have finalizer after rotation") + + oldUserObj := GetRabbitMQUser("nova-old") + oldUserHasFinalizer := false + for _, f := range oldUserObj.Finalizers { + if f == oldUserFinalizer { + oldUserHasFinalizer = true + break + } + } + g.Expect(oldUserHasFinalizer).Should(BeFalse(), "Old user finalizer should be removed after rotation") + }, timeout, interval).Should(Succeed()) + }) + }) + + Context("Multi-Service RabbitMQ Cluster Management", func() { + var dataplaneNodeSetName types.NamespacedName + var novaServiceName, neutronServiceName, ironicServiceName types.NamespacedName + var novaUser, neutronUser, ironicUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + dataplaneNodeSetName = types.NamespacedName{ + Name: "compute-multiservice", + Namespace: namespace, + } + novaServiceName = types.NamespacedName{Name: "nova", Namespace: namespace} + neutronServiceName = types.NamespacedName{Name: "neutron", Namespace: namespace} + ironicServiceName = types.NamespacedName{Name: "ironic-neutron-agent", Namespace: namespace} + + // Create config secrets for different services using DIFFERENT clusters + CreateNovaCellConfigSecret("cell1", "nova-cell1", "rabbitmq-cell1") + CreateNeutronAgentConfigSecret("dhcp", "neutron", "rabbitmq-network") + CreateIronicNeutronAgentConfigSecret("ironic", "rabbitmq-baremetal") + + // Create RabbitMQ users + novaUser = CreateRabbitMQUser("nova-cell1") + neutronUser = CreateRabbitMQUser("neutron") + ironicUser = CreateRabbitMQUser("ironic") + + // Create services + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{"edpmServiceType": "nova"}) + CreateDataPlaneServiceFromSpec(neutronServiceName, map[string]interface{}{"edpmServiceType": "neutron-dhcp"}) + CreateDataPlaneServiceFromSpec(ironicServiceName, map[string]interface{}{"edpmServiceType": "ironic-neutron-agent"}) + + DeferCleanup(th.DeleteService, novaServiceName) + DeferCleanup(th.DeleteService, neutronServiceName) + DeferCleanup(th.DeleteService, ironicServiceName) + + // Create migration secrets required by each service + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "nova-migration-ssh-key"}) + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "neutron-migration-ssh-key"}) + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "ironic-neutron-agent-migration-ssh-key"}) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, novaUser) + _ = k8sClient.Delete(ctx, neutronUser) + _ = k8sClient.Delete(ctx, ironicUser) + }) + + It("Should manage service-specific finalizers independently across different clusters", func() { + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-multiservice"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-multiservice"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset with multiple services + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova", "neutron", "ironic-neutron-agent"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(dataplaneNodeSetName)) }) + + // Setup + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Deploy all services + deployName := types.NamespacedName{Name: "deploy-all-services", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deployName, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // Each service should have its own finalizer on its respective user + // Format: nodeset.os/{hash}-{service} + Eventually(func(g Gomega) { + // Nova should have finalizer with -nova suffix + novaUserObj := GetRabbitMQUser("nova-cell1") + novaHasFinalizer := false + for _, f := range novaUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" && len(f) >= 17 && f[len(f)-5:] == "-nova" { + novaHasFinalizer = true + break + } + } + g.Expect(novaHasFinalizer).Should(BeTrue(), "Nova user should have service-specific finalizer") + + // Neutron should have finalizer with -neutron suffix + neutronUserObj := GetRabbitMQUser("neutron") + neutronHasFinalizer := false + for _, f := range neutronUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" && len(f) >= 19 && f[len(f)-8:] == "-neutron" { + neutronHasFinalizer = true + break + } + } + g.Expect(neutronHasFinalizer).Should(BeTrue(), "Neutron user should have service-specific finalizer") + + // Ironic should have finalizer with -ironic suffix + ironicUserObj := GetRabbitMQUser("ironic") + ironicHasFinalizer := false + for _, f := range ironicUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" && len(f) >= 19 && f[len(f)-7:] == "-ironic" { + ironicHasFinalizer = true + break + } + } + g.Expect(ironicHasFinalizer).Should(BeTrue(), "Ironic user should have service-specific finalizer") + }, timeout, interval).Should(Succeed()) + + // Verify that finalizers are independent (each user has exactly 1 finalizer) + Eventually(func(g Gomega) { + novaCount := 0 + for _, f := range GetRabbitMQUser("nova-cell1").Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + novaCount++ + } + } + g.Expect(novaCount).Should(Equal(1), "Nova user should have exactly 1 finalizer") + + neutronCount := 0 + for _, f := range GetRabbitMQUser("neutron").Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + neutronCount++ + } + } + g.Expect(neutronCount).Should(Equal(1), "Neutron user should have exactly 1 finalizer") + + ironicCount := 0 + for _, f := range GetRabbitMQUser("ironic").Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + ironicCount++ + } + } + g.Expect(ironicCount).Should(Equal(1), "Ironic user should have exactly 1 finalizer") + }, timeout, interval).Should(Succeed()) + }) + }) + + Context("Deployment Timing and Secret Changes", func() { + var dataplaneNodeSetName types.NamespacedName + var novaServiceName types.NamespacedName + var novaUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + dataplaneNodeSetName = types.NamespacedName{ + Name: "compute-edge", + Namespace: namespace, + } + novaServiceName = types.NamespacedName{ + Name: "nova", + Namespace: namespace, + } + + CreateNovaCellConfigSecret("cell1", "nova-user1", "rabbitmq-cell1") + novaUser = CreateRabbitMQUser("nova-user1") + + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{ + "edpmServiceType": "nova", + }) + DeferCleanup(th.DeleteService, novaServiceName) + + // Create nova-migration-ssh-key secret required by nova service + CreateSSHSecret(types.NamespacedName{ + Namespace: namespace, + Name: "nova-migration-ssh-key", + }) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, novaUser) + }) + + It("Should use deployment completion time not creation time", func() { + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-timing"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-timing"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(dataplaneNodeSetName)) }) + + // Setup + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Create deployment (creation time = now) + deployName := types.NamespacedName{Name: "deploy-before-secret", Namespace: namespace} + CreateDataplaneDeployment(deployName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + }) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneDeployment(deployName)) }) + + // Wait a moment + time.Sleep(time.Second * 2) + + // Rotate secret (secret modified time = now, AFTER deployment creation) + UpdateNovaCellConfigSecret("cell1", "nova-user2", "rabbitmq-cell1") + newUser := CreateRabbitMQUser("nova-user2") + DeferCleanup(func() { _ = k8sClient.Delete(ctx, newUser) }) + + // Wait for secret to propagate + time.Sleep(time.Second * 2) + + // Now complete the deployment (completion time = now, AFTER secret change) + SimulateDeploymentComplete(deployName, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // Since deployment COMPLETED after secret change, it should track the node + // and manage finalizers for the NEW user (nova-user2), not the old one + Eventually(func(g Gomega) { + newUserObj := GetRabbitMQUser("nova-user2") + hasNewFinalizer := false + for _, f := range newUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasNewFinalizer = true + break + } + } + g.Expect(hasNewFinalizer).Should(BeTrue(), "New user should have finalizer (deployment completed after secret change)") + }, timeout, interval).Should(Succeed()) + }) + + It("Should reset tracking when secret changes during deployment", func() { + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-reset"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-reset"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset with 2 nodes + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + "compute-1": { + HostName: "compute-1", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.101", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(dataplaneNodeSetName)) }) + + // Setup + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-1"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Initial full deployment with nova-user1 to establish baseline + deployInitialName := types.NamespacedName{Name: "deploy-initial", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployInitialName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deployInitialName, dataplaneNodeSetName.Name, []string{"compute-0", "compute-1"}) + + // Wait for finalizer to be added + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + hasOldFinalizer := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasOldFinalizer = true + break + } + } + g.Expect(hasOldFinalizer).Should(BeTrue(), "Initial user should have finalizer after full deployment") + }, timeout, interval).Should(Succeed()) + + // Deploy first node again with same credentials (partial deployment) + deploy1Name := types.NamespacedName{Name: "deploy-node0", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy1Name, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-0", + })) + SimulateDeploymentComplete(deploy1Name, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // After partial deployment, old user should still have finalizer (not all nodes updated) + Consistently(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + hasOldFinalizer := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasOldFinalizer = true + break + } + } + g.Expect(hasOldFinalizer).Should(BeTrue(), "Old user should keep finalizer after partial deployment") + }, time.Second*5, interval).Should(Succeed()) + + // Change secret (this triggers credential rotation) + UpdateNovaCellConfigSecret("cell1", "nova-user2", "rabbitmq-cell1") + newUser := CreateRabbitMQUser("nova-user2") + DeferCleanup(func() { _ = k8sClient.Delete(ctx, newUser) }) + + // Wait for secret to propagate + time.Sleep(time.Second * 2) + + // Deploy both nodes with new secret + // This simulates a redeploy after secret rotation + deploy2Name := types.NamespacedName{Name: "deploy-both-new", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploy2Name, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deploy2Name, dataplaneNodeSetName.Name, []string{"compute-0", "compute-1"}) + + // After all nodes deployed with new credentials: + // 1. New user should have finalizer + // 2. Old user finalizer should be removed + Eventually(func(g Gomega) { + newUserObj := GetRabbitMQUser("nova-user2") + newUserHasFinalizer := false + for _, f := range newUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + newUserHasFinalizer = true + break + } + } + g.Expect(newUserHasFinalizer).Should(BeTrue(), "New user should have finalizer after full deployment with new credentials") + + oldUserObj := GetRabbitMQUser("nova-user1") + oldUserHasFinalizer := false + for _, f := range oldUserObj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + oldUserHasFinalizer = true + break + } + } + g.Expect(oldUserHasFinalizer).Should(BeFalse(), "Old user finalizer should be removed after rotation completes") + }, timeout, interval).Should(Succeed()) + }) + + PIt("Should add finalizers immediately during partial deployment (improved protection)", func() { + // This test demonstrates the improved behavior: finalizers are added as soon as + // ANY node starts using credentials, providing protection during rolling updates + // TODO: This test needs more work on credential rotation tracking in the controller + + // Setup network infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-partial"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-partial"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset with 2 nodes + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + "compute-1": { + HostName: "compute-1", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.101", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + DeferCleanup(func() { th.DeleteInstance(GetDataplaneNodeSet(dataplaneNodeSetName)) }) + + // Setup + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-1"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Initial full deployment with user1 + deployInitialName := types.NamespacedName{Name: "deploy-initial-partial", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployInitialName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deployInitialName, dataplaneNodeSetName.Name, []string{"compute-0", "compute-1"}) + + // Verify user1 has finalizer after full deployment + Eventually(func(g Gomega) { + user1 := GetRabbitMQUser("nova-user1") + hasFinalizer := false + for _, f := range user1.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasFinalizer = true + break + } + } + g.Expect(hasFinalizer).Should(BeTrue(), "User1 should have finalizer after full deployment") + }, timeout, interval).Should(Succeed()) + + // Change secret to user2 + UpdateNovaCellConfigSecret("cell1", "nova-user2", "rabbitmq-cell1") + user2 := CreateRabbitMQUser("nova-user2") + DeferCleanup(func() { _ = k8sClient.Delete(ctx, user2) }) + + // Wait for secret to propagate + time.Sleep(time.Second * 2) + + // PARTIAL deployment with new user2 (only compute-0) + // This is the key test: user2 should get finalizer immediately + deployPartialName := types.NamespacedName{Name: "deploy-partial-node0", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployPartialName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "ansibleLimit": "compute-0", + })) + SimulateDeploymentComplete(deployPartialName, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // CRITICAL TEST: After partial deployment (1 of 2 nodes): + // - user2 should have finalizer (NEW BEHAVIOR - immediate protection) + // - user1 should STILL have finalizer (compute-1 still using it) + Eventually(func(g Gomega) { + user2Obj := GetRabbitMQUser("nova-user2") + user2HasFinalizer := false + for _, f := range user2Obj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + user2HasFinalizer = true + break + } + } + g.Expect(user2HasFinalizer).Should(BeTrue(), "User2 should have finalizer immediately after ANY node uses it (partial deployment)") + + user1Obj := GetRabbitMQUser("nova-user1") + user1HasFinalizer := false + for _, f := range user1Obj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + user1HasFinalizer = true + break + } + } + g.Expect(user1HasFinalizer).Should(BeTrue(), "User1 should keep finalizer during partial deployment (compute-1 still using it)") + }, timeout, interval).Should(Succeed()) + + // Complete deployment with user2 (both nodes) + deployCompleteName := types.NamespacedName{Name: "deploy-complete-both", Namespace: namespace} + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deployCompleteName, map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + })) + SimulateDeploymentComplete(deployCompleteName, dataplaneNodeSetName.Name, []string{"compute-0", "compute-1"}) + + // After full deployment: user2 keeps finalizer, user1 finalizer removed + Eventually(func(g Gomega) { + user2Obj := GetRabbitMQUser("nova-user2") + user2HasFinalizer := false + for _, f := range user2Obj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + user2HasFinalizer = true + break + } + } + g.Expect(user2HasFinalizer).Should(BeTrue(), "User2 should keep finalizer after full deployment") + + user1Obj := GetRabbitMQUser("nova-user1") + user1HasFinalizer := false + for _, f := range user1Obj.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + user1HasFinalizer = true + break + } + } + g.Expect(user1HasFinalizer).Should(BeFalse(), "User1 finalizer should be removed after all nodes migrated") + }, timeout, interval).Should(Succeed()) + }) + }) + + When("A NodeSet with 2 nodes is created", func() { + var dataplaneNodeSetName types.NamespacedName + var dataplaneSSHSecretName types.NamespacedName + var caBundleSecretName types.NamespacedName + var dataplaneNetConfigName types.NamespacedName + var dnsMasqName types.NamespacedName + var dataplaneNode0Name types.NamespacedName + var dataplaneNode1Name types.NamespacedName + var novaServiceName types.NamespacedName + + BeforeEach(func() { + // Set OPERATOR_SERVICES to point to services directory + err := os.Setenv("OPERATOR_SERVICES", "../../../config/services") + Expect(err).NotTo(HaveOccurred()) + + dnsMasqName = types.NamespacedName{ + Name: "dnsmasq-rabbitmq-test", + Namespace: namespace, + } + dataplaneNodeSetName = types.NamespacedName{ + Name: "edpm-compute-rabbitmq-test", + Namespace: namespace, + } + dataplaneSSHSecretName = types.NamespacedName{ + Namespace: namespace, + Name: "dataplane-ansible-ssh-private-key-secret", + } + caBundleSecretName = types.NamespacedName{ + Namespace: namespace, + Name: "combined-ca-bundle", + } + dataplaneNetConfigName = types.NamespacedName{ + Namespace: namespace, + Name: "dataplane-netconfig-rabbitmq-test", + } + dataplaneNode0Name = types.NamespacedName{ + Namespace: namespace, + Name: "edpm-compute-0", + } + dataplaneNode1Name = types.NamespacedName{ + Namespace: namespace, + Name: "edpm-compute-1", + } + novaServiceName = types.NamespacedName{ + Namespace: namespace, + Name: "nova", + } + }) + + BeforeEach(func() { + // Create nova service + CreateDataPlaneServiceFromSpec(novaServiceName, map[string]interface{}{ + "edpmServiceType": "nova", + }) + DeferCleanup(th.DeleteService, novaServiceName) + + // Create network infrastructure + DeferCleanup(th.DeleteInstance, CreateNetConfig(dataplaneNetConfigName, DefaultNetConfigSpec())) + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + // Create nodeset with 2 nodes + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "edpm-compute-0": { + HostName: "edpm-compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + { + Name: "ctlplane", + SubnetName: "subnet1", + }, + }, + }, + "edpm-compute-1": { + HostName: "edpm-compute-1", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.101", + }, + Networks: []infrav1.IPSetNetwork{ + { + Name: "ctlplane", + SubnetName: "subnet1", + }, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + DeferCleanup(th.DeleteInstance, CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec)) + + // Create SSH and CA secrets + CreateSSHSecret(dataplaneSSHSecretName) + CreateCABundleSecret(caBundleSecretName) + + // Simulate IP sets + SimulateIPSetComplete(dataplaneNode0Name) + SimulateIPSetComplete(dataplaneNode1Name) + SimulateDNSDataComplete(dataplaneNodeSetName) + }) + + It("Should correctly count nodes without IP address aliases", func() { + // Verify that getAllNodeNamesFromNodeset returns only 2 nodes, not 4 + // This validates the fix for Bug 3 where IP addresses were counted as separate nodes + Eventually(func(g Gomega) { + nodeset := GetDataplaneNodeSet(dataplaneNodeSetName) + // Should have exactly 2 nodes defined (not 4 with hostName and ansibleHost) + g.Expect(nodeset.Spec.Nodes).Should(HaveLen(2)) + }, th.Timeout, th.Interval).Should(Succeed()) + }) + }) + + Context("When RabbitMQ users have cleanup-blocked finalizer", func() { + var novaUser *rabbitmqv1.RabbitMQUser + + BeforeEach(func() { + // Create Nova cell config secret + CreateNovaCellConfigSecret("cell1", "nova-user1", "rabbitmq-cell1") + + // Create RabbitMQ user with cleanup-blocked finalizer + novaUser = CreateRabbitMQUser("nova-user1") + + // Manually add the cleanup-blocked finalizer to simulate migration scenario + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + user.Finalizers = append(user.Finalizers, rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer) + g.Expect(k8sClient.Update(ctx, user)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Setup infrastructure + netConfigName := types.NamespacedName{Namespace: namespace, Name: "dataplane-netconfig-cleanup"} + DeferCleanup(th.DeleteInstance, CreateNetConfig(netConfigName, DefaultNetConfigSpec())) + + dnsMasqName := types.NamespacedName{Namespace: namespace, Name: "dnsmasq-cleanup"} + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + + CreateSSHSecret(types.NamespacedName{ + Namespace: namespace, + Name: "nova-migration-ssh-key", + }) + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, novaUser) + }) + + It("Should remove cleanup-blocked finalizer during normal reconciliation", func() { + // Verify cleanup-blocked finalizer is present initially + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + g.Expect(user.Finalizers).Should(ContainElement(rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Create nodeset with Nova service + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "compute-0": { + HostName: "compute-0", + Ansible: dataplanev1.AnsibleOpts{ + AnsibleHost: "192.168.122.100", + }, + Networks: []infrav1.IPSetNetwork{ + {Name: "ctlplane", SubnetName: "subnet1"}, + }, + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + "managementNetwork": "ctlplane", + "ansible": map[string]interface{}{ + "ansibleUser": "cloud-admin", + }, + }, + } + dataplaneNodeSetName := types.NamespacedName{ + Name: "edpm-nodeset-cleanup-test", + Namespace: namespace, + } + DeferCleanup(th.DeleteInstance, CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec)) + + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + CreateCABundleSecret(types.NamespacedName{Namespace: namespace, Name: "combined-ca-bundle"}) + SimulateIPSetComplete(types.NamespacedName{Namespace: namespace, Name: "compute-0"}) + SimulateDNSDataComplete(dataplaneNodeSetName) + + // Deploy the nodeset + deploymentName := types.NamespacedName{ + Name: "cleanup-deployment", + Namespace: namespace, + } + deploymentSpec := map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + } + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(deploymentName, deploymentSpec)) + SimulateDeploymentComplete(deploymentName, dataplaneNodeSetName.Name, []string{"compute-0"}) + + // Verify the cleanup-blocked finalizer is removed + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + g.Expect(user.Finalizers).ShouldNot(ContainElement(rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer), + "cleanup-blocked finalizer should be automatically removed during reconciliation") + }, timeout, interval).Should(Succeed()) + + // Verify our nodeset finalizer is still added + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + hasNodesetFinalizer := false + for _, f := range user.Finalizers { + if len(f) > 11 && f[:11] == "nodeset.os/" { + hasNodesetFinalizer = true + break + } + } + g.Expect(hasNodesetFinalizer).Should(BeTrue(), + "nodeset finalizer should still be present") + }, timeout, interval).Should(Succeed()) + }) + + It("Should remove cleanup-blocked finalizer during nodeset deletion", func() { + // Create SSH secret required by nodeset + CreateSSHSecret(types.NamespacedName{Namespace: namespace, Name: "dataplane-ansible-ssh-private-key-secret"}) + + // Create a simple nodeset - just need it to exist for testing deletion cleanup + nodeSetSpec := map[string]interface{}{ + "preProvisioned": true, + "services": []string{"nova"}, + "nodes": map[string]dataplanev1.NodeSection{ + "cleanup-node": { + HostName: "cleanup-node", + }, + }, + "nodeTemplate": map[string]interface{}{ + "ansibleSSHPrivateKeySecret": "dataplane-ansible-ssh-private-key-secret", + }, + } + dataplaneNodeSetName := types.NamespacedName{ + Name: "edpm-nodeset-deletion-cleanup", + Namespace: namespace, + } + CreateDataplaneNodeSet(dataplaneNodeSetName, nodeSetSpec) + + // Wait for nodeset to be ready and have finalizerHash set + Eventually(func(g Gomega) { + nodeset := GetDataplaneNodeSet(dataplaneNodeSetName) + g.Expect(nodeset.Status.FinalizerHash).ShouldNot(BeEmpty()) + }, timeout, interval).Should(Succeed()) + + // Verify cleanup-blocked finalizer is still present on the user + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + g.Expect(user.Finalizers).Should(ContainElement(rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Delete the nodeset + nodeset := GetDataplaneNodeSet(dataplaneNodeSetName) + Expect(k8sClient.Delete(ctx, nodeset)).Should(Succeed()) + + // Wait for nodeset to be deleted + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, dataplaneNodeSetName, nodeset) + g.Expect(err).Should(HaveOccurred()) + }, timeout, interval).Should(Succeed()) + + // Verify cleanup-blocked finalizer is removed from the user + Eventually(func(g Gomega) { + user := GetRabbitMQUser("nova-user1") + g.Expect(user.Finalizers).ShouldNot(ContainElement(rabbitmqv1.RabbitMQUserCleanupBlockedFinalizer), + "cleanup-blocked finalizer should be removed during nodeset deletion") + }, timeout, interval).Should(Succeed()) + }) + }) +}) diff --git a/test/functional/dataplane/suite_test.go b/test/functional/dataplane/suite_test.go index 0b751b0bd..fd2672e7b 100644 --- a/test/functional/dataplane/suite_test.go +++ b/test/functional/dataplane/suite_test.go @@ -42,6 +42,7 @@ import ( corev1 "k8s.io/api/core/v1" infrav1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1" openstackv1 "github.com/openstack-k8s-operators/openstack-operator/api/core/v1beta1" dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/api/dataplane/v1beta1" @@ -175,6 +176,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = infrav1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = rabbitmqv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) err = openstackv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = certmgrv1.AddToScheme(scheme.Scheme)