diff --git a/cloudstack_annotation_test.go b/cloudstack_annotation_test.go new file mode 100644 index 00000000..22e8c150 --- /dev/null +++ b/cloudstack_annotation_test.go @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cloudstack + +import ( + "context" + "errors" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +const annotationKey = "service.beta.kubernetes.io/cloudstack-load-balancer-ip-associated-by-controller" + +var servicesResource = schema.GroupResource{Resource: "services"} + +func annotatedService(annotations map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default", Annotations: annotations}, + } +} + +func patchActions(client *fake.Clientset) []k8stesting.PatchAction { + var patches []k8stesting.PatchAction + for _, action := range client.Actions() { + if patch, ok := action.(k8stesting.PatchAction); ok { + patches = append(patches, patch) + } + } + return patches +} + +func failPatches(client *fake.Clientset, results ...error) { + attempt := 0 + client.PrependReactor("patch", "services", func(k8stesting.Action) (bool, runtime.Object, error) { + err := results[min(attempt, len(results)-1)] + attempt++ + return err != nil, nil, err + }) +} + +// TestSetServiceAnnotation checks when an annotation is patched and that only the one key is sent. +func TestSetServiceAnnotation(t *testing.T) { + t.Run("no client builder skips the update", func(t *testing.T) { + cs := &CSCloud{} + + if err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true"); err != nil { + t.Errorf("setServiceAnnotation() unexpected error: %v", err) + } + }) + + t.Run("client cannot be built", func(t *testing.T) { + cs := &CSCloud{clientBuilder: &fakeClientBuilder{err: errors.New("no config")}} + + err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true") + + if err == nil || !strings.Contains(err.Error(), "failed to get Kubernetes client: no config") { + t.Errorf("setServiceAnnotation() error = %v, want the client error", err) + } + }) + + t.Run("missing service is skipped", func(t *testing.T) { + client := fake.NewSimpleClientset() + cs := &CSCloud{clientBuilder: &fakeClientBuilder{client: client}} + + if err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true"); err != nil { + t.Errorf("setServiceAnnotation() unexpected error: %v", err) + } + if got := len(patchActions(client)); got != 0 { + t.Errorf("got %d patches, want none", got) + } + }) + + t.Run("service lookup fails", func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "services", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom") + }) + cs := &CSCloud{clientBuilder: &fakeClientBuilder{client: client}} + + err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true") + + if err == nil || !strings.Contains(err.Error(), "failed to get service: boom") { + t.Errorf("setServiceAnnotation() error = %v, want the lookup error", err) + } + }) + + t.Run("annotation already set is not patched", func(t *testing.T) { + client := fake.NewSimpleClientset(annotatedService(map[string]string{annotationKey: "true"})) + cs := &CSCloud{clientBuilder: &fakeClientBuilder{client: client}} + + if err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true"); err != nil { + t.Errorf("setServiceAnnotation() unexpected error: %v", err) + } + if got := len(patchActions(client)); got != 0 { + t.Errorf("got %d patches, want none", got) + } + }) + + t.Run("different value is merge patched", func(t *testing.T) { + client := fake.NewSimpleClientset(annotatedService(map[string]string{annotationKey: "false", "other": "kept"})) + cs := &CSCloud{clientBuilder: &fakeClientBuilder{client: client}} + + if err := cs.setServiceAnnotation(context.TODO(), annotatedService(nil), annotationKey, "true"); err != nil { + t.Fatalf("setServiceAnnotation() unexpected error: %v", err) + } + + patches := patchActions(client) + if len(patches) != 1 { + t.Fatalf("got %d patches, want 1", len(patches)) + } + wantPatch := `{"metadata":{"annotations":{"` + annotationKey + `":"true"}}}` + if patches[0].GetName() != "web" || patches[0].GetNamespace() != "default" || + patches[0].GetPatchType() != types.MergePatchType || string(patches[0].GetPatch()) != wantPatch { + t.Errorf("patch = %s %s/%s %s, want a merge patch of web %s", patches[0].GetPatchType(), patches[0].GetNamespace(), patches[0].GetName(), patches[0].GetPatch(), wantPatch) + } + svc, err := client.CoreV1().Services("default").Get(context.TODO(), "web", metav1.GetOptions{}) + if err != nil || svc.Annotations[annotationKey] != "true" || svc.Annotations["other"] != "kept" { + t.Errorf("annotations after the patch = %v, %v, want the key updated and the other one kept", svc.Annotations, err) + } + }) +} + +// TestPatchServiceAnnotation checks how a failing patch is retried or given up on. +func TestPatchServiceAnnotation(t *testing.T) { + conflict := apierrors.NewConflict(servicesResource, "web", errors.New("changed")) + tests := []struct { + name string + results []error + wantPatches int + wantErr string + }{ + {name: "first attempt succeeds", results: []error{nil}, wantPatches: 1}, + {name: "conflict then success", results: []error{conflict, nil}, wantPatches: 2}, + {name: "conflicts exhaust the retries", results: []error{conflict}, wantPatches: 3, wantErr: "failed to update service annotation after 3 retries due to conflicts"}, + {name: "service deleted meanwhile", results: []error{apierrors.NewNotFound(servicesResource, "web")}, wantPatches: 1}, + {name: "other error is not retried", results: []error{errors.New("boom")}, wantPatches: 1, wantErr: "failed to patch service annotation: boom"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(annotatedService(nil)) + failPatches(client, tt.results...) + cs := &CSCloud{} + + err := cs.patchServiceAnnotation(context.TODO(), client, "default", "web", annotationKey, "true") + + if tt.wantErr == "" && err != nil { + t.Errorf("patchServiceAnnotation() unexpected error: %v", err) + } + if tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)) { + t.Errorf("patchServiceAnnotation() error = %v, want it to contain %q", err, tt.wantErr) + } + if got := len(patchActions(client)); got != tt.wantPatches { + t.Errorf("got %d patch attempts, want %d", got, tt.wantPatches) + } + }) + } +} diff --git a/cloudstack_loadbalancer_entrypoints_test.go b/cloudstack_loadbalancer_entrypoints_test.go new file mode 100644 index 00000000..0b94c203 --- /dev/null +++ b/cloudstack_loadbalancer_entrypoints_test.go @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cloudstack + +import ( + "context" + "errors" + "reflect" + "slices" + "strings" + "testing" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type entryPointEnv struct { + cs *CSCloud + lb *cloudstack.MockLoadBalancerServiceIface + vm *cloudstack.MockVirtualMachineServiceIface + network *cloudstack.MockNetworkServiceIface + address *cloudstack.MockAddressServiceIface + fw *cloudstack.MockFirewallServiceIface + service *corev1.Service +} + +func newEntryPointEnv(t *testing.T, projectID string) *entryPointEnv { + t.Helper() + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + + e := &entryPointEnv{ + lb: cloudstack.NewMockLoadBalancerServiceIface(ctrl), + vm: cloudstack.NewMockVirtualMachineServiceIface(ctrl), + network: cloudstack.NewMockNetworkServiceIface(ctrl), + address: cloudstack.NewMockAddressServiceIface(ctrl), + fw: cloudstack.NewMockFirewallServiceIface(ctrl), + service: &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default", UID: "test-uid"}}, + } + e.cs = &CSCloud{ + projectID: projectID, + client: &cloudstack.CloudStackClient{ + LoadBalancer: e.lb, VirtualMachine: e.vm, Network: e.network, Address: e.address, Firewall: e.fw, + }, + } + return e +} + +func tcpRule(id, port, ip, ipID string) *cloudstack.LoadBalancerRule { + return &cloudstack.LoadBalancerRule{ + Id: id, Name: "atestuid-tcp-" + port, Publicport: port, Protocol: "tcp", + Publicip: ip, Publicipid: ipID, Networkid: "net-1", + } +} + +// expectRuleListing expects the lookup of the Service's rules and checks the filter it sends. +func (e *entryPointEnv) expectRuleListing(t *testing.T, rules []*cloudstack.LoadBalancerRule, err error) *gomock.Call { + t.Helper() + e.lb.EXPECT().NewListLoadBalancerRulesParams().Return(&cloudstack.ListLoadBalancerRulesParams{}) + return e.lb.EXPECT().ListLoadBalancerRules(gomock.Any()). + DoAndReturn(func(p *cloudstack.ListLoadBalancerRulesParams) (*cloudstack.ListLoadBalancerRulesResponse, error) { + if keyword, _ := p.GetKeyword(); keyword != "atestuid" { + t.Errorf("rule listing keyword = %q, want atestuid", keyword) + } + if listAll, _ := p.GetListall(); !listAll { + t.Errorf("rule listing does not set listall") + } + if project, _ := p.GetProjectid(); project != e.cs.projectID { + t.Errorf("rule listing project = %q, want %q", project, e.cs.projectID) + } + if err != nil { + return nil, err + } + return &cloudstack.ListLoadBalancerRulesResponse{Count: len(rules), LoadBalancerRules: rules}, nil + }) +} + +// TestGetLoadBalancerStatus checks what Kubernetes is told about a Service's load balancer and that the +// rules are looked up by the Service's load balancer name in its project. +func TestGetLoadBalancerStatus(t *testing.T) { + tests := []struct { + name string + projectID string + rules []*cloudstack.LoadBalancerRule + err error + wantStatus *corev1.LoadBalancerStatus + wantExists bool + wantErr string + }{ + {name: "no rules means no load balancer"}, + { + name: "rules give the ingress IP", + rules: []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, + wantStatus: &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.1"}}}, + wantExists: true, + }, + { + name: "rules in a project", + projectID: testProjectID, + rules: []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, + wantStatus: &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.1"}}}, + wantExists: true, + }, + {name: "listing fails", err: errors.New("boom"), wantErr: "error retrieving load balancer rules: boom"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := newEntryPointEnv(t, tt.projectID) + env.expectRuleListing(t, tt.rules, tt.err) + + status, exists, err := env.cs.GetLoadBalancer(context.TODO(), testClusterName, env.service) + + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("GetLoadBalancer() error = %v, want %q", err, tt.wantErr) + } + return + } + if err != nil || exists != tt.wantExists { + t.Fatalf("GetLoadBalancer() = %v, %v, want exists %v", exists, err, tt.wantExists) + } + if (status == nil) != (tt.wantStatus == nil) || (status != nil && !reflect.DeepEqual(status, tt.wantStatus)) { + t.Errorf("GetLoadBalancer() status = %+v, want %+v", status, tt.wantStatus) + } + }) + } +} + +func (e *entryPointEnv) expectNodeVMs(vms ...*cloudstack.VirtualMachine) { + e.vm.EXPECT().NewListVirtualMachinesParams().Return(&cloudstack.ListVirtualMachinesParams{}) + e.vm.EXPECT().ListVirtualMachines(gomock.Any()). + Return(&cloudstack.ListVirtualMachinesResponse{Count: len(vms), VirtualMachines: vms}, nil) +} + +func nodeVM(id, name string) *cloudstack.VirtualMachine { + return &cloudstack.VirtualMachine{Id: id, Name: name, Nic: []cloudstack.Nic{{Networkid: "net-1"}}} +} + +// expectRuleMembers answers the member listing of each rule by the rule ID the listing was built for. +func (e *entryPointEnv) expectRuleMembers(members map[string][]string, failFor string) { + e.lb.EXPECT().NewListLoadBalancerRuleInstancesParams(gomock.Any()). + DoAndReturn(func(id string) *cloudstack.ListLoadBalancerRuleInstancesParams { + p := &cloudstack.ListLoadBalancerRuleInstancesParams{} + p.SetId(id) + return p + }).AnyTimes() + e.lb.EXPECT().ListLoadBalancerRuleInstances(gomock.Any()). + DoAndReturn(func(p *cloudstack.ListLoadBalancerRuleInstancesParams) (*cloudstack.ListLoadBalancerRuleInstancesResponse, error) { + id, _ := p.GetId() + if id == failFor { + return nil, errors.New("boom") + } + var vms []*cloudstack.VirtualMachine + for _, vmID := range members[id] { + vms = append(vms, &cloudstack.VirtualMachine{Id: vmID}) + } + return &cloudstack.ListLoadBalancerRuleInstancesResponse{Count: len(vms), LoadBalancerRuleInstances: vms}, nil + }).AnyTimes() +} + +// recordMembershipChanges records the hosts assigned to and removed from each rule. +func (e *entryPointEnv) recordMembershipChanges() (assigned, removed map[string][]string) { + assigned, removed = map[string][]string{}, map[string][]string{} + + e.lb.EXPECT().NewAssignToLoadBalancerRuleParams(gomock.Any()). + DoAndReturn(func(id string) *cloudstack.AssignToLoadBalancerRuleParams { + p := &cloudstack.AssignToLoadBalancerRuleParams{} + p.SetId(id) + return p + }).AnyTimes() + e.lb.EXPECT().AssignToLoadBalancerRule(gomock.Any()). + DoAndReturn(func(p *cloudstack.AssignToLoadBalancerRuleParams) (*cloudstack.AssignToLoadBalancerRuleResponse, error) { + id, _ := p.GetId() + ids, _ := p.GetVirtualmachineids() + assigned[id] = ids + return &cloudstack.AssignToLoadBalancerRuleResponse{}, nil + }).AnyTimes() + e.lb.EXPECT().NewRemoveFromLoadBalancerRuleParams(gomock.Any()). + DoAndReturn(func(id string) *cloudstack.RemoveFromLoadBalancerRuleParams { + p := &cloudstack.RemoveFromLoadBalancerRuleParams{} + p.SetId(id) + return p + }).AnyTimes() + e.lb.EXPECT().RemoveFromLoadBalancerRule(gomock.Any()). + DoAndReturn(func(p *cloudstack.RemoveFromLoadBalancerRuleParams) (*cloudstack.RemoveFromLoadBalancerRuleResponse, error) { + id, _ := p.GetId() + ids, _ := p.GetVirtualmachineids() + removed[id] = ids + return &cloudstack.RemoveFromLoadBalancerRuleResponse{}, nil + }).AnyTimes() + + return assigned, removed +} + +// TestUpdateLoadBalancerMembership checks that each rule is brought in line with the nodes by +// assigning only the missing hosts and removing only the stale ones. +func TestUpdateLoadBalancerMembership(t *testing.T) { + tests := []struct { + name string + members map[string][]string + wantAssigned map[string][]string + wantRemoved map[string][]string + }{ + { + name: "rules already match", + members: map[string][]string{"rule-80": {"vm-1", "vm-2"}, "rule-8080": {"vm-2", "vm-1"}}, + }, + { + name: "a node was added", + members: map[string][]string{"rule-80": {"vm-1"}, "rule-8080": {"vm-1", "vm-2"}}, + wantAssigned: map[string][]string{"rule-80": {"vm-2"}}, + }, + { + name: "a node was removed", + members: map[string][]string{"rule-80": {"vm-1", "vm-2", "vm-3"}, "rule-8080": {"vm-1", "vm-2"}}, + wantRemoved: map[string][]string{"rule-80": {"vm-3"}}, + }, + { + name: "nodes were replaced on one rule only", + members: map[string][]string{"rule-80": {"vm-1", "vm-3"}, "rule-8080": {"vm-1", "vm-2"}}, + wantAssigned: map[string][]string{"rule-80": {"vm-2"}}, + wantRemoved: map[string][]string{"rule-80": {"vm-3"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := newEntryPointEnv(t, testProjectID) + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{ + tcpRule("rule-80", "80", "203.0.113.1", "ip-1"), + tcpRule("rule-8080", "8080", "203.0.113.1", "ip-1"), + }, nil) + env.expectNodeVMs(nodeVM("vm-1", "node-1"), nodeVM("vm-2", "node-2"), nodeVM("vm-9", "other")) + env.expectRuleMembers(tt.members, "") + assigned, removed := env.recordMembershipChanges() + + if err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1", "node-2")); err != nil { + t.Fatalf("UpdateLoadBalancer() unexpected error: %v", err) + } + + assertMembershipChanges(t, "assigned", assigned, tt.wantAssigned) + assertMembershipChanges(t, "removed", removed, tt.wantRemoved) + }) + } +} + +func assertMembershipChanges(t *testing.T, what string, got, want map[string][]string) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s hosts on rules %v, want %v", what, got, want) + return + } + for rule, wantIDs := range want { + gotIDs := slices.Clone(got[rule]) + slices.Sort(gotIDs) + if !slices.Equal(gotIDs, wantIDs) { + t.Errorf("%s hosts on %s = %v, want %v", what, rule, gotIDs, wantIDs) + } + } +} + +// TestUpdateLoadBalancerFailures checks that a failing lookup stops the update before any rule is changed. +func TestUpdateLoadBalancerFailures(t *testing.T) { + t.Run("rule listing fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, nil, errors.New("boom")) + + err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1")) + + if err == nil || !strings.Contains(err.Error(), "error retrieving load balancer rules: boom") { + t.Errorf("UpdateLoadBalancer() error = %v, want the listing error", err) + } + }) + + t.Run("no node is known to CloudStack", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.expectNodeVMs(nodeVM("vm-9", "other")) + + err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1")) + + if err == nil || !strings.Contains(err.Error(), "none of the hosts matched") { + t.Errorf("UpdateLoadBalancer() error = %v, want the host matching error", err) + } + }) + + t.Run("member listing fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.expectNodeVMs(nodeVM("vm-1", "node-1")) + env.expectRuleMembers(nil, "rule-80") + + err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1")) + + if err == nil || !strings.Contains(err.Error(), "error retrieving associated instances: boom") { + t.Errorf("UpdateLoadBalancer() error = %v, want the member listing error", err) + } + }) + + t.Run("assigning hosts fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.expectNodeVMs(nodeVM("vm-1", "node-1")) + env.expectRuleMembers(map[string][]string{"rule-80": {}}, "") + env.lb.EXPECT().NewAssignToLoadBalancerRuleParams("rule-80").Return(&cloudstack.AssignToLoadBalancerRuleParams{}) + env.lb.EXPECT().AssignToLoadBalancerRule(gomock.Any()).Return(nil, errors.New("boom")) + + err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1")) + + if err == nil || !strings.Contains(err.Error(), "error assigning hosts to load balancer rule atestuid-tcp-80: boom") { + t.Errorf("UpdateLoadBalancer() error = %v, want the assign error", err) + } + }) + + t.Run("removing hosts fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.expectNodeVMs(nodeVM("vm-1", "node-1")) + env.expectRuleMembers(map[string][]string{"rule-80": {"vm-1", "vm-3"}}, "") + env.lb.EXPECT().NewRemoveFromLoadBalancerRuleParams("rule-80").Return(&cloudstack.RemoveFromLoadBalancerRuleParams{}) + env.lb.EXPECT().RemoveFromLoadBalancerRule(gomock.Any()).Return(nil, errors.New("boom")) + + err := env.cs.UpdateLoadBalancer(context.TODO(), testClusterName, env.service, nodesNamed("node-1")) + + if err == nil || !strings.Contains(err.Error(), "error removing hosts from load balancer rule atestuid-tcp-80: boom") { + t.Errorf("UpdateLoadBalancer() error = %v, want the remove error", err) + } + }) +} + +// expectRuleDeletion expects the firewall cleanup and the deletion of one rule on a Firewall network. +func (e *entryPointEnv) expectRuleDeletion(t *testing.T, ruleID, ipID string) { + t.Helper() + e.network.EXPECT().GetNetworkByID("net-1", gomock.Any()).Return(&cloudstack.Network{Id: "net-1"}, 1, nil) + e.fw.EXPECT().NewListFirewallRulesParams().Return(&cloudstack.ListFirewallRulesParams{}) + e.fw.EXPECT().ListFirewallRules(gomock.Any()). + DoAndReturn(func(p *cloudstack.ListFirewallRulesParams) (*cloudstack.ListFirewallRulesResponse, error) { + if got, _ := p.GetIpaddressid(); got != ipID { + t.Errorf("firewall rules listed for IP %q, want %q", got, ipID) + } + return &cloudstack.ListFirewallRulesResponse{}, nil + }) + e.lb.EXPECT().NewDeleteLoadBalancerRuleParams(ruleID).Return(&cloudstack.DeleteLoadBalancerRuleParams{}) + e.lb.EXPECT().DeleteLoadBalancerRule(gomock.Any()).Return(&cloudstack.DeleteLoadBalancerRuleResponse{}, nil) +} + +func (e *entryPointEnv) expectIPLookup(t *testing.T, ipID string, ip *cloudstack.PublicIpAddress, err error) { + t.Helper() + e.address.EXPECT().GetPublicIpAddressByID(gomock.Any(), gomock.Any()). + DoAndReturn(func(id string, opts ...cloudstack.OptionFunc) (*cloudstack.PublicIpAddress, int, error) { + if id != ipID || appliedProjectID(t, opts) != e.cs.projectID { + t.Errorf("IP lookup of %q in project %q, want %q in %q", id, appliedProjectID(t, opts), ipID, e.cs.projectID) + } + if err != nil { + return nil, 0, err + } + return ip, 1, nil + }) +} + +func (e *entryPointEnv) expectRulesUsingIP(t *testing.T, ipID string, others []*cloudstack.LoadBalancerRule, err error) { + t.Helper() + e.lb.EXPECT().NewListLoadBalancerRulesParams().Return(&cloudstack.ListLoadBalancerRulesParams{}) + e.lb.EXPECT().ListLoadBalancerRules(gomock.Any()). + DoAndReturn(func(p *cloudstack.ListLoadBalancerRulesParams) (*cloudstack.ListLoadBalancerRulesResponse, error) { + if got, _ := p.GetPublicipid(); got != ipID { + t.Errorf("rules listed for IP %q, want %q", got, ipID) + } + if listAll, _ := p.GetListall(); !listAll { + t.Errorf("rules using the IP are listed without listall") + } + if project, _ := p.GetProjectid(); project != e.cs.projectID { + t.Errorf("rules using the IP listed in project %q, want %q", project, e.cs.projectID) + } + if err != nil { + return nil, err + } + return &cloudstack.ListLoadBalancerRulesResponse{Count: len(others), LoadBalancerRules: others}, nil + }) +} + +func (e *entryPointEnv) expectIPRelease(t *testing.T, ipID string, err error) { + t.Helper() + e.address.EXPECT().NewDisassociateIpAddressParams(ipID).Return(&cloudstack.DisassociateIpAddressParams{}) + e.address.EXPECT().DisassociateIpAddress(gomock.Any()).Return(&cloudstack.DisassociateIpAddressResponse{}, err) +} + +// TestEnsureLoadBalancerDeletedPublicIP checks which public IPs are released once the Service's +// rules are gone: one the controller allocated, and a user supplied one only when the controller +// associated it and no other Service still has rules on it. +func TestEnsureLoadBalancerDeletedPublicIP(t *testing.T) { + const ipID = "ip-1" + allocated := &cloudstack.PublicIpAddress{Id: ipID, Allocated: "2024-01-01T00:00:00+0000"} + otherRule := tcpRule("other-rule", "8080", "203.0.113.1", ipID) + + tests := []struct { + name string + userIP string + annotation string + expect func(e *entryPointEnv, t *testing.T) + wantErrorMsg string + }{ + { + name: "IP allocated by the controller is released", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPRelease(t, ipID, nil) + }, + }, + { + name: "failure to release the IP is returned", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPRelease(t, ipID, errors.New("boom")) + }, + wantErrorMsg: "error releasing load balancer IP 203.0.113.1: boom", + }, + { + name: "user IP the controller did not associate is kept", + userIP: "203.0.113.1", + expect: func(e *entryPointEnv, t *testing.T) {}, + }, + { + name: "user IP associated by the controller is released when no rule uses it", + userIP: "203.0.113.1", + annotation: "true", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPLookup(t, ipID, allocated, nil) + e.expectRulesUsingIP(t, ipID, nil, nil) + e.expectIPRelease(t, ipID, nil) + }, + }, + { + name: "user IP associated by the controller is kept while another rule uses it", + userIP: "203.0.113.1", + annotation: "true", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPLookup(t, ipID, allocated, nil) + e.expectRulesUsingIP(t, ipID, []*cloudstack.LoadBalancerRule{otherRule}, nil) + }, + }, + { + name: "user IP is kept when it cannot be looked up", + userIP: "203.0.113.1", + annotation: "true", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPLookup(t, ipID, nil, errors.New("boom")) + }, + }, + { + name: "user IP is kept when the rules using it cannot be listed", + userIP: "203.0.113.1", + annotation: "true", + expect: func(e *entryPointEnv, t *testing.T) { + e.expectIPLookup(t, ipID, allocated, nil) + e.expectRulesUsingIP(t, ipID, nil, errors.New("boom")) + }, + }, + } + + for _, tt := range tests { + for _, projectID := range []string{"", testProjectID} { + t.Run(tt.name+"/project "+projectID, func(t *testing.T) { + env := newEntryPointEnv(t, projectID) + env.service.Spec.LoadBalancerIP = tt.userIP + if tt.annotation != "" { + env.service.Annotations = map[string]string{ServiceAnnotationLoadBalancerIPAssociatedByController: tt.annotation} + } + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", ipID)}, nil) + env.expectRuleDeletion(t, "rule-80", ipID) + tt.expect(env, t) + + err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service) + + if tt.wantErrorMsg == "" && err != nil { + t.Errorf("EnsureLoadBalancerDeleted() unexpected error: %v", err) + } + if tt.wantErrorMsg != "" && (err == nil || err.Error() != tt.wantErrorMsg) { + t.Errorf("EnsureLoadBalancerDeleted() error = %v, want %q", err, tt.wantErrorMsg) + } + }) + } + } +} + +// TestEnsureLoadBalancerDeletedWithUnparsableRule checks that a rule whose protocol or port the +// provider cannot parse does not fail the deletion: an unknown protocol leaves the rule alone, and +// an unreadable port deletes the rule without touching the firewall. +func TestEnsureLoadBalancerDeletedWithUnparsableRule(t *testing.T) { + t.Run("unknown protocol leaves the rule alone", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.service.Spec.LoadBalancerIP = "203.0.113.1" + rule := tcpRule("rule-1", "80", "203.0.113.1", "ip-1") + rule.Protocol = "icmp" + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{rule}, nil) + + if err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service); err != nil { + t.Errorf("EnsureLoadBalancerDeleted() unexpected error: %v", err) + } + }) + + t.Run("unreadable port deletes the rule without firewall cleanup", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.service.Spec.LoadBalancerIP = "203.0.113.1" + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-1", "http", "203.0.113.1", "ip-1")}, nil) + env.lb.EXPECT().NewDeleteLoadBalancerRuleParams("rule-1").Return(&cloudstack.DeleteLoadBalancerRuleParams{}) + env.lb.EXPECT().DeleteLoadBalancerRule(gomock.Any()).Return(&cloudstack.DeleteLoadBalancerRuleResponse{}, nil) + + if err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service); err != nil { + t.Errorf("EnsureLoadBalancerDeleted() unexpected error: %v", err) + } + }) +} + +// TestEnsureLoadBalancerDeletedFailures checks that a failing lookup is returned before any rule is deleted. +func TestEnsureLoadBalancerDeletedFailures(t *testing.T) { + t.Run("rule listing fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, nil, errors.New("boom")) + + err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service) + + if err == nil || !strings.Contains(err.Error(), "error retrieving load balancer rules: boom") { + t.Errorf("EnsureLoadBalancerDeleted() error = %v, want the listing error", err) + } + }) + + t.Run("network of the rule cannot be looked up", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.network.EXPECT().GetNetworkByID("net-1", gomock.Any()).Return(nil, 0, errors.New("boom")) + + err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service) + + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("EnsureLoadBalancerDeleted() error = %v, want the network error", err) + } + }) + + t.Run("deleting the rule fails", func(t *testing.T) { + env := newEntryPointEnv(t, "") + env.expectRuleListing(t, []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")}, nil) + env.network.EXPECT().GetNetworkByID("net-1", gomock.Any()).Return(&cloudstack.Network{Id: "net-1"}, 1, nil) + env.fw.EXPECT().NewListFirewallRulesParams().Return(&cloudstack.ListFirewallRulesParams{}) + env.fw.EXPECT().ListFirewallRules(gomock.Any()).Return(&cloudstack.ListFirewallRulesResponse{}, nil) + env.lb.EXPECT().NewDeleteLoadBalancerRuleParams("rule-80").Return(&cloudstack.DeleteLoadBalancerRuleParams{}) + env.lb.EXPECT().DeleteLoadBalancerRule(gomock.Any()).Return(nil, errors.New("boom")) + + err := env.cs.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, env.service) + + if err == nil || !strings.Contains(err.Error(), "error deleting load balancer rule atestuid-tcp-80: boom") { + t.Errorf("EnsureLoadBalancerDeleted() error = %v, want the delete error", err) + } + }) +} + +// TestEnsureLoadBalancerRejectsBeforeCreatingAnything checks that a Service or lookup that cannot +// be served fails before an IP, rule or firewall entry is created: the mocks have no expectation +// for any of those calls, so one would fail the test. +func TestEnsureLoadBalancerRejectsBeforeCreatingAnything(t *testing.T) { + http := corev1.ServicePort{Name: "http", Port: 80, NodePort: 30000, Protocol: corev1.ProtocolTCP} + + tests := []struct { + name string + ports []corev1.ServicePort + setup func(e *ensureLBTestEnv) + wantErr string + }{ + { + name: "no ports", + setup: func(e *ensureLBTestEnv) {}, + wantErr: "requested load balancer with no ports", + }, + { + name: "rule listing fails", + ports: []corev1.ServicePort{http}, + setup: func(e *ensureLBTestEnv) { + e.lb.EXPECT().NewListLoadBalancerRulesParams().Return(&cloudstack.ListLoadBalancerRulesParams{}) + e.lb.EXPECT().ListLoadBalancerRules(gomock.Any()).Return(nil, errors.New("boom")) + }, + wantErr: "error retrieving load balancer rules: boom", + }, + { + name: "unsupported session affinity", + ports: []corev1.ServicePort{http}, + setup: func(e *ensureLBTestEnv) { + e.service.Spec.SessionAffinity = "Bogus" + e.expectRules() + }, + wantErr: "unsupported load balancer affinity: Bogus", + }, + { + name: "nodes cannot be listed", + ports: []corev1.ServicePort{http}, + setup: func(e *ensureLBTestEnv) { + e.expectRules() + e.vm.EXPECT().NewListVirtualMachinesParams().Return(&cloudstack.ListVirtualMachinesParams{}) + e.vm.EXPECT().ListVirtualMachines(gomock.Any()).Return(nil, errors.New("boom")) + }, + wantErr: "error retrieving list of hosts: boom", + }, + { + name: "network cannot be looked up", + ports: []corev1.ServicePort{http}, + setup: func(e *ensureLBTestEnv) { + e.expectRules() + e.expectHosts() + e.network.EXPECT().GetNetworkByID("net-1", gomock.Any()).Return(nil, 0, errors.New("boom")) + }, + wantErr: "boom", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + env := newEnsureLBTestEnv(ctrl, nil, tt.ports) + tt.setup(env) + + status, err := env.cs.EnsureLoadBalancer(context.TODO(), testClusterName, env.service, env.nodes) + + if status != nil || err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("EnsureLoadBalancer() = %v, %v, want an error containing %q", status, err, tt.wantErr) + } + }) + } +} + +// TestEnsureLoadBalancerReportsHostnameAnnotation checks that a hostname annotation replaces the +// IP in the status Kubernetes publishes, while the rules are still created on the IP. +func TestEnsureLoadBalancerReportsHostnameAnnotation(t *testing.T) { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + http := corev1.ServicePort{Name: "http", Port: 80, NodePort: 30000, Protocol: corev1.ProtocolTCP} + env := newEnsureLBTestEnv(ctrl, map[string]string{ServiceAnnotationLoadBalancerLoadbalancerHostname: "lb.example.com"}, []corev1.ServicePort{http}) + env.expectHosts() + env.expectRules() + env.network.EXPECT().GetNetworkByID("net-1", gomock.Any()).Return(virtualRouterNetwork(), 1, nil).Times(2) + env.address.EXPECT().NewAssociateIpAddressParams().Return(&cloudstack.AssociateIpAddressParams{}) + env.address.EXPECT().AssociateIpAddress(gomock.Any()).Return(&cloudstack.AssociateIpAddressResponse{Id: "ip-new", Ipaddress: "10.0.0.9"}, nil) + env.lb.EXPECT().NewCreateLoadBalancerRuleParams("roundrobin", "atestuid-tcp-80", 30000, 80).Return(&cloudstack.CreateLoadBalancerRuleParams{}) + env.lb.EXPECT().CreateLoadBalancerRule(gomock.Any()).Return(&cloudstack.CreateLoadBalancerRuleResponse{ + Id: "rule-new", Name: "atestuid-tcp-80", Publicip: "10.0.0.9", Publicipid: "ip-new", Protocol: "tcp", + }, nil) + env.lb.EXPECT().NewAssignToLoadBalancerRuleParams("rule-new").Return(&cloudstack.AssignToLoadBalancerRuleParams{}) + env.lb.EXPECT().AssignToLoadBalancerRule(gomock.Any()).Return(&cloudstack.AssignToLoadBalancerRuleResponse{}, nil) + env.expectFirewallListings(&cloudstack.ListFirewallRulesResponse{}, 1) + env.firewall.EXPECT().NewCreateFirewallRuleParams("ip-new", "tcp").Return(&cloudstack.CreateFirewallRuleParams{}) + env.firewall.EXPECT().CreateFirewallRule(gomock.Any()).Return(&cloudstack.CreateFirewallRuleResponse{}, nil) + + status, err := env.cs.EnsureLoadBalancer(context.TODO(), testClusterName, env.service, env.nodes) + + want := &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{Hostname: "lb.example.com"}}} + if err != nil || !reflect.DeepEqual(status, want) { + t.Errorf("EnsureLoadBalancer() = %v, %v, want %v", status, err, want) + } +} diff --git a/cloudstack_node_lookup_test.go b/cloudstack_node_lookup_test.go new file mode 100644 index 00000000..f6aff4f1 --- /dev/null +++ b/cloudstack_node_lookup_test.go @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cloudstack + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + cloudprovider "k8s.io/cloud-provider" +) + +const testProjectID = "6a3c1b9e-4f3d-4b43-9a6a-0c3d1f2e7b11" + +type projectRecorder struct{ projectID string } + +func (r *projectRecorder) SetProjectid(id string) { r.projectID = id } + +// appliedProjectID returns the project id the lookup options would put on a request. +func appliedProjectID(t *testing.T, opts []cloudstack.OptionFunc) string { + t.Helper() + recorder := &projectRecorder{} + for _, opt := range opts { + if err := opt(nil, recorder); err != nil { + t.Fatalf("applying lookup option: %v", err) + } + } + return recorder.projectID +} + +type vmLookupEnv struct { + cs *CSCloud + vm *cloudstack.MockVirtualMachineServiceIface +} + +func newVMLookupEnv(t *testing.T, projectID string) *vmLookupEnv { + t.Helper() + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + + vm := cloudstack.NewMockVirtualMachineServiceIface(ctrl) + return &vmLookupEnv{ + cs: &CSCloud{ + client: &cloudstack.CloudStackClient{VirtualMachine: vm}, + projectID: projectID, + }, + vm: vm, + } +} + +// expectByName expects one lookup of the instance by name, scoped to the CSCloud's project. +func (e *vmLookupEnv) expectByName(t *testing.T, name string, result *cloudstack.VirtualMachine, count int, err error) *gomock.Call { + t.Helper() + return e.vm.EXPECT().GetVirtualMachineByName(gomock.Any(), gomock.Any()). + DoAndReturn(func(got string, opts ...cloudstack.OptionFunc) (*cloudstack.VirtualMachine, int, error) { + if got != name { + t.Errorf("GetVirtualMachineByName name = %q, want %q", got, name) + } + if gotProject := appliedProjectID(t, opts); gotProject != e.cs.projectID { + t.Errorf("GetVirtualMachineByName project = %q, want %q", gotProject, e.cs.projectID) + } + return result, count, err + }) +} + +// expectByID expects one lookup of the instance by ID, scoped to the CSCloud's project. +func (e *vmLookupEnv) expectByID(t *testing.T, id string, result *cloudstack.VirtualMachine, count int, err error) *gomock.Call { + t.Helper() + return e.vm.EXPECT().GetVirtualMachineByID(gomock.Any(), gomock.Any()). + DoAndReturn(func(got string, opts ...cloudstack.OptionFunc) (*cloudstack.VirtualMachine, int, error) { + if got != id { + t.Errorf("GetVirtualMachineByID id = %q, want %q", got, id) + } + if gotProject := appliedProjectID(t, opts); gotProject != e.cs.projectID { + t.Errorf("GetVirtualMachineByID project = %q, want %q", gotProject, e.cs.projectID) + } + return result, count, err + }) +} + +func testInstance() *cloudstack.VirtualMachine { + return &cloudstack.VirtualMachine{ + Id: "vm-1", + Name: "node-1", + Zonename: "zone-a", + Serviceofferingname: "Small Instance", + Nic: []cloudstack.Nic{{Ipaddress: "10.0.0.1"}}, + } +} + +type instanceLookupOp struct { + name string + call func(cs *CSCloud, key string) (any, error) + want any + notFound any + notFoundErr error + errPrefix string +} + +var instanceLookupOps = []instanceLookupOp{ + { + name: "NodeAddresses", + call: func(cs *CSCloud, key string) (any, error) { + return cs.NodeAddresses(context.TODO(), types.NodeName(key)) + }, + want: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "10.0.0.1"}}, + notFound: []corev1.NodeAddress(nil), + notFoundErr: cloudprovider.InstanceNotFound, + errPrefix: "error retrieving node addresses", + }, + { + name: "InstanceID", + call: func(cs *CSCloud, key string) (any, error) { + return cs.InstanceID(context.TODO(), types.NodeName(key)) + }, + want: "vm-1", + notFound: "", + notFoundErr: cloudprovider.InstanceNotFound, + errPrefix: "error retrieving instance ID", + }, + { + name: "InstanceType", + call: func(cs *CSCloud, key string) (any, error) { + return cs.InstanceType(context.TODO(), types.NodeName(key)) + }, + want: "SmallInstance", + notFound: "", + notFoundErr: cloudprovider.InstanceNotFound, + errPrefix: "error retrieving instance type", + }, +} + +var providerIDLookupOps = []instanceLookupOp{ + { + name: "NodeAddressesByProviderID", + call: func(cs *CSCloud, key string) (any, error) { + return cs.NodeAddressesByProviderID(context.TODO(), key) + }, + want: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "10.0.0.1"}}, + notFound: []corev1.NodeAddress(nil), + notFoundErr: cloudprovider.InstanceNotFound, + errPrefix: "error retrieving node addresses", + }, + { + name: "InstanceTypeByProviderID", + call: func(cs *CSCloud, key string) (any, error) { + return cs.InstanceTypeByProviderID(context.TODO(), key) + }, + want: "SmallInstance", + notFound: "", + notFoundErr: cloudprovider.InstanceNotFound, + errPrefix: "error retrieving instance type", + }, + { + name: "InstanceExistsByProviderID", + call: func(cs *CSCloud, key string) (any, error) { + return cs.InstanceExistsByProviderID(context.TODO(), key) + }, + want: true, + notFound: false, + errPrefix: "error retrieving instance", + }, +} + +type lookupScenario struct { + name string + projectID string + result *cloudstack.VirtualMachine + count int + err error +} + +func lookupScenarios() []lookupScenario { + return []lookupScenario{ + {name: "found", result: testInstance(), count: 1}, + {name: "found in a project", projectID: testProjectID, result: testInstance(), count: 1}, + {name: "not found", count: 0, err: errors.New("no match")}, + {name: "api error", count: 1, err: errors.New("boom")}, + } +} + +func assertLookupOutcome(t *testing.T, op instanceLookupOp, sc lookupScenario, got any, err error) { + t.Helper() + switch { + case sc.err == nil: + if err != nil { + t.Fatalf("%s() unexpected error: %v", op.name, err) + } + if !reflect.DeepEqual(got, op.want) { + t.Errorf("%s() = %v, want %v", op.name, got, op.want) + } + case sc.count == 0: + if err != op.notFoundErr { + t.Errorf("%s() error = %v, want %v", op.name, err, op.notFoundErr) + } + if !reflect.DeepEqual(got, op.notFound) { + t.Errorf("%s() = %v, want %v", op.name, got, op.notFound) + } + default: + if err == nil || !strings.HasPrefix(err.Error(), op.errPrefix) || !strings.Contains(err.Error(), sc.err.Error()) { + t.Errorf("%s() error = %v, want prefix %q wrapping %q", op.name, err, op.errPrefix, sc.err) + } + } +} + +// TestInstanceLookupsByName checks that every node-name lookup asks CloudStack for that name in +// the configured project and maps a miss and an API failure to the errors the node controller expects. +func TestInstanceLookupsByName(t *testing.T) { + for _, op := range instanceLookupOps { + for _, sc := range lookupScenarios() { + t.Run(op.name+"/"+sc.name, func(t *testing.T) { + env := newVMLookupEnv(t, sc.projectID) + env.expectByName(t, "node-1", sc.result, sc.count, sc.err) + + got, err := op.call(env.cs, "node-1") + + assertLookupOutcome(t, op, sc, got, err) + }) + } + } +} + +// TestInstanceLookupsByProviderID checks that every provider ID lookup strips the scheme before +// asking CloudStack for the instance and accepts a bare instance ID as well. +func TestInstanceLookupsByProviderID(t *testing.T) { + providerIDs := map[string]string{ + "with scheme": "external-cloudstack://vm-1", + "bare id": "vm-1", + } + + for _, op := range providerIDLookupOps { + for _, sc := range lookupScenarios() { + for idName, providerID := range providerIDs { + t.Run(op.name+"/"+sc.name+"/"+idName, func(t *testing.T) { + env := newVMLookupEnv(t, sc.projectID) + env.expectByID(t, "vm-1", sc.result, sc.count, sc.err) + + got, err := op.call(env.cs, providerID) + + assertLookupOutcome(t, op, sc, got, err) + }) + } + } + } +} + +// TestInstanceTypeStripsInvalidLabelCharacters checks that the service offering name is turned +// into a value that is valid in a node label. +func TestInstanceTypeStripsInvalidLabelCharacters(t *testing.T) { + tests := map[string]string{ + "Small Instance": "SmallInstance", + "Medium": "Medium", + "Big (x2)": "Bigx2", + } + + for offering, want := range tests { + t.Run(offering, func(t *testing.T) { + env := newVMLookupEnv(t, "") + env.expectByName(t, "node-1", &cloudstack.VirtualMachine{Serviceofferingname: offering}, 1, nil) + + got, err := env.cs.InstanceType(context.TODO(), "node-1") + + if err != nil || got != want { + t.Errorf("InstanceType() = %q, %v, want %q", got, err, want) + } + }) + } +} + +// TestInstanceExists checks that the node is resolved to its ID by name before the existence check. +func TestInstanceExists(t *testing.T) { + tests := []struct { + name string + byName func(e *vmLookupEnv, t *testing.T) + byID func(e *vmLookupEnv, t *testing.T) + want bool + wantErr error + errContains string + }{ + { + name: "instance exists", + byName: func(e *vmLookupEnv, t *testing.T) { e.expectByName(t, "node-1", testInstance(), 1, nil) }, + byID: func(e *vmLookupEnv, t *testing.T) { e.expectByID(t, "vm-1", testInstance(), 1, nil) }, + want: true, + }, + { + name: "node unknown to CloudStack", + byName: func(e *vmLookupEnv, t *testing.T) { e.expectByName(t, "node-1", nil, 0, errors.New("no match")) }, + wantErr: cloudprovider.InstanceNotFound, + }, + { + name: "instance deleted after the name lookup", + byName: func(e *vmLookupEnv, t *testing.T) { e.expectByName(t, "node-1", testInstance(), 1, nil) }, + byID: func(e *vmLookupEnv, t *testing.T) { e.expectByID(t, "vm-1", nil, 0, errors.New("no match")) }, + want: false, + }, + { + name: "api error on the ID lookup", + byName: func(e *vmLookupEnv, t *testing.T) { e.expectByName(t, "node-1", testInstance(), 1, nil) }, + byID: func(e *vmLookupEnv, t *testing.T) { e.expectByID(t, "vm-1", nil, 1, errors.New("boom")) }, + errContains: "error retrieving instance: boom", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := newVMLookupEnv(t, testProjectID) + tt.byName(env, t) + if tt.byID != nil { + tt.byID(env, t) + } + + got, err := env.cs.InstanceExists(context.TODO(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}) + + if got != tt.want { + t.Errorf("InstanceExists() = %v, want %v", got, tt.want) + } + if tt.wantErr != nil && err != tt.wantErr { + t.Errorf("InstanceExists() error = %v, want %v", err, tt.wantErr) + } + if tt.errContains != "" && (err == nil || !strings.Contains(err.Error(), tt.errContains)) { + t.Errorf("InstanceExists() error = %v, want it to contain %q", err, tt.errContains) + } + }) + } +} + +// TestInstanceMetadata checks the metadata the node controller sets on a new node and that a +// failure of any of the four lookups behind it is returned instead of a half-filled result. +func TestInstanceMetadata(t *testing.T) { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}} + lookups := []string{"InstanceID", "InstanceType", "NodeAddresses", "GetZoneByNodeName"} + + t.Run("all lookups succeed", func(t *testing.T) { + env := newVMLookupEnv(t, testProjectID) + env.cs.region = "region-1" + env.vm.EXPECT().GetVirtualMachineByName(gomock.Any(), gomock.Any()).Times(len(lookups)). + DoAndReturn(func(name string, opts ...cloudstack.OptionFunc) (*cloudstack.VirtualMachine, int, error) { + if name != "node-1" || appliedProjectID(t, opts) != testProjectID { + t.Errorf("lookup of %q in project %q, want node-1 in %q", name, appliedProjectID(t, opts), testProjectID) + } + return testInstance(), 1, nil + }) + + got, err := env.cs.InstanceMetadata(context.TODO(), node) + if err != nil { + t.Fatalf("InstanceMetadata() unexpected error: %v", err) + } + + want := &cloudprovider.InstanceMetadata{ + ProviderID: "external-cloudstack://vm-1", + InstanceType: "SmallInstance", + NodeAddresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "10.0.0.1"}}, + Zone: "zone-a", + Region: "region-1", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("InstanceMetadata() = %+v, want %+v", got, want) + } + }) + + for failAt, lookup := range lookups { + t.Run("failure in "+lookup, func(t *testing.T) { + env := newVMLookupEnv(t, "") + calls := 0 + env.vm.EXPECT().GetVirtualMachineByName(gomock.Any(), gomock.Any()).Times(failAt + 1). + DoAndReturn(func(string, ...cloudstack.OptionFunc) (*cloudstack.VirtualMachine, int, error) { + calls++ + if calls == failAt+1 { + return nil, 1, errors.New("boom") + } + return testInstance(), 1, nil + }) + + got, err := env.cs.InstanceMetadata(context.TODO(), node) + + if got != nil || err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("InstanceMetadata() = %+v, %v, want nil and an error containing boom", got, err) + } + }) + } +} + +// TestUnsupportedInstanceOperations checks the operations the provider leaves to Kubernetes. +func TestUnsupportedInstanceOperations(t *testing.T) { + cs := &CSCloud{} + node := &corev1.Node{} + + if shutdown, err := cs.InstanceShutdownByProviderID(context.TODO(), "vm-1"); shutdown || err != cloudprovider.NotImplemented { + t.Errorf("InstanceShutdownByProviderID() = %v, %v, want false, NotImplemented", shutdown, err) + } + if shutdown, err := cs.InstanceShutdown(context.TODO(), node); shutdown || err != cloudprovider.NotImplemented { + t.Errorf("InstanceShutdown() = %v, %v, want false, NotImplemented", shutdown, err) + } + if err := cs.AddSSHKeyToAllInstances(context.TODO(), "user", nil); err != cloudprovider.NotImplemented { + t.Errorf("AddSSHKeyToAllInstances() = %v, want NotImplemented", err) + } + if name, err := cs.CurrentNodeName(context.TODO(), "host-1"); err != nil || name != "host-1" { + t.Errorf("CurrentNodeName() = %q, %v, want host-1", name, err) + } +} diff --git a/cloudstack_zone_test.go b/cloudstack_zone_test.go new file mode 100644 index 00000000..77f5290d --- /dev/null +++ b/cloudstack_zone_test.go @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cloudstack + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/apache/cloudstack-go/v2/cloudstack" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + restclient "k8s.io/client-go/rest" + cloudprovider "k8s.io/cloud-provider" +) + +type fakeClientBuilder struct { + client kubernetes.Interface + err error +} + +func (b *fakeClientBuilder) Config(string) (*restclient.Config, error) { return nil, b.err } +func (b *fakeClientBuilder) ConfigOrDie(string) *restclient.Config { return nil } +func (b *fakeClientBuilder) Client(string) (kubernetes.Interface, error) { + return b.client, b.err +} +func (b *fakeClientBuilder) ClientOrDie(string) kubernetes.Interface { return b.client } + +var _ cloudprovider.ControllerClientBuilder = &fakeClientBuilder{} + +type zoneLookup struct { + name string + call func(cs *CSCloud) (cloudprovider.Zone, error) + // expect registers the one CloudStack lookup the call makes. + expect func(e *vmLookupEnv, t *testing.T, result *cloudstack.VirtualMachine, count int, err error) + // notFound is the text of the error for an instance CloudStack does not know. + notFound string +} + +var zoneLookups = []zoneLookup{ + { + name: "GetZoneByNodeName", + call: func(cs *CSCloud) (cloudprovider.Zone, error) { + return cs.GetZoneByNodeName(context.TODO(), "node-1") + }, + expect: func(e *vmLookupEnv, t *testing.T, result *cloudstack.VirtualMachine, count int, err error) { + e.expectByName(t, "node-1", result, count, err) + }, + notFound: "could not find node: node-1", + }, + { + name: "GetZoneByProviderID", + call: func(cs *CSCloud) (cloudprovider.Zone, error) { + return cs.GetZoneByProviderID(context.TODO(), "external-cloudstack://vm-1") + }, + expect: func(e *vmLookupEnv, t *testing.T, result *cloudstack.VirtualMachine, count int, err error) { + e.expectByID(t, "vm-1", result, count, err) + }, + notFound: "could not find node by ID: external-cloudstack://vm-1", + }, +} + +// TestZoneLookups checks that a zone is resolved from the instance's zone name in the configured +// project, that the region is the configured one or else the zone name, and that failures read +// differently for a missing instance and an API error. +func TestZoneLookups(t *testing.T) { + for _, lookup := range zoneLookups { + t.Run(lookup.name+"/region falls back to the zone", func(t *testing.T) { + env := newVMLookupEnv(t, "") + lookup.expect(env, t, testInstance(), 1, nil) + + zone, err := lookup.call(env.cs) + + if err != nil || zone.FailureDomain != "zone-a" || zone.Region != "zone-a" { + t.Errorf("%s() = %+v, %v, want zone-a in region zone-a", lookup.name, zone, err) + } + }) + + t.Run(lookup.name+"/configured region in a project", func(t *testing.T) { + env := newVMLookupEnv(t, testProjectID) + env.cs.region = "region-1" + lookup.expect(env, t, testInstance(), 1, nil) + + zone, err := lookup.call(env.cs) + + if err != nil || zone.FailureDomain != "zone-a" || zone.Region != "region-1" { + t.Errorf("%s() = %+v, %v, want zone-a in region region-1", lookup.name, zone, err) + } + }) + + t.Run(lookup.name+"/instance not found", func(t *testing.T) { + env := newVMLookupEnv(t, "") + lookup.expect(env, t, nil, 0, errors.New("no match")) + + _, err := lookup.call(env.cs) + + if err == nil || err.Error() != lookup.notFound { + t.Errorf("%s() error = %v, want %q", lookup.name, err, lookup.notFound) + } + }) + + t.Run(lookup.name+"/api error", func(t *testing.T) { + env := newVMLookupEnv(t, "") + lookup.expect(env, t, nil, 1, errors.New("boom")) + + _, err := lookup.call(env.cs) + + if err == nil || err.Error() != "error retrieving zone: boom" { + t.Errorf("%s() error = %v, want %q", lookup.name, err, "error retrieving zone: boom") + } + }) + } +} + +func podNamed(name, namespace, nodeName string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: corev1.PodSpec{NodeName: nodeName}, + } +} + +func setPodEnvironment(t *testing.T, podName, namespace string) { + t.Helper() + t.Setenv("HOSTNAME", podName) + t.Setenv("POD_NAMESPACE", namespace) +} + +// TestGetNodeNameFromPod checks that the node is read from the CCM's own pod and that every way +// that can fail is reported. +func TestGetNodeNameFromPod(t *testing.T) { + setPodEnvironment(t, "ccm-pod", "ccm-ns") + + tests := []struct { + name string + builder *fakeClientBuilder + want string + errPrefix string + }{ + { + name: "no client builder", + errPrefix: "clientBuilder not initialized", + }, + { + name: "client cannot be built", + builder: &fakeClientBuilder{err: errors.New("no config")}, + errPrefix: "failed to get Kubernetes client: no config", + }, + { + name: "pod does not exist", + builder: &fakeClientBuilder{client: fake.NewSimpleClientset()}, + errPrefix: "failed to get pod ccm-ns/ccm-pod from Kubernetes API", + }, + { + name: "pod not scheduled yet", + builder: &fakeClientBuilder{client: fake.NewSimpleClientset(podNamed("ccm-pod", "ccm-ns", ""))}, + errPrefix: "pod ccm-ns/ccm-pod does not have a nodeName assigned yet", + }, + { + name: "pod scheduled", + builder: &fakeClientBuilder{client: fake.NewSimpleClientset(podNamed("ccm-pod", "ccm-ns", "node-1"))}, + want: "node-1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs := &CSCloud{} + if tt.builder != nil { + cs.clientBuilder = tt.builder + } + + got, err := cs.getNodeNameFromPod(context.TODO()) + + if tt.errPrefix != "" { + if err == nil || !strings.HasPrefix(err.Error(), tt.errPrefix) { + t.Errorf("getNodeNameFromPod() error = %v, want prefix %q", err, tt.errPrefix) + } + return + } + if err != nil || got != tt.want { + t.Errorf("getNodeNameFromPod() = %q, %v, want %q", got, err, tt.want) + } + }) + } +} + +// TestGetZone checks that the zone is looked up once through the pod's node and then served from the cache. +func TestGetZone(t *testing.T) { + t.Run("cached zone makes no lookup", func(t *testing.T) { + env := newVMLookupEnv(t, "") + env.cs.zone = "zone-a" + env.cs.region = "region-1" + + zone, err := env.cs.GetZone(context.TODO()) + + if err != nil || zone.FailureDomain != "zone-a" || zone.Region != "region-1" { + t.Errorf("GetZone() = %+v, %v, want zone-a in region region-1", zone, err) + } + }) + + t.Run("zone of the pod's node is looked up once", func(t *testing.T) { + setPodEnvironment(t, "ccm-pod", "ccm-ns") + env := newVMLookupEnv(t, testProjectID) + env.cs.clientBuilder = &fakeClientBuilder{client: fake.NewSimpleClientset(podNamed("ccm-pod", "ccm-ns", "node-1"))} + env.expectByName(t, "node-1", testInstance(), 1, nil).Times(1) + + for i := 0; i < 2; i++ { + zone, err := env.cs.GetZone(context.TODO()) + if err != nil || zone.FailureDomain != "zone-a" || zone.Region != "zone-a" { + t.Fatalf("GetZone() call %d = %+v, %v, want zone-a in region zone-a", i+1, zone, err) + } + } + }) + + failures := []struct { + name string + builder cloudprovider.ControllerClientBuilder + result *cloudstack.VirtualMachine + count int + err error + errPrefix string + }{ + { + name: "node of the pod unknown", + errPrefix: "failed to get node name for retrieving the zone", + }, + { + name: "instance not found", + builder: &fakeClientBuilder{client: fake.NewSimpleClientset(podNamed("ccm-pod", "ccm-ns", "node-1"))}, + count: 0, + err: errors.New("no match"), + errPrefix: "could not find CloudStack instance with name node-1 for retrieving the zone", + }, + { + name: "api error", + builder: &fakeClientBuilder{client: fake.NewSimpleClientset(podNamed("ccm-pod", "ccm-ns", "node-1"))}, + count: 1, + err: errors.New("boom"), + errPrefix: "error getting instance for retrieving the zone", + }, + } + + for _, tt := range failures { + t.Run(tt.name, func(t *testing.T) { + setPodEnvironment(t, "ccm-pod", "ccm-ns") + env := newVMLookupEnv(t, "") + env.cs.clientBuilder = tt.builder + if tt.err != nil { + env.expectByName(t, "node-1", tt.result, tt.count, tt.err) + } + + _, err := env.cs.GetZone(context.TODO()) + + if err == nil || !strings.HasPrefix(err.Error(), tt.errPrefix) { + t.Errorf("GetZone() error = %v, want prefix %q", err, tt.errPrefix) + } + }) + } +}