From 65e4f64e973d6062667c0abbf7cee841c889abb5 Mon Sep 17 00:00:00 2001 From: Vinicius Caldo Silva Date: Mon, 31 Aug 2026 21:35:53 +0200 Subject: [PATCH 1/4] [feature] add support for reserved IPv4 range in NodeBalancer backend allocation --- Makefile | 2 +- cloud/linode/cloud.go | 8 + cloud/linode/fake_linode_test.go | 46 +++++ cloud/linode/loadbalancers.go | 64 +++++- cloud/linode/loadbalancers_test.go | 13 ++ cloud/linode/nodebalancer_backend_range.go | 153 ++++++++++++++ .../linode/nodebalancer_backend_range_test.go | 193 ++++++++++++++++++ cloud/linode/options/options.go | 39 ++-- deploy/chart/templates/daemonset.yaml | 11 +- deploy/chart/values.yaml | 8 + hack/test-helm-networking.sh | 53 +++++ main.go | 1 + 12 files changed, 562 insertions(+), 29 deletions(-) create mode 100644 cloud/linode/nodebalancer_backend_range.go create mode 100644 cloud/linode/nodebalancer_backend_range_test.go create mode 100755 hack/test-helm-networking.sh diff --git a/Makefile b/Makefile index bbb01e9d..bce02949 100644 --- a/Makefile +++ b/Makefile @@ -271,6 +271,7 @@ helm-template: #Verify template works when region and apiToken are passed, and when it is passed as reference. @helm template foo deploy/chart --set apiToken="apiToken",region="us-east" > /dev/null @helm template foo deploy/chart --set secretRef.apiTokenRef="apiToken",secretRef.name="api",secretRef.regionRef="us-east" > /dev/null + @bash hack/test-helm-networking.sh .PHONY: serve-docs serve-docs: @@ -287,4 +288,3 @@ serve-docs: build-docs: # Build the documentation site the way GitHub Pages does docker run --rm --volume "$(shell pwd):/srv/jekyll" $(DOCS_IMAGE) jekyll build - diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 4ad18e0f..6803b8a7 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -213,6 +213,14 @@ func setupNodeBalancerBackendSubnet(linodeClient client.Client) error { if options.Options.NodeBalancerBackendIPv4SubnetID != 0 && options.Options.NodeBalancerBackendIPv4SubnetName != "" { return fmt.Errorf("cannot have both --nodebalancer-backend-ipv4-subnet-id and --nodebalancer-backend-ipv4-subnet-name set") } + if options.Options.NodeBalancerBackendIPv4ReservedRange != "" { + if err := validateNodeBalancerBackendIPv4Reservation( + options.Options.NodeBalancerBackendIPv4Subnet, + options.Options.NodeBalancerBackendIPv4ReservedRange, + ); err != nil { + return err + } + } switch { case options.Options.DisableNodeBalancerVPCBackends: diff --git a/cloud/linode/fake_linode_test.go b/cloud/linode/fake_linode_test.go index 737c629f..2863d390 100644 --- a/cloud/linode/fake_linode_test.go +++ b/cloud/linode/fake_linode_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" "github.com/linode/linodego/v2" ) @@ -43,6 +44,10 @@ type fakeRequest struct { func newFake(t *testing.T) *fakeAPI { t.Helper() + services.Mu.Lock() + services.VpcIDs = make(map[string]int) + services.SubnetIDs = make(map[string]int) + services.Mu.Unlock() fake := &fakeAPI{ t: t, @@ -127,6 +132,24 @@ func (f *fakeAPI) setupRoutes() { _, _ = w.Write(rr) }) + f.mux.HandleFunc("GET /v4/vpcs/{vpcId}/subnets/{subnetId}", func(w http.ResponseWriter, r *http.Request) { + subnetID, err := strconv.Atoi(r.PathValue("subnetId")) + if err != nil { + f.t.Fatal(err) + } + subnet, ok := f.subnet[subnetID] + if !ok { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":[{"reason":"Not Found"}]}`)) + return + } + resp, err := json.Marshal(subnet) + if err != nil { + f.t.Fatal(err) + } + _, _ = w.Write(resp) + }) + f.mux.HandleFunc("GET /v4/vpcs", func(w http.ResponseWriter, r *http.Request) { res := 0 data := []linodego.VPC{} @@ -367,6 +390,20 @@ func (f *fakeAPI) setupRoutes() { } f.nb[strconv.Itoa(nb.ID)] = &nb + for _, backendVPC := range nbco.BackendVPCs { + if backendVPC.IPv4Range == "" { + continue + } + subnet, ok := f.subnet[backendVPC.SubnetID] + if !ok { + f.t.Fatalf("subnet %d not found", backendVPC.SubnetID) + } + subnet.Nodebalancers = append(subnet.Nodebalancers, linodego.VPCSubnetNodebalancers{ + ID: nb.ID, + Ipv4Range: backendVPC.IPv4Range, + }) + } + for _, nbcco := range nbco.Configs { if nbcco.Protocol == "https" { if !strings.Contains(nbcco.SSLCert, "BEGIN CERTIFICATE") { @@ -649,6 +686,15 @@ func (f *fakeAPI) setupRoutes() { delete(f.nbn, k) } } + for _, subnet := range f.subnet { + nodebalancers := subnet.Nodebalancers[:0] + for _, nodeBalancer := range subnet.Nodebalancers { + if nodeBalancer.ID != nid { + nodebalancers = append(nodebalancers, nodeBalancer) + } + } + subnet.Nodebalancers = nodebalancers + } }) f.mux.HandleFunc("DELETE /v4/nodebalancers/{nodeBalancerId}/configs/{configId}/nodes/{nodeId}", func(w http.ResponseWriter, r *http.Request) { diff --git a/cloud/linode/loadbalancers.go b/cloud/linode/loadbalancers.go index 852c4e4b..0c6c6098 100644 --- a/cloud/linode/loadbalancers.go +++ b/cloud/linode/loadbalancers.go @@ -790,6 +790,31 @@ func (l *loadbalancers) getVPCCreateOptions(ctx context.Context, service *v1.Ser } } + if options.Options.NodeBalancerBackendIPv4ReservedRange != "" { + vpcID, err := l.getVPCIDForSVC(ctx, service) + if err != nil { + return nil, fmt.Errorf("failed to resolve VPC for NodeBalancer backend allocation: %w", err) + } + subnet, err := l.client.GetVPCSubnet(ctx, vpcID, subnetID) + if err != nil { + return nil, fmt.Errorf("failed to get VPC subnet %d for NodeBalancer backend allocation: %w", subnetID, err) + } + backendIPv4Range, err := allocateNodeBalancerBackendIPv4Range( + options.Options.NodeBalancerBackendIPv4Subnet, + options.Options.NodeBalancerBackendIPv4ReservedRange, + subnet, + ) + if err != nil { + return nil, err + } + return []linodego.NodeBalancerBackendVPCOptions{ + { + SubnetID: subnetID, + IPv4Range: backendIPv4Range, + }, + }, nil + } + // Precedence 2: If the user wants to overwrite the default VPC name or subnet name // and have specified it in the annotations, use it to set subnetID // and auto-allocate subnets from it for the NodeBalancer @@ -816,7 +841,7 @@ func (l *loadbalancers) getVPCCreateOptions(ctx context.Context, service *v1.Ser } // Precedence 4: If the user has specified a NodeBalancerBackendIPv4Subnet, use that - // and auto-allocate subnets from it for the NodeBalancer + // and auto-allocate subnets from it for the NodeBalancer. if options.Options.NodeBalancerBackendIPv4Subnet != "" { vpcCreateOpts := []linodego.NodeBalancerBackendVPCOptions{ { @@ -1120,7 +1145,7 @@ func (l *loadbalancers) getSubnetIDForSVC(ctx context.Context, service *v1.Servi return subnetID, nil } - specifiedVPCName, vpcOk := service.GetAnnotations()[annotations.NodeBalancerBackendVPCName] + _, vpcOk := service.GetAnnotations()[annotations.NodeBalancerBackendVPCName] specifiedSubnetName, subnetOk := service.GetAnnotations()[annotations.NodeBalancerBackendSubnetName] // If no VPCName or SubnetName is specified in annotations, but NodeBalancerBackendIPv4SubnetID is set, @@ -1129,11 +1154,7 @@ func (l *loadbalancers) getSubnetIDForSVC(ctx context.Context, service *v1.Servi return options.Options.NodeBalancerBackendIPv4SubnetID, nil } - vpcName := options.Options.VPCNames[0] - if vpcOk { - vpcName = specifiedVPCName - } - vpcID, err := services.GetVPCID(ctx, l.client, vpcName) + vpcID, err := l.getVPCIDForSVC(ctx, service) if err != nil { return 0, err } @@ -1147,6 +1168,18 @@ func (l *loadbalancers) getSubnetIDForSVC(ctx context.Context, service *v1.Servi return services.GetSubnetID(ctx, l.client, vpcID, subnetName) } +func (l *loadbalancers) getVPCIDForSVC(ctx context.Context, service *v1.Service) (int, error) { + if len(options.Options.VPCNames) == 0 { + return 0, fmt.Errorf("CCM not configured with VPC, cannot create NodeBalancer with specified annotation") + } + + vpcName := options.Options.VPCNames[0] + if specifiedVPCName, ok := service.GetAnnotations()[annotations.NodeBalancerBackendVPCName]; ok { + vpcName = specifiedVPCName + } + return services.GetVPCID(ctx, l.client, vpcName) +} + // buildLoadBalancerRequest returns a linodego.NodeBalancer // requests for service across nodes. func (l *loadbalancers) buildLoadBalancerRequest(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*linodego.NodeBalancer, error) { @@ -1584,6 +1617,19 @@ func validateNodeBalancerBackendIPv4Range(backendIPv4Range string) error { if !withinCIDR { return fmt.Errorf("IPv4 range %s is not within the subnet %s", backendIPv4Range, options.Options.NodeBalancerBackendIPv4Subnet) } + if options.Options.NodeBalancerBackendIPv4ReservedRange != "" { + reserved, err := parseIPv4Prefix(options.Options.NodeBalancerBackendIPv4ReservedRange) + if err != nil { + return fmt.Errorf("invalid reserved NodeBalancer backend range: %w", err) + } + backend, err := parseIPv4Prefix(backendIPv4Range) + if err != nil { + return fmt.Errorf("invalid IPv4 range: %w", err) + } + if prefixesOverlap(backend, reserved) { + return fmt.Errorf("IPv4 range %s overlaps the reserved NodeBalancer backend range %s", backend, reserved) + } + } return nil } @@ -1608,5 +1654,7 @@ func isCIDRWithinCIDR(outer, inner string) (bool, error) { if err != nil { return false, fmt.Errorf("invalid CIDR: %w", err) } - return ipNet1.Contains(ipNet2.IP), nil + outerOnes, outerBits := ipNet1.Mask.Size() + innerOnes, innerBits := ipNet2.Mask.Size() + return outerBits == innerBits && outerOnes <= innerOnes && ipNet1.Contains(ipNet2.IP), nil } diff --git a/cloud/linode/loadbalancers_test.go b/cloud/linode/loadbalancers_test.go index d5f8f717..da8e840a 100644 --- a/cloud/linode/loadbalancers_test.go +++ b/cloud/linode/loadbalancers_test.go @@ -5881,13 +5881,26 @@ func Test_validateNodeBalancerBackendIPv4Range(t *testing.T) { args: args{backendIPv4Range: "10.100.0.0"}, wantErr: true, }, + { + name: "Reserved IPv4 range", + args: args{backendIPv4Range: "10.100.0.252/30"}, + wantErr: true, + }, + { + name: "Range extends outside backend subnet", + args: args{backendIPv4Range: "10.100.0.0/23"}, + wantErr: true, + }, } nbBackendSubnet := options.Options.NodeBalancerBackendIPv4Subnet + nbBackendReservedRange := options.Options.NodeBalancerBackendIPv4ReservedRange defer func() { options.Options.NodeBalancerBackendIPv4Subnet = nbBackendSubnet + options.Options.NodeBalancerBackendIPv4ReservedRange = nbBackendReservedRange }() options.Options.NodeBalancerBackendIPv4Subnet = "10.100.0.0/24" + options.Options.NodeBalancerBackendIPv4ReservedRange = "10.100.0.252/30" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cloud/linode/nodebalancer_backend_range.go b/cloud/linode/nodebalancer_backend_range.go new file mode 100644 index 00000000..255bd9b9 --- /dev/null +++ b/cloud/linode/nodebalancer_backend_range.go @@ -0,0 +1,153 @@ +package linode + +import ( + "encoding/binary" + "fmt" + "net/netip" + + "github.com/linode/linodego/v2" +) + +const nodeBalancerBackendRangePrefix = 30 + +func allocateNodeBalancerBackendIPv4Range(backendCIDR, reservedCIDR string, subnet *linodego.VPCSubnet) (string, error) { + if subnet == nil { + return "", fmt.Errorf("cannot allocate NodeBalancer backend range from a nil VPC subnet") + } + + if err := validateNodeBalancerBackendIPv4Reservation(backendCIDR, reservedCIDR); err != nil { + return "", err + } + + backend, err := parseIPv4Prefix(backendCIDR) + if err != nil { + return "", fmt.Errorf("invalid NodeBalancer backend CIDR: %w", err) + } + reserved, err := parseIPv4Prefix(reservedCIDR) + if err != nil { + return "", fmt.Errorf("invalid reserved NodeBalancer backend range: %w", err) + } + + subnetPrefix, err := parseIPv4Prefix(subnet.IPv4) + if err != nil { + return "", fmt.Errorf("invalid VPC subnet CIDR: %w", err) + } + if !prefixContains(subnetPrefix, backend) { + return "", fmt.Errorf("NodeBalancer backend CIDR %s is not within VPC subnet %s", backend, subnetPrefix) + } + + rangeCount := uint64(1) << uint(nodeBalancerBackendRangePrefix-backend.Bits()) + + assigned := make([]netip.Prefix, 0, len(subnet.Nodebalancers)) + for _, nodeBalancer := range subnet.Nodebalancers { + if nodeBalancer.Ipv4Range == "" { + continue + } + assignedRange, err := parseIPv4Prefix(nodeBalancer.Ipv4Range) + if err != nil { + return "", fmt.Errorf("invalid IPv4 range %q on NodeBalancer %d: %w", nodeBalancer.Ipv4Range, nodeBalancer.ID, err) + } + if prefixesOverlap(assignedRange, reserved) { + return "", fmt.Errorf("reserved NodeBalancer backend range %s is already assigned to NodeBalancer %d", reserved, nodeBalancer.ID) + } + if prefixesOverlap(assignedRange, backend) { + assigned = append(assigned, assignedRange) + } + } + + for index := uint64(0); index < rangeCount-1; index++ { + candidate, err := ipv4SubprefixAt(backend, nodeBalancerBackendRangePrefix, index) + if err != nil { + return "", err + } + if !overlapsAnyPrefix(candidate, assigned) { + return candidate.String(), nil + } + } + + return "", fmt.Errorf("NodeBalancer backend CIDR %s has no available /30 ranges", backend) +} + +func validateNodeBalancerBackendIPv4Reservation(backendCIDR, reservedCIDR string) error { + if backendCIDR == "" { + return fmt.Errorf("NodeBalancer backend IPv4 subnet is required when a reserved range is configured") + } + + reserved, err := parseIPv4Prefix(reservedCIDR) + if err != nil { + return fmt.Errorf("invalid reserved NodeBalancer backend range: %w", err) + } + if reserved.Bits() != nodeBalancerBackendRangePrefix { + return fmt.Errorf("reserved NodeBalancer backend range %s must be a /30", reserved) + } + + expected, err := highestNodeBalancerBackendIPv4Range(backendCIDR) + if err != nil { + return fmt.Errorf("invalid NodeBalancer backend subnet: %w", err) + } + if reserved != expected { + return fmt.Errorf("reserved NodeBalancer backend range %s must be the highest /30 in %s (%s)", reserved, backendCIDR, expected) + } + return nil +} + +func highestNodeBalancerBackendIPv4Range(backendCIDR string) (netip.Prefix, error) { + backend, err := parseIPv4Prefix(backendCIDR) + if err != nil { + return netip.Prefix{}, err + } + if backend.Bits() >= nodeBalancerBackendRangePrefix { + return netip.Prefix{}, fmt.Errorf("NodeBalancer backend CIDR %s has no allocable /30 after reserving its highest /30", backend) + } + + rangeCount := uint64(1) << uint(nodeBalancerBackendRangePrefix-backend.Bits()) + return ipv4SubprefixAt(backend, nodeBalancerBackendRangePrefix, rangeCount-1) +} + +func parseIPv4Prefix(value string) (netip.Prefix, error) { + prefix, err := netip.ParsePrefix(value) + if err != nil { + return netip.Prefix{}, err + } + if !prefix.Addr().Is4() { + return netip.Prefix{}, fmt.Errorf("%s is not an IPv4 prefix", value) + } + return prefix.Masked(), nil +} + +func ipv4SubprefixAt(parent netip.Prefix, childBits int, index uint64) (netip.Prefix, error) { + if !parent.Addr().Is4() || childBits < parent.Bits() || childBits > 32 { + return netip.Prefix{}, fmt.Errorf("cannot select /%d from prefix %s", childBits, parent) + } + + childCount := uint64(1) << uint(childBits-parent.Bits()) + if index >= childCount { + return netip.Prefix{}, fmt.Errorf("subprefix index %d is outside prefix %s", index, parent) + } + + baseBytes := parent.Masked().Addr().As4() + base := uint64(binary.BigEndian.Uint32(baseBytes[:])) + blockSize := uint64(1) << uint(32-childBits) + address := base + index*blockSize + var addressBytes [4]byte + binary.BigEndian.PutUint32(addressBytes[:], uint32(address)) + + return netip.PrefixFrom(netip.AddrFrom4(addressBytes), childBits), nil +} + +func prefixContains(outer, inner netip.Prefix) bool { + return outer.Bits() <= inner.Bits() && outer.Contains(inner.Addr()) +} + +func prefixesOverlap(first, second netip.Prefix) bool { + return first.Contains(second.Addr()) || second.Contains(first.Addr()) +} + +func overlapsAnyPrefix(candidate netip.Prefix, prefixes []netip.Prefix) bool { + for _, prefix := range prefixes { + if prefixesOverlap(candidate, prefix) { + return true + } + } + return false +} diff --git a/cloud/linode/nodebalancer_backend_range_test.go b/cloud/linode/nodebalancer_backend_range_test.go new file mode 100644 index 00000000..ae4eb0aa --- /dev/null +++ b/cloud/linode/nodebalancer_backend_range_test.go @@ -0,0 +1,193 @@ +package linode + +import ( + "context" + "testing" + + "github.com/golang/mock/gomock" + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/options" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" + "github.com/linode/linodego/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAllocateNodeBalancerBackendIPv4Range(t *testing.T) { + tests := []struct { + name string + backendCIDR string + subnet *linodego.VPCSubnet + want string + wantErr string + }{ + { + name: "selects the first available range", + backendCIDR: "10.63.88.0/21", + subnet: &linodego.VPCSubnet{ + IPv4: "10.63.80.0/20", + }, + want: "10.63.88.0/30", + }, + { + name: "skips assigned ranges", + backendCIDR: "10.63.88.0/21", + subnet: &linodego.VPCSubnet{ + IPv4: "10.63.80.0/20", + Nodebalancers: []linodego.VPCSubnetNodebalancers{ + {ID: 1, Ipv4Range: "10.63.88.0/30"}, + {ID: 2, Ipv4Range: "10.63.88.4/30"}, + }, + }, + want: "10.63.88.8/30", + }, + { + name: "supports non RFC1918 prefixes", + backendCIDR: "100.96.0.0/26", + subnet: &linodego.VPCSubnet{ + IPv4: "100.96.0.0/24", + }, + want: "100.96.0.0/30", + }, + { + name: "rejects a reserved range already in use", + backendCIDR: "10.63.88.0/21", + subnet: &linodego.VPCSubnet{ + IPv4: "10.63.80.0/20", + Nodebalancers: []linodego.VPCSubnetNodebalancers{ + {ID: 42, Ipv4Range: "10.63.95.252/30"}, + }, + }, + wantErr: "reserved NodeBalancer backend range 10.63.95.252/30 is already assigned to NodeBalancer 42", + }, + { + name: "rejects a range outside the VPC subnet", + backendCIDR: "10.64.0.0/21", + subnet: &linodego.VPCSubnet{ + IPv4: "10.63.80.0/20", + }, + wantErr: "NodeBalancer backend CIDR 10.64.0.0/21 is not within VPC subnet 10.63.80.0/20", + }, + { + name: "rejects a backend with only the reserved range", + backendCIDR: "10.63.95.252/30", + subnet: &linodego.VPCSubnet{ + IPv4: "10.63.80.0/20", + }, + wantErr: "NodeBalancer backend CIDR 10.63.95.252/30 has no allocable /30 after reserving its highest /30", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reserved, reservedErr := highestNodeBalancerBackendIPv4Range(tt.backendCIDR) + if reservedErr != nil { + if tt.wantErr != "" { + require.EqualError(t, reservedErr, tt.wantErr) + return + } + require.NoError(t, reservedErr) + } + got, err := allocateNodeBalancerBackendIPv4Range(tt.backendCIDR, reserved.String(), tt.subnet) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestHighestNodeBalancerBackendIPv4Range(t *testing.T) { + got, err := highestNodeBalancerBackendIPv4Range("10.63.88.0/21") + require.NoError(t, err) + assert.Equal(t, "10.63.95.252/30", got.String()) +} + +func TestValidateNodeBalancerBackendIPv4Reservation(t *testing.T) { + require.NoError(t, validateNodeBalancerBackendIPv4Reservation("10.63.88.0/21", "10.63.95.252/30")) + require.EqualError( + t, + validateNodeBalancerBackendIPv4Reservation("10.63.88.0/21", "10.63.95.248/30"), + "reserved NodeBalancer backend range 10.63.95.248/30 must be the highest /30 in 10.63.88.0/21 (10.63.95.252/30)", + ) +} + +func TestGetVPCCreateOptionsWithReservedBackendRange(t *testing.T) { + previousVPCNames := options.Options.VPCNames + previousSubnetNames := options.Options.SubnetNames + previousBackendSubnet := options.Options.NodeBalancerBackendIPv4Subnet + previousReservedRange := options.Options.NodeBalancerBackendIPv4ReservedRange + previousBackendSubnetID := options.Options.NodeBalancerBackendIPv4SubnetID + t.Cleanup(func() { + options.Options.VPCNames = previousVPCNames + options.Options.SubnetNames = previousSubnetNames + options.Options.NodeBalancerBackendIPv4Subnet = previousBackendSubnet + options.Options.NodeBalancerBackendIPv4ReservedRange = previousReservedRange + options.Options.NodeBalancerBackendIPv4SubnetID = previousBackendSubnetID + }) + + tests := []struct { + name string + annotations map[string]string + subnetID int + }{ + { + name: "default subnet", + }, + { + name: "service subnet override", + annotations: map[string]string{ + annotations.NodeBalancerBackendSubnetName: "test-subnet", + }, + }, + { + name: "configured subnet ID", + subnetID: 456, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + linodeClient := mocks.NewMockClient(ctrl) + + options.Options.VPCNames = []string{"test-vpc"} + options.Options.SubnetNames = []string{"test-subnet"} + options.Options.NodeBalancerBackendIPv4Subnet = "10.63.88.0/21" + options.Options.NodeBalancerBackendIPv4ReservedRange = "10.63.95.252/30" + options.Options.NodeBalancerBackendIPv4SubnetID = tt.subnetID + + services.Mu.Lock() + services.VpcIDs = map[string]int{"test-vpc": 123} + services.SubnetIDs = map[string]int{"test-subnet": 456} + services.Mu.Unlock() + + linodeClient.EXPECT(). + GetVPCSubnet(gomock.Any(), 123, 456). + Return(&linodego.VPCSubnet{ + ID: 456, + IPv4: "10.63.80.0/20", + Nodebalancers: []linodego.VPCSubnetNodebalancers{ + {ID: 1, Ipv4Range: "10.63.88.0/30"}, + }, + }, nil) + + l := &loadbalancers{client: linodeClient} + got, err := l.getVPCCreateOptions(context.Background(), &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: tt.annotations, + }, + }) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, 456, got[0].SubnetID) + assert.Equal(t, "10.63.88.4/30", got[0].IPv4Range) + assert.False(t, got[0].IPv4RangeAutoAssign) + }) + } +} diff --git a/cloud/linode/options/options.go b/cloud/linode/options/options.go index 047645e2..d76f57e7 100644 --- a/cloud/linode/options/options.go +++ b/cloud/linode/options/options.go @@ -20,23 +20,24 @@ var Options struct { SubnetIDs []int LoadBalancerType string // Deprecated no-op options retained so existing deployments continue to start. - BGPNodeSelector string - IpHolderSuffix string - LinodeExternalNetwork *net.IPNet - NodeBalancerTags []string - DefaultNBType string - NodeBalancerBackendIPv4Subnet string - NodeBalancerBackendIPv4SubnetID int - NodeBalancerBackendIPv4SubnetName string - DisableNodeBalancerVPCBackends bool - GlobalStopChannel chan<- struct{} - EnableIPv6ForLoadBalancers bool - EnableIPv6ForNodeBalancerBackends bool - AllocateNodeCIDRs bool - DisableIPv6NodeCIDRAllocation bool - ClusterCIDRIPv4 string - NodeCIDRMaskSizeIPv4 int - NodeCIDRMaskSizeIPv6 int - NodeBalancerPrefix string - LinodeTagFilter string + BGPNodeSelector string + IpHolderSuffix string + LinodeExternalNetwork *net.IPNet + NodeBalancerTags []string + DefaultNBType string + NodeBalancerBackendIPv4Subnet string + NodeBalancerBackendIPv4ReservedRange string + NodeBalancerBackendIPv4SubnetID int + NodeBalancerBackendIPv4SubnetName string + DisableNodeBalancerVPCBackends bool + GlobalStopChannel chan<- struct{} + EnableIPv6ForLoadBalancers bool + EnableIPv6ForNodeBalancerBackends bool + AllocateNodeCIDRs bool + DisableIPv6NodeCIDRAllocation bool + ClusterCIDRIPv4 string + NodeCIDRMaskSizeIPv4 int + NodeCIDRMaskSizeIPv6 int + NodeBalancerPrefix string + LinodeTagFilter string } diff --git a/deploy/chart/templates/daemonset.yaml b/deploy/chart/templates/daemonset.yaml index 2cca11c3..7d6b447f 100644 --- a/deploy/chart/templates/daemonset.yaml +++ b/deploy/chart/templates/daemonset.yaml @@ -135,10 +135,16 @@ spec: {{- if not $clusterCIDR }} {{- fail "clusterCIDR is required if route-controller is enabled" }} {{- end }} - - --configure-cloud-routes={{ default true .Values.routeController.configureCloudRoutes }} + {{- if hasKey .Values.routeController "configureCloudRoutes" }} + - --configure-cloud-routes={{ .Values.routeController.configureCloudRoutes }} + {{- else }} + - --configure-cloud-routes=true + {{- end }} {{- with .Values.routeController.routeReconciliationPeriod }} - --route-reconciliation-period={{ . }} {{- end }} + {{- else if hasKey .Values "configureCloudRoutes" }} + - --configure-cloud-routes={{ .Values.configureCloudRoutes }} {{- end }} {{- with $vpcNames }} - --vpc-names={{ . }} @@ -185,6 +191,9 @@ spec: {{- if .Values.nodeBalancerBackendIPv4Subnet }} - --nodebalancer-backend-ipv4-subnet={{ .Values.nodeBalancerBackendIPv4Subnet }} {{- end }} + {{- if .Values.nodeBalancerBackendIPv4ReservedRange }} + - --nodebalancer-backend-ipv4-reserved-range={{ .Values.nodeBalancerBackendIPv4ReservedRange }} + {{- end }} {{- if .Values.nodeBalancerPrefix }} - --nodebalancer-prefix={{ .Values.nodeBalancerPrefix }} {{- end }} diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 4b31dbba..8b2718da 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -94,6 +94,10 @@ tolerations: # clusterCIDR: 10.192.0.0/10 # configureCloudRoutes: true +# Set the generic cloud-provider route configuration flag without enabling the +# Linode route controller. +# configureCloudRoutes: false + # This section adds ability to enable nodeipam-controller for ccm # enableNodeIPAM: false # clusterCIDR: 10.192.0.0/10 @@ -133,6 +137,10 @@ tolerations: # nodeBalancerBackendIPv4Subnet is the subnet to use for the backend ips of the NodeBalancer # nodeBalancerBackendIPv4Subnet: "" +# nodeBalancerBackendIPv4ReservedRange is a /30 within nodeBalancerBackendIPv4Subnet +# that must never be allocated to a NodeBalancer. +# nodeBalancerBackendIPv4ReservedRange: "" + # nodeBalancerBackendIPv4SubnetID is the subnet id to use for the backend ips of the NodeBalancer # nodeBalancerBackendIPv4SubnetID: "" diff --git a/hack/test-helm-networking.sh b/hack/test-helm-networking.sh new file mode 100755 index 00000000..73a267fe --- /dev/null +++ b/hack/test-helm-networking.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +chart="$repo_root/deploy/chart" + +assert_contains() { + local output=$1 + local expected=$2 + + if ! grep -Fq -- "$expected" <<<"$output"; then + printf 'expected rendered output to contain: %s\n' "$expected" >&2 + exit 1 + fi +} + +assert_not_contains() { + local output=$1 + local unexpected=$2 + + if grep -Fq -- "$unexpected" <<<"$output"; then + printf 'expected rendered output not to contain: %s\n' "$unexpected" >&2 + exit 1 + fi +} + +route_disabled="$( + helm template route-disabled "$chart" \ + --set secretRef.apiTokenRef=apiToken \ + --set secretRef.name=api \ + --set secretRef.regionRef=us-east \ + --set configureCloudRoutes=false \ + --set nodeBalancerBackendIPv4Subnet=10.63.88.0/21 \ + --set nodeBalancerBackendIPv4ReservedRange=10.63.95.252/30 +)" +assert_contains "$route_disabled" "--configure-cloud-routes=false" +assert_contains "$route_disabled" "--nodebalancer-backend-ipv4-reserved-range=10.63.95.252/30" +assert_not_contains "$route_disabled" "--enable-route-controller=true" +assert_not_contains "$route_disabled" "--allocate-node-cidrs=true" + +route_controller_disabled="$( + helm template route-controller-disabled "$chart" \ + --set secretRef.apiTokenRef=apiToken \ + --set secretRef.name=api \ + --set secretRef.regionRef=us-east \ + --set routeController.vpcNames=test-vpc \ + --set routeController.subnetNames=test-subnet \ + --set routeController.clusterCIDR=10.0.0.0/16 \ + --set routeController.configureCloudRoutes=false +)" +assert_contains "$route_controller_disabled" "--enable-route-controller=true" +assert_contains "$route_controller_disabled" "--configure-cloud-routes=false" diff --git a/main.go b/main.go index 46131fe8..966cfc17 100644 --- a/main.go +++ b/main.go @@ -93,6 +93,7 @@ func main() { command.Flags().StringVar(&ccmOptions.Options.IpHolderSuffix, "ip-holder-suffix", "", "DEPRECATED: no effect; retained for backwards compatibility") command.Flags().StringVar(&ccmOptions.Options.DefaultNBType, "default-nodebalancer-type", string(linodego.NBTypeCommon), "default type of NodeBalancer to create (options: common, premium, premium_40GB)") command.Flags().StringVar(&ccmOptions.Options.NodeBalancerBackendIPv4Subnet, "nodebalancer-backend-ipv4-subnet", "", "ipv4 subnet to use for NodeBalancer backends") + command.Flags().StringVar(&ccmOptions.Options.NodeBalancerBackendIPv4ReservedRange, "nodebalancer-backend-ipv4-reserved-range", "", "ipv4 range within the NodeBalancer backend subnet that must never be allocated") command.Flags().StringSliceVar(&ccmOptions.Options.NodeBalancerTags, "nodebalancer-tags", []string{}, "Linode tags to apply to all NodeBalancers") command.Flags().BoolVar(&ccmOptions.Options.EnableIPv6ForLoadBalancers, "enable-ipv6-for-loadbalancers", false, "set both IPv4 and IPv6 addresses for all LoadBalancer services (when disabled, only IPv4 is used)") command.Flags().BoolVar(&ccmOptions.Options.EnableIPv6ForNodeBalancerBackends, "enable-ipv6-for-nodebalancer-backends", false, "use public IPv6 addresses for NodeBalancer service backends, including VPC-backed NodeBalancers (when enabled, may update existing services during reconciliation and all selected backend nodes must have public IPv6)") From aa03fc1d350cace6cd122b94baed2bf75def99a1 Mon Sep 17 00:00:00 2001 From: Vinicius Caldo Silva Date: Tue, 22 Sep 2026 12:44:28 +0200 Subject: [PATCH 2/4] lint --- cloud/linode/fake_linode_test.go | 3 ++- cloud/linode/nodebalancer_backend_range.go | 7 ++++--- cloud/linode/nodebalancer_backend_range_test.go | 9 +++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cloud/linode/fake_linode_test.go b/cloud/linode/fake_linode_test.go index 2863d390..cb5e96b5 100644 --- a/cloud/linode/fake_linode_test.go +++ b/cloud/linode/fake_linode_test.go @@ -13,8 +13,9 @@ import ( "strings" "testing" - "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" "github.com/linode/linodego/v2" + + "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" ) const apiVersion = "v4" diff --git a/cloud/linode/nodebalancer_backend_range.go b/cloud/linode/nodebalancer_backend_range.go index 255bd9b9..e4fa3088 100644 --- a/cloud/linode/nodebalancer_backend_range.go +++ b/cloud/linode/nodebalancer_backend_range.go @@ -39,7 +39,8 @@ func allocateNodeBalancerBackendIPv4Range(backendCIDR, reservedCIDR string, subn rangeCount := uint64(1) << uint(nodeBalancerBackendRangePrefix-backend.Bits()) assigned := make([]netip.Prefix, 0, len(subnet.Nodebalancers)) - for _, nodeBalancer := range subnet.Nodebalancers { + for index := range subnet.Nodebalancers { + nodeBalancer := &subnet.Nodebalancers[index] if nodeBalancer.Ipv4Range == "" { continue } @@ -144,8 +145,8 @@ func prefixesOverlap(first, second netip.Prefix) bool { } func overlapsAnyPrefix(candidate netip.Prefix, prefixes []netip.Prefix) bool { - for _, prefix := range prefixes { - if prefixesOverlap(candidate, prefix) { + for index := range prefixes { + if prefixesOverlap(candidate, prefixes[index]) { return true } } diff --git a/cloud/linode/nodebalancer_backend_range_test.go b/cloud/linode/nodebalancer_backend_range_test.go index ae4eb0aa..a1897f3c 100644 --- a/cloud/linode/nodebalancer_backend_range_test.go +++ b/cloud/linode/nodebalancer_backend_range_test.go @@ -5,15 +5,16 @@ import ( "testing" "github.com/golang/mock/gomock" - "github.com/linode/linode-cloud-controller-manager/cloud/annotations" - "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" - "github.com/linode/linode-cloud-controller-manager/cloud/linode/options" - "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" "github.com/linode/linodego/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/linode/linode-cloud-controller-manager/cloud/annotations" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/options" + "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" ) func TestAllocateNodeBalancerBackendIPv4Range(t *testing.T) { From c1faeaf587f7002aae491639ec682f8493b63192 Mon Sep 17 00:00:00 2001 From: Vinicius Caldo Silva Date: Thu, 24 Sep 2026 17:17:18 +0200 Subject: [PATCH 3/4] add docs --- docs/configuration/environment.md | 12 ++++++---- docs/configuration/loadbalancer.md | 38 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/configuration/environment.md b/docs/configuration/environment.md index da29979d..2079b32b 100644 --- a/docs/configuration/environment.md +++ b/docs/configuration/environment.md @@ -53,6 +53,7 @@ The CCM supports the following flags: | `--default-nodebalancer-type` | String | `common` | Default type of NodeBalancer to create (options: common, premium, premium_40gb). Note: NodeBalancer types should always be specified in lowercase. | | `--nodebalancer-tags` | String (comma separated) | | Linode tags to apply to all NodeBalancers | | `--nodebalancer-backend-ipv4-subnet` | String | `""` | ipv4 subnet to use for NodeBalancer backends | +| `--nodebalancer-backend-ipv4-reserved-range` | String | `""` | /30 within `--nodebalancer-backend-ipv4-subnet` that must never be allocated to a NodeBalancer. Must be the highest /30 of that subnet. | | `--nodebalancer-backend-ipv4-subnet-id` | Int | `""` | ipv4 subnet id to use for NodeBalancer backends | | `--nodebalancer-backend-ipv4-subnet-name` | String | `""` | ipv4 subnet name to use for NodeBalancer backends | | `--disable-nodebalancer-vpc-backends` | Boolean | `false` | don't use VPC specific ip-addresses for nodebalancer backend ips when running in VPC (set to `true` for backward compatibility if needed) | @@ -128,13 +129,16 @@ If no specific subnet is specified, by default, CCM will use the `default` subne `--nodebalancer-backend-ipv4-subnet` can be used to make sure if nodebalancer backend ips are manually specified in service annotation, they lie within the specified subnet range. +`--nodebalancer-backend-ipv4-reserved-range` keeps one /30 of `--nodebalancer-backend-ipv4-subnet` free so it is never handed to a NodeBalancer. It requires `--nodebalancer-backend-ipv4-subnet` to be set and must be the highest /30 of that subnet, otherwise CCM fails to start. When it is set, CCM allocates backend ranges itself instead of letting the Linode API auto-assign them. See [Reserving a NodeBalancer backend IPv4 range](loadbalancer.md#reserving-a-nodebalancer-backend-ipv4-range). + If CCM is started with multiple flags for nodebalancer backend subnet, following order of precedence is used for backend ip addresses: 1. NodeBalancerBackendIPv4Range annotation on service - 2. NodeBalancerBackendVPCName and NodeBalancerBackendSubnetName annotation on service - 3. NodeBalancerBackendIPv4SubnetID/NodeBalancerBackendIPv4SubnetName flag set when starting CCM - 4. NodeBalancerBackendIPv4Subnet flag when starting CCM - 5. Default to using the subnet ID of the service's VPC + 2. NodeBalancerBackendIPv4ReservedRange flag set when starting CCM (CCM picks the /30 itself) + 3. NodeBalancerBackendVPCName and NodeBalancerBackendSubnetName annotation on service + 4. NodeBalancerBackendIPv4SubnetID/NodeBalancerBackendIPv4SubnetName flag set when starting CCM + 5. NodeBalancerBackendIPv4Subnet flag when starting CCM + 6. Default to using the subnet ID of the service's VPC ## Troubleshooting diff --git a/docs/configuration/loadbalancer.md b/docs/configuration/loadbalancer.md index 915da1c0..2dc6819a 100644 --- a/docs/configuration/loadbalancer.md +++ b/docs/configuration/loadbalancer.md @@ -285,6 +285,44 @@ For a complete working example, see `examples/vpc-frontend-example.yaml`. If CCM is started with `--nodebalancer-backend-ipv4-subnet` flag, then it will not allow provisioning of nodebalancer unless subnet specified in service annotation lie within the subnet specified using the flag. This is to prevent accidental overlap between nodebalancer backend ips and pod CIDRs. +### Reserving a NodeBalancer backend IPv4 range + +`--nodebalancer-backend-ipv4-reserved-range` marks one /30 inside `--nodebalancer-backend-ipv4-subnet` as off-limits, so CCM never assigns it to a NodeBalancer. Use it to keep a backend range available for an address block managed outside the cluster. + +```yaml +spec: + template: + spec: + containers: + - name: ccm-linode + args: + - --nodebalancer-backend-ipv4-subnet=10.100.0.0/24 + - --nodebalancer-backend-ipv4-reserved-range=10.100.0.252/30 +``` + +The equivalent Helm values are `nodeBalancerBackendIPv4Subnet` and `nodeBalancerBackendIPv4ReservedRange`. + +CCM validates the reserved range at startup and exits with an error unless: + +- `--nodebalancer-backend-ipv4-subnet` is also set +- the reserved range is a /30 +- the reserved range is the highest /30 of the backend subnet (`10.100.0.252/30` for `10.100.0.0/24`) + +When a reserved range is configured, CCM allocates backend ranges itself instead of letting the Linode API auto-assign them. For each new NodeBalancer it picks the lowest /30 in the backend subnet that is not already assigned to a NodeBalancer in the VPC subnet, and never the reserved one. Provisioning fails if: + +- the backend subnet is not within the service's VPC subnet +- every /30 other than the reserved one is already assigned +- the reserved /30 is already assigned to an existing NodeBalancer + +The `linode-loadbalancer-backend-ipv4-range` annotation still takes precedence over this allocation, but a range that overlaps the reserved /30 is rejected: + +```yaml +metadata: + annotations: + # rejected when --nodebalancer-backend-ipv4-reserved-range=10.100.0.252/30 + service.beta.kubernetes.io/linode-loadbalancer-backend-ipv4-range: "10.100.0.252/30" +``` + ## Advanced Configuration ### Using Existing NodeBalancers From 0c17b7f371fd81016edd4ea89c3d7e407dc3b9ec Mon Sep 17 00:00:00 2001 From: Vinicius Caldo Silva Date: Thu, 24 Sep 2026 17:58:25 +0200 Subject: [PATCH 4/4] add chainsaw tests for NodeBalancer with reserved IPv4 backend range --- docs/development/testing.md | 2 + .../chainsaw-test.yaml | 281 ++++++++++++++++++ .../create-pods-services.yaml | 47 +++ .../create-reserved-range-service.yaml | 19 ++ .../create-second-service.yaml | 17 ++ e2e/test/scripts/get-nb-vpc-config.sh | 19 ++ 6 files changed, 385 insertions(+) create mode 100644 e2e/test/lb-with-reserved-backend-ipv4-range/chainsaw-test.yaml create mode 100644 e2e/test/lb-with-reserved-backend-ipv4-range/create-pods-services.yaml create mode 100644 e2e/test/lb-with-reserved-backend-ipv4-range/create-reserved-range-service.yaml create mode 100644 e2e/test/lb-with-reserved-backend-ipv4-range/create-second-service.yaml create mode 100755 e2e/test/scripts/get-nb-vpc-config.sh diff --git a/docs/development/testing.md b/docs/development/testing.md index c0b32029..600058df 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -75,6 +75,8 @@ mise run cleanup-cluster `mise run e2e-test` includes the IPv6 backend test slice. It is intended for the CAPL-provisioned clusters; do not use it against an existing LKE cluster. +The `lb-with-reserved-backend-ipv4-range` scenario is the one exception to the "a test only touches its own namespace" rule. It adds `--nodebalancer-backend-ipv4-subnet` and `--nodebalancer-backend-ipv4-reserved-range` to the `ccm-linode` DaemonSet, waits for the rollout, and removes both flags again when the test finishes. It is marked `concurrent: false` so Chainsaw runs it on its own, and it carries no `lke` label because it expects VPC-backed NodeBalancers and a DaemonSet it is allowed to edit. If the run is interrupted between the patch and the cleanup, remove the two flags by hand before running other scenarios. + ## End-to-End Tests with LKE The scenarios labelled `lke` can run against an existing LKE cluster. Use a disposable LKE cluster that already runs the CCM image you want to validate. These tests create namespaces and LoadBalancer Services, and some scenarios create or modify NodeBalancers, Cloud Firewalls, and reserved IPs in the cluster's region. diff --git a/e2e/test/lb-with-reserved-backend-ipv4-range/chainsaw-test.yaml b/e2e/test/lb-with-reserved-backend-ipv4-range/chainsaw-test.yaml new file mode 100644 index 00000000..ba35448b --- /dev/null +++ b/e2e/test/lb-with-reserved-backend-ipv4-range/chainsaw-test.yaml @@ -0,0 +1,281 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: lb-with-reserved-backend-ipv4-range + labels: + all: +spec: + namespace: "lb-with-reserved-backend-ipv4-range" + # This test reconfigures the cluster-wide CCM daemonset with + # --nodebalancer-backend-ipv4-subnet and --nodebalancer-backend-ipv4-reserved-range, + # which changes how every NodeBalancer gets its backend range. It must not run + # while another test is provisioning NodeBalancers. + concurrent: false + bindings: + # A /24 inside the 10.0.0.0/8 VPC subnet the test cluster runs in. Nodes take + # their VPC addresses from the bottom of that subnet and pod CIDRs come out of + # 10.192.0.0/10, so this block is free for NodeBalancer backends. + - name: backendSubnet + value: 10.100.0.0/24 + # The highest /30 of backendSubnet, which is the only value the CCM accepts. + - name: reservedRange + value: 10.100.0.252/30 + catch: + - script: + shell: bash + shellArgs: + - -c + content: | + set -euo pipefail + echo "Test failed. Fetching CCM logs..." + kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-with-reserved-backend-ipv4-range" | tail -100 + steps: + - name: Configure CCM with a reserved NodeBalancer backend range + try: + - script: + shell: bash + shellArgs: + - -c + env: + - name: BACKEND_SUBNET + value: ($backendSubnet) + - name: RESERVED_RANGE + value: ($reservedRange) + content: | + set -euo pipefail + + # Rebuild the arg list so re-running the step is idempotent. + args=$(kubectl -n kube-system get daemonset ccm-linode -o json | jq -c \ + --arg subnet "--nodebalancer-backend-ipv4-subnet=$BACKEND_SUBNET" \ + --arg reserved "--nodebalancer-backend-ipv4-reserved-range=$RESERVED_RANGE" \ + '[.spec.template.spec.containers[0].args[] + | select(test("^--nodebalancer-backend-ipv4-(subnet|reserved-range)=") | not)] + + [$subnet, $reserved]') + + if ! kubectl -n kube-system patch daemonset ccm-linode --type='json' \ + -p "[{\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/args\", \"value\": $args}]" >/dev/null; then + echo "Unable to patch the ccm-linode daemonset" + fi + + # CCM rejects an invalid reserved range at startup, so a successful + # rollout also covers the startup validation. + if ! kubectl -n kube-system rollout status daemonset/ccm-linode --timeout=240s; then + echo "CCM did not roll out with the reserved backend range flags" + fi + + applied=$(kubectl -n kube-system get daemonset ccm-linode -o json \ + | jq -r '.spec.template.spec.containers[0].args[]' \ + | grep -c -- "--nodebalancer-backend-ipv4-reserved-range=$RESERVED_RANGE" || true) + if [[ "$applied" != "1" ]]; then + echo "Reserved backend range flag is not set on the ccm-linode daemonset" + fi + check: + ($error == null): true + (contains($stdout, 'Unable to patch the ccm-linode daemonset')): false + (contains($stdout, 'CCM did not roll out with the reserved backend range flags')): false + (contains($stdout, 'Reserved backend range flag is not set on the ccm-linode daemonset')): false + cleanup: + - script: + shell: bash + shellArgs: + - -c + content: | + set -euo pipefail + + # Restore the daemonset so the tests that run after this one see the + # default backend allocation behaviour. + args=$(kubectl -n kube-system get daemonset ccm-linode -o json | jq -c \ + '[.spec.template.spec.containers[0].args[] + | select(test("^--nodebalancer-backend-ipv4-(subnet|reserved-range)=") | not)]') + + if ! kubectl -n kube-system patch daemonset ccm-linode --type='json' \ + -p "[{\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/args\", \"value\": $args}]" >/dev/null; then + echo "Unable to restore the ccm-linode daemonset" + fi + + if ! kubectl -n kube-system rollout status daemonset/ccm-linode --timeout=240s; then + echo "CCM did not roll out after restoring its flags" + fi + check: + ($error == null): true + (contains($stdout, 'Unable to restore the ccm-linode daemonset')): false + (contains($stdout, 'CCM did not roll out after restoring its flags')): false + - name: Create pods and service + try: + - apply: + file: create-pods-services.yaml + catch: + - describe: + apiVersion: v1 + kind: Pod + - describe: + apiVersion: v1 + kind: Service + - name: Check endpoints exist and loadbalancer ip is assigned + try: + - assert: + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: svc-test + (subsets[0].addresses != null): true + (subsets[0].ports != null): true + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: svc-test + status: + (loadBalancer.ingress[0].ip != null): true + - name: Create a second service once the first NodeBalancer exists + try: + - apply: + file: create-second-service.yaml + catch: + - describe: + apiVersion: v1 + kind: Service + - name: Check that the second loadbalancer ip is assigned + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: svc-test-2 + status: + (loadBalancer.ingress[0].ip != null): true + - name: Check the allocated backend ranges avoid the reserved range + try: + - script: + shell: bash + shellArgs: + - -c + env: + - name: BACKEND_SUBNET + value: ($backendSubnet) + - name: RESERVED_RANGE + value: ($reservedRange) + content: | + set -euo pipefail + + # BACKEND_SUBNET is a /24, so every valid allocation out of it looks + # like ./30. + subnet_base=${BACKEND_SUBNET%/*} + subnet_base=${subnet_base%.*} + + check_range() { + local svcname=$1 + local range=$2 + + if [[ -z "$range" || "$range" == "null" ]]; then + echo "Error: no backend ipv4 range found for $svcname" + return 0 + fi + + if [[ "${range#*/}" != "30" || "${range%/*}" != "$subnet_base".* ]]; then + echo "Error: backend range $range of $svcname is not a /30 within $BACKEND_SUBNET" + return 0 + fi + + local last_octet=${range%/*} + last_octet=${last_octet##*.} + if (( last_octet % 4 != 0 )); then + echo "Error: backend range $range of $svcname is not aligned on a /30 boundary" + fi + + if [[ "$range" == "$RESERVED_RANGE" ]]; then + echo "Error: backend range $range of $svcname is the reserved range" + fi + + return 0 + } + + vpcconfig=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN \ + LINODE_URL=$LINODE_URL ../scripts/get-nb-vpc-config.sh svc-test) + vpcconfig2=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN \ + LINODE_URL=$LINODE_URL ../scripts/get-nb-vpc-config.sh svc-test-2) + + range=$(echo "$vpcconfig" | jq -r '.ipv4_range') + range2=$(echo "$vpcconfig2" | jq -r '.ipv4_range') + echo "svc-test backend range: $range" + echo "svc-test-2 backend range: $range2" + + check_range svc-test "$range" + check_range svc-test-2 "$range2" + + if [[ "$range" == "$range2" ]]; then + echo "Error: both NodeBalancers were given the same backend range $range" + fi + + # No NodeBalancer in the VPC subnet may hold the reserved range. + vpc_id=$(echo "$vpcconfig" | jq -r '.vpc_id') + subnet_id=$(echo "$vpcconfig" | jq -r '.subnet_id') + subnet=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "$LINODE_URL/v4beta/vpcs/$vpc_id/subnets/$subnet_id") + + echo "NodeBalancers in subnet $subnet_id: $(echo "$subnet" | jq -c '[.nodebalancers[]? | {id, ipv4_range}]')" + + if echo "$subnet" | jq -e --arg reserved "$RESERVED_RANGE" \ + '.nodebalancers[]? | select(.ipv4_range == $reserved)' >/dev/null; then + echo "Error: the reserved range $RESERVED_RANGE is assigned to a NodeBalancer" + fi + check: + ($error == null): true + (contains($stdout, 'Error:')): false + - name: Reject a service asking for the reserved range + try: + - apply: + file: create-reserved-range-service.yaml + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: svc-test-reserved + status: + (loadBalancer.ingress[0].ip == null): true + - script: + shell: bash + shellArgs: + - -c + env: + - name: RESERVED_RANGE + value: ($reservedRange) + content: | + set -euo pipefail + + message="" + for i in {1..10}; do + events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed \ + --sort-by='.lastTimestamp' -o json) + message=$(echo "$events" | jq -r '.items[].message' \ + | grep "overlaps the reserved NodeBalancer backend range" || true) + if [[ -n "$message" ]]; then + echo "Warning event found: $message" + break + fi + sleep 10 + done + + if [[ -z "$message" ]]; then + echo "Error: no SyncLoadBalancerFailed event rejecting the reserved range" + fi + + service_ip=$(kubectl get svc svc-test-reserved -n $NAMESPACE \ + -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [[ -n "$service_ip" ]]; then + echo "Error: svc-test-reserved was given the ip $service_ip" + fi + check: + ($error == null): true + (contains($stdout, 'Error:')): false + catch: + - describe: + apiVersion: v1 + kind: Service diff --git a/e2e/test/lb-with-reserved-backend-ipv4-range/create-pods-services.yaml b/e2e/test/lb-with-reserved-backend-ipv4-range/create-pods-services.yaml new file mode 100644 index 00000000..e93c03dc --- /dev/null +++ b/e2e/test/lb-with-reserved-backend-ipv4-range/create-pods-services.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: with-reserved-backend-ipv4-range + name: test +spec: + replicas: 1 + selector: + matchLabels: + app: with-reserved-backend-ipv4-range + template: + metadata: + labels: + app: with-reserved-backend-ipv4-range + spec: + containers: + - image: appscode/test-server:2.3 + name: test + ports: + - name: http-1 + containerPort: 8080 + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-test + labels: + app: with-reserved-backend-ipv4-range +spec: + type: LoadBalancer + selector: + app: with-reserved-backend-ipv4-range + ports: + - name: http-1 + protocol: TCP + port: 80 + targetPort: 8080 + sessionAffinity: None diff --git a/e2e/test/lb-with-reserved-backend-ipv4-range/create-reserved-range-service.yaml b/e2e/test/lb-with-reserved-backend-ipv4-range/create-reserved-range-service.yaml new file mode 100644 index 00000000..f7d0a56b --- /dev/null +++ b/e2e/test/lb-with-reserved-backend-ipv4-range/create-reserved-range-service.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-test-reserved + annotations: + service.beta.kubernetes.io/linode-loadbalancer-backend-ipv4-range: ($reservedRange) + labels: + app: with-reserved-backend-ipv4-range +spec: + type: LoadBalancer + selector: + app: with-reserved-backend-ipv4-range + ports: + - name: http-1 + protocol: TCP + port: 80 + targetPort: 8080 + sessionAffinity: None diff --git a/e2e/test/lb-with-reserved-backend-ipv4-range/create-second-service.yaml b/e2e/test/lb-with-reserved-backend-ipv4-range/create-second-service.yaml new file mode 100644 index 00000000..675599f1 --- /dev/null +++ b/e2e/test/lb-with-reserved-backend-ipv4-range/create-second-service.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: svc-test-2 + labels: + app: with-reserved-backend-ipv4-range +spec: + type: LoadBalancer + selector: + app: with-reserved-backend-ipv4-range + ports: + - name: http-1 + protocol: TCP + port: 80 + targetPort: 8080 + sessionAffinity: None diff --git a/e2e/test/scripts/get-nb-vpc-config.sh b/e2e/test/scripts/get-nb-vpc-config.sh new file mode 100755 index 00000000..f3ea4fb2 --- /dev/null +++ b/e2e/test/scripts/get-nb-vpc-config.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR=$(dirname "$0") + +svcname="svc-test" +if [[ -n "$1" ]]; then + svcname="$1" +fi + +# Get the nodebalancer backing the service +nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN $SCRIPT_DIR/get-nb-id.sh $svcname) + +# Print the VPC config of that nodebalancer, which carries the backend ipv4 range +curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "$LINODE_URL/v4beta/nodebalancers/$nbid/vpcs" | jq -c ".data[] | select(.nodebalancer_id == $nbid)"