Skip to content

Set ipMode Proxy on load balancer ingress when PROXY protocol is enabled #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wzzirro wants to merge 2 commits into
base: main
Choose a base branch
from
+325 −692
Open

Set ipMode Proxy on load balancer ingress when PROXY protocol is enabled #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9ec3e67
Set ipMode Proxy on load balancer ingress when PROXY protocol is enabled
Oct 23, 2026
2f2fb99
Test the Proxy IP mode reported by GetLoadBalancer
Oct 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions cloudstack_loadbalancer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (

corev1 "k8s.io/api/core/v1"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/utils/ptr"
)

const (
Expand Down Expand Up @@ -137,7 +138,7 @@ func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, serv
klog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr)

status := &corev1.LoadBalancerStatus{}
status.Ingress = append(status.Ingress, corev1.LoadBalancerIngress{IP: lb.ipAddr})
status.Ingress = append(status.Ingress, loadBalancerIngress(service, lb.ipAddr))

return status, true, nil
}
Expand Down Expand Up @@ -257,11 +258,23 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s
return status, nil
}
// Default to IP
status.Ingress = []corev1.LoadBalancerIngress{{IP: lb.ipAddr}}
status.Ingress = []corev1.LoadBalancerIngress{loadBalancerIngress(service, lb.ipAddr)}

return status, nil
}

// loadBalancerIngress returns the status entry for the load balancer IP. With the proxy
// protocol enabled the IP mode is Proxy, so kube-proxy does not short-circuit in-cluster
// traffic to the IP straight to the backends: it has to go through the load balancer,
// which adds the PROXY header the backends expect.
func loadBalancerIngress(service *corev1.Service, ip string) corev1.LoadBalancerIngress {
ingress := corev1.LoadBalancerIngress{IP: ip}
if getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyProtocol, false) {
ingress.IPMode = ptr.To(corev1.LoadBalancerIPModeProxy)
}
return ingress
}

// UpdateLoadBalancer updates hosts under the specified load balancer.
func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error {
klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes)
Expand Down
26 changes: 19 additions & 7 deletions cloudstack_loadbalancer_entrypoints_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"go.uber.org/mock/gomock"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
)

type entryPointEnv struct {
Expand Down Expand Up @@ -98,13 +99,14 @@ func (e *entryPointEnv) expectRuleListing(t *testing.T, rules []*cloudstack.Load
// rules are looked up by the Service's load balancer name in its project.
func TestGetLoadBalancerStatus(t *testing.T) {
tests := []struct {
name string
projectID string
rules []*cloudstack.LoadBalancerRule
err error
wantStatus *corev1.LoadBalancerStatus
wantExists bool
wantErr string
name string
projectID string
annotations map[string]string
rules []*cloudstack.LoadBalancerRule
err error
wantStatus *corev1.LoadBalancerStatus
wantExists bool
wantErr string
}{
{name: "no rules means no load balancer"},
{
Expand All @@ -120,12 +122,22 @@ func TestGetLoadBalancerStatus(t *testing.T) {
wantStatus: &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: "203.0.113.1"}}},
wantExists: true,
},
{
name: "proxy protocol reports the Proxy IP mode",
annotations: map[string]string{ServiceAnnotationLoadBalancerProxyProtocol: "true"},
rules: []*cloudstack.LoadBalancerRule{tcpRule("rule-80", "80", "203.0.113.1", "ip-1")},
wantStatus: &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{
{IP: "203.0.113.1", IPMode: ptr.To(corev1.LoadBalancerIPModeProxy)},
}},
wantExists: true,
},
{name: "listing fails", err: errors.New("boom"), wantErr: "error retrieving load balancer rules: boom"},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env := newEntryPointEnv(t, tt.projectID)
env.service.Annotations = tt.annotations
env.expectRuleListing(t, tt.rules, tt.err)

status, exists, err := env.cs.GetLoadBalancer(context.TODO(), testClusterName, env.service)
Expand Down
3 changes: 3 additions & 0 deletions cloudstack_loadbalancer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5250,6 +5250,9 @@ func TestEnsureLoadBalancer(t *testing.T) {
if len(status.Ingress) != 1 || status.Ingress[0].IP != "10.0.0.1" {
t.Errorf("status.Ingress = %v, want IP 10.0.0.1", status.Ingress)
}
if mode := status.Ingress[0].IPMode; mode == nil || *mode != corev1.LoadBalancerIPModeProxy {
t.Errorf("status.Ingress[0].IPMode = %v, want %q", mode, corev1.LoadBalancerIPModeProxy)
}
if proto, _ := updateParams.GetProtocol(); proto != "tcp-proxy" {
t.Errorf("updated protocol = %q, want %q", proto, "tcp-proxy")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/cloudstack-ccm/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/names"
"k8s.io/cloud-provider/options"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
Expand All @@ -49,7 +50,7 @@ func main() {

fss := cliflag.NamedFlagSets{}

command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop)
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, names.CCMControllerAliases(), fss, wait.NeverStop)

// TODO: once we switch everything over to Cobra commands, we can go back to calling
// cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
Expand Down
160 changes: 71 additions & 89 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,119 +8,101 @@ require (
github.com/spf13/pflag v1.0.5
go.uber.org/mock v0.5.0
gopkg.in/gcfg.v1 v1.2.3
k8s.io/api v0.24.17
k8s.io/apimachinery v0.24.17
k8s.io/client-go v0.24.17
k8s.io/cloud-provider v0.24.17
k8s.io/component-base v0.24.17
k8s.io/klog/v2 v2.80.1
k8s.io/api v0.32.5
k8s.io/apimachinery v0.32.5
k8s.io/client-go v0.32.5
k8s.io/cloud-provider v0.32.5
k8s.io/component-base v0.32.5
k8s.io/klog/v2 v2.130.1
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
)

require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
cel.dev/expr v0.18.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful v2.16.0+incompatible // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/cel-go v0.22.0 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/spf13/cobra v1.6.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.14 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
go.etcd.io/etcd/client/v3 v3.5.14 // indirect
go.opentelemetry.io/contrib v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.0 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.etcd.io/etcd/api/v3 v3.5.16 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
go.etcd.io/etcd/client/v3 v3.5.16 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
google.golang.org/grpc v1.64.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
golang.org/x/time v0.7.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/warnings.v0 v0.1.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiserver v0.24.17 // indirect
k8s.io/component-helpers v0.24.17 // indirect
k8s.io/controller-manager v0.24.17 // indirect
k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
k8s.io/apiserver v0.32.5 // indirect
k8s.io/component-helpers v0.32.5 // indirect
k8s.io/controller-manager v0.32.5 // indirect
k8s.io/kms v0.32.5 // indirect
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)

replace (
golang.org/x/sync => golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
golang.org/x/sys => golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
golang.org/x/tools => golang.org/x/tools v0.0.0-20190313210603-aa82965741a9
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.24.17
)

replace k8s.io/kubectl => k8s.io/kubectl v0.24.17

replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.24.17

replace k8s.io/sample-controller => k8s.io/sample-controller v0.24.17

replace k8s.io/component-helpers => k8s.io/component-helpers v0.24.17

replace k8s.io/controller-manager => k8s.io/controller-manager v0.24.17

replace k8s.io/mount-utils => k8s.io/mount-utils v0.24.17
Loading
Loading