Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/dev/e2e-ote-ccm-aws.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,10 @@ pattern, use process substitution so that each invocation gets its own stdin:
```sh
BIN=./openshift-tests/bin/cloud-controller-manager-aws-tests-ext

# Run all AWSServiceLBNetworkSecurityGroup tests
# Run all NLB security group tests
while IFS= read -r t; do
echo "=== Running: $t"; $BIN run-test "$t" < /dev/null
done < <($BIN list tests | jq -r '.[].name' | grep "AWSServiceLBNetworkSecurityGroup")
done < <($BIN list tests | jq -r '.[].name' | grep "NLB")

# Run all upstream loadbalancer tests
while IFS= read -r t; do
Expand All @@ -127,7 +127,7 @@ run_test(){
while IFS= read -r t; do
echo "=== Running: $t";
$BIN run-test "$t" < /dev/null;
done < <($BIN list tests | jq -r '.[].name' | grep "AWSServiceLBNetworkSecurityGroup"); }
done < <($BIN list tests | jq -r '.[].name' | grep "NLB"); }

run_test | tee -a e2e-ote.log

Expand All @@ -137,7 +137,7 @@ grep -E "(name\"\:|\"result\")" e2e-ote.log
### Run a specific test (HyperShift hosted cluster)

When running against a HyperShift hosted cluster, `KUBECONFIG` must point to the
**guest** (hosted) cluster. Additionally, the AWSServiceLBNetworkSecurityGroup
**guest** (hosted) cluster. Additionally, the NLB security group
tests need access to the management cluster to validate the CCM cloud-config,
which lives in the hosted control plane namespace.

Expand Down
1 change: 0 additions & 1 deletion openshift-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ The test suite is organized into separate sub-projects, each with independent de

### Feature Gates Tested

- `AWSServiceLBNetworkSecurityGroup` - Managed security groups for NLBs
- `VSphereMixedNodeEnv` - Platform-type node labels on vSphere

## Development
Expand Down
6 changes: 0 additions & 6 deletions openshift-tests/ccm-aws-tests/e2e/aws/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,6 @@ func findAWSLoadBalancerByDNSName(ctx context.Context, elbClient *elbv2.Client,
return nil, nil
}

// isFeatureEnabled is a convenience wrapper around common.IsFeatureEnabled.
// Deprecated: Use common.IsFeatureEnabled directly instead.
func isFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
return common.IsFeatureEnabled(ctx, featureName)
}

// createAWSClientEC2 creates an AWS EC2 client using default credentials configured in the environment.
// It forces the public regional endpoint to avoid VPC private endpoint DNS
// resolution issues when running from a management cluster (HyperShift).
Expand Down
69 changes: 10 additions & 59 deletions openshift-tests/ccm-aws-tests/e2e/aws/loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,16 @@ import (
const (
e2eTestPrefixLoadBalancer = "[cloud-provider-aws-e2e-openshift] loadbalancer"

// featureGateAWSServiceLBNetworkSecurityGroup is the name of the feature gate
// that enables managed security groups for Network Load Balancers.
//
// Future improvement: Use typed constant from github.com/openshift/api/features
// when available: features.FeatureGateAWSServiceLBNetworkSecurityGroup
// featureGateAWSServiceLBNetworkSecurityGroup is the name of the former feature gate
// that enabled managed security groups for Network Load Balancers.
// The feature is now GA and always enabled; the constant is kept for test naming stability.
featureGateAWSServiceLBNetworkSecurityGroup = "AWSServiceLBNetworkSecurityGroup"

annotationLBType = "service.beta.kubernetes.io/aws-load-balancer-type"
)

// TestAWSServiceLBNetworkSecurityGroup validates the AWSServiceLBNetworkSecurityGroup feature gate functionality.
//
// This test suite validates that Network Load Balancers (NLB) are properly configured with security groups
// when the AWSServiceLBNetworkSecurityGroup feature gate is enabled. This feature allows the cloud controller
// to manage security groups for NLB services, improving security posture and reducing manual configuration.
//
// All tests automatically skip if the AWSServiceLBNetworkSecurityGroup feature gate is not enabled.
// TestAWSServiceLBNetworkSecurityGroup validates that Network Load Balancers (NLB) are properly
// configured with managed security groups. This feature is now GA and always enabled.
var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBalancer, featureGateAWSServiceLBNetworkSecurityGroup), func() {
f := framework.NewDefaultFramework("cloud-provider-aws")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
Expand All @@ -52,31 +45,10 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
ns = f.Namespace
})

// Checker function to verify if the feature gate is enabled for the group of tests for feature AWSServiceLBNetworkSecurityGroup.
isNLBFeatureEnabled := func(ctx context.Context) {
By(fmt.Sprintf("checking if %s feature gate is enabled", featureGateAWSServiceLBNetworkSecurityGroup))
featureEnabled, err := isFeatureEnabled(ctx, featureGateAWSServiceLBNetworkSecurityGroup)
framework.ExpectNoError(err, fmt.Sprintf("failed to check if %s feature is enabled", featureGateAWSServiceLBNetworkSecurityGroup))
if !featureEnabled {
Skip(fmt.Sprintf("%s feature gate is not enabled", featureGateAWSServiceLBNetworkSecurityGroup))
}
}

// Test: [cloud-provider-aws-e2e-openshift] loadbalancer NLB [OCPFeatureGate:AWSServiceLBNetworkSecurityGroup] should have NLBSecurityGroupMode with 'Managed' value in cloud-config
//
// Validates that the cloud controller manager's configuration contains the proper NLBSecurityGroupMode setting
// when the AWSServiceLBNetworkSecurityGroup feature gate is enabled.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
//
// Expected Results:
// - ConfigMap exists and contains cloud.conf key
// - Configuration includes: NLBSecurityGroupMode set to 'Managed'
// - The test must fail if the feature gate is enabled and the configuration does not include NLBSecurityGroupMode set to 'Managed'
// - The test must skip if the feature gate is not enabled
// Validates that the cloud controller manager's configuration contains NLBSecurityGroupMode set to 'Managed'.
It("should have NLBSecurityGroupMode with 'Managed value in cloud-config", func(ctx context.Context) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
isNLBFeatureEnabled(ctx)
common.SkipIfManagementClusterTestsDisabled()

By("getting cloud-config ConfigMap")
Expand All @@ -87,27 +59,22 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
managed, err := common.IsNLBSecurityGroupModeManaged(cm)
framework.ExpectNoError(err, "failed to check NLBSecurityGroupMode in cloud-config")
Expect(managed).To(BeTrue(),
"NLBSecurityGroupMode must be set to 'Managed' in cloud-config when feature gate is enabled")
"NLBSecurityGroupMode must be set to 'Managed' in cloud-config")

framework.Logf("Successfully validated cloud-config contains NLBSecurityGroupMode = Managed")
})

// Test: [cloud-provider-aws-e2e-openshift] loadbalancer NLB [OCPFeatureGate:AWSServiceLBNetworkSecurityGroup] should create NLB service with security group attached
//
// Creates a new Service type loadBalancer Network Load Balancer (NLB) and validates that security groups are
// automatically attached to the NLB when the AWSServiceLBNetworkSecurityGroup feature is enabled.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
// automatically attached to the NLB.
//
// Expected Results:
// - Service type loadBalancer Network Load Balancer (NLB) is created successfully
// - Backend pods start and become ready
// - Load balancer has one or more security groups attached when NLBSecurityGroupMode = Managed
// - The test must fail if the feature gate is enabled and the NLB does not have security groups attached
// - The test must skip if the feature gate is not enabled
// - The test must fail if the NLB does not have security groups attached
It("should create NLB service with security group attached", func(ctx context.Context) {
isNLBFeatureEnabled(ctx)

By("creatomg required AWS clients")
elbClient, err := createAWSClientLoadBalancer(ctx)
Expand Down Expand Up @@ -140,10 +107,9 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
// Test: [cloud-provider-aws-e2e-openshift] loadbalancer NLB [OCPFeatureGate:AWSServiceLBNetworkSecurityGroup] should have security groups attached to default ingress controller NLB
//
// Validates that the default OpenShift ingress controller's Service type loadBalancer Network Load Balancer (NLB) has security groups
// attached when the AWSServiceLBNetworkSecurityGroup feature is enabled and the router uses NLB type.
// attached and the router uses NLB type.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
// - The default ingress controller is using NLB type
//
// Expected Result:
Expand All @@ -154,7 +120,6 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
//
// Note: Skips if the default ingress controller is not using NLB type
It("should have security groups attached to default ingress controller NLB", func(ctx context.Context) {
isNLBFeatureEnabled(ctx)

By("creatomg required AWS clients")
elbClient, err := createAWSClientLoadBalancer(ctx)
Expand Down Expand Up @@ -231,9 +196,6 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
// Creates a Service type loadBalancer Network Load Balancer (NLB), modifies the service specification,
// and validates that security groups remain attached after the update.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
//
// Expected Results:
// - Service type loadBalancer Network Load Balancer (NLB) is created successfully
// - Load balancer has security groups attached before update
Expand All @@ -242,7 +204,6 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
// - Security group rules are updated to include the new port 443
// - The test must fail if security groups are removed after service update
It("should update security group rules when service is updated", func(ctx context.Context) {
isNLBFeatureEnabled(ctx)

By("creatomg required AWS clients")
ec2Client, err := createAWSClientEC2(ctx)
Expand Down Expand Up @@ -342,18 +303,13 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
// Creates a Service type loadBalancer Network Load Balancer (NLB), captures the attached security group IDs,
// deletes the service, and validates that the managed security groups are properly cleaned up.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
//
// Expected Results:
// - Service type loadBalancer Network Load Balancer (NLB) is created successfully
// - Load balancer has security groups attached
// - After service deletion, load balancer is removed
// - Managed security groups are cleaned up (deleted or detached)
// - The test must fail if managed security groups are not cleaned up
// - The test must skip if the feature gate is not enabled
It("should cleanup security groups when service is deleted", func(ctx context.Context) {
isNLBFeatureEnabled(ctx)

By("creatomg required AWS clients")
ec2Client, err := createAWSClientEC2(ctx)
Expand Down Expand Up @@ -425,18 +381,13 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
// Creates a Service type loadBalancer Network Load Balancer (NLB) and validates that the attached
// security group has the correct ingress rules matching the service port specifications.
//
// Prerequisites:
// - AWSServiceLBNetworkSecurityGroup feature gate is enabled
//
// Expected Results:
// - Service type loadBalancer Network Load Balancer (NLB) is created successfully
// - Load balancer has security groups attached
// - Security group ingress rules match the service port specifications
// - Security group rules allow traffic for all defined service ports
// - The test must fail if security group rules don't match service ports
// - The test must skip if the feature gate is not enabled
It("should have correct security group rules for service ports", func(ctx context.Context) {
isNLBFeatureEnabled(ctx)

By("creatomg required AWS clients")
ec2Client, err := createAWSClientEC2(ctx)
Expand Down
2 changes: 1 addition & 1 deletion openshift-tests/ccm-aws-tests/e2e/common/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func GetKubeClient(ctx context.Context) (clientset.Interface, error) {
//
// Parameters:
// - ctx: Context for the API call
// - featureName: Name of the feature gate to check (e.g., "AWSServiceLBNetworkSecurityGroup")
// - featureName: Name of the feature gate to check (e.g., "VSphereMixedNodeEnv")
//
// Returns:
// - bool: true if the feature is enabled, false if disabled or not found
Expand Down
28 changes: 3 additions & 25 deletions pkg/cloud/aws/aws_config_transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,37 +125,15 @@ func marshalAWSConfig(cfg *awsconfig.CloudConfig) (string, error) {
return buf.String(), nil
}

// isFeatureGateEnabled safely checks if a feature gate is enabled without panicking
// if the feature is not registered. Returns false if features is nil or if the
// feature is not in the known features list.
func isFeatureGateEnabled(features featuregates.FeatureGate, featureName string) bool {
// features.Enabled returns panic if the feature is not registered in FeatureGates,
// this functions prevents the panic by returning false if the feature is not registered in FeatureGates.
if features == nil || len(featureName) == 0 {
return false
}
for _, known := range features.KnownFeatures() {
if string(known) == featureName {
return features.Enabled(known)
}
}
return false
}

func setOpenShiftDefaults(cfg *awsconfig.CloudConfig, features featuregates.FeatureGate) {
if cfg.Global.ClusterServiceLoadBalancerHealthProbeMode == "" {
// OpenShift uses Shared mode by default.
// This attaches the health check for Cluster scope services to the "kube-proxy"
// health check endpoint served by OVN.
cfg.Global.ClusterServiceLoadBalancerHealthProbeMode = "Shared"
}
if isFeatureGateEnabled(features, "AWSServiceLBNetworkSecurityGroup") {
if cfg.Global.NLBSecurityGroupMode != awsconfig.NLBSecurityGroupModeManaged {
// When the feature gate AWSServiceLBNetworkSecurityGroup is enabled,
// OpenShift configures the AWS CCM to manage security groups for
// Network Load Balancer (NLB) Services.
klog.Infof("Enforcing cloud provider AWS configuration NLBSecurityGroupMode to Managed")
cfg.Global.NLBSecurityGroupMode = awsconfig.NLBSecurityGroupModeManaged
}
if cfg.Global.NLBSecurityGroupMode != awsconfig.NLBSecurityGroupModeManaged {
klog.Infof("Enforcing cloud provider AWS configuration NLBSecurityGroupMode to Managed")
cfg.Global.NLBSecurityGroupMode = awsconfig.NLBSecurityGroupModeManaged
}
}
Loading