From 337b2127084248193d64d45cb675a539bf9c230b Mon Sep 17 00:00:00 2001 From: Penghao Date: Sat, 14 Feb 2026 17:02:46 +0800 Subject: [PATCH] test: add storage BYOK feature tests Signed-off-by: Penghao --- test/extended/storage/const.go | 187 ++++++++++++++++++++++++++++++ test/extended/storage/csi_byok.go | 139 ++++++++++++++++++++++ 2 files changed, 326 insertions(+) create mode 100644 test/extended/storage/const.go create mode 100644 test/extended/storage/csi_byok.go diff --git a/test/extended/storage/const.go b/test/extended/storage/const.go new file mode 100644 index 000000000000..7ff96f8fd90d --- /dev/null +++ b/test/extended/storage/const.go @@ -0,0 +1,187 @@ +package storage + +// ProvisionerType represents the type of CSI provisioner +type ProvisionerType string + +const ( + // ProvisionerTypeBlock represents block storage provisioners (AWS EBS,Azure Disk, etc.) + ProvisionerTypeBlock ProvisionerType = "block" + // ProvisionerTypeFile represents file storage provisioners (AWS EFS, Azure File, etc.) + ProvisionerTypeFile ProvisionerType = "file" +) + +// FeatureSupport defines which features a provisioner supports +type FeatureSupport struct { + SupportsBYOK bool +} + +// ProvisionerInfo contains information about a CSI provisioner +type ProvisionerInfo struct { + Name string + Type ProvisionerType + Features FeatureSupport + EncryptionKeyName string // The parameter name for encryption key in StorageClass + ManagedStorageClassNames []string // List of managed/preset storage classes name +} + +// PlatformConfig contains configuration for a cloud platform +type PlatformConfig struct { + Provisioners []ProvisionerInfo + DefaultStorageClass string +} + +// Platforms maps cloud platforms to their configuration +var Platforms = map[string]PlatformConfig{ + "aws": { + Provisioners: []ProvisionerInfo{ + { + Name: "ebs.csi.aws.com", + Type: ProvisionerTypeBlock, + Features: FeatureSupport{ + SupportsBYOK: true, + }, + EncryptionKeyName: "kmsKeyId", + ManagedStorageClassNames: []string{"gp2-csi", "gp3-csi"}, + }, + { + Name: "efs.csi.aws.com", + Type: ProvisionerTypeFile, + Features: FeatureSupport{}, + EncryptionKeyName: "", + ManagedStorageClassNames: []string{"efs-sc"}, + }, + }, + DefaultStorageClass: "gp3-csi", + }, + "gcp": { + Provisioners: []ProvisionerInfo{ + { + Name: "pd.csi.storage.gke.io", + Type: ProvisionerTypeBlock, + Features: FeatureSupport{ + SupportsBYOK: true, + }, + EncryptionKeyName: "disk-encryption-kms-key", + ManagedStorageClassNames: []string{"standard-csi", "ssd-csi"}, + }, + { + Name: "filestore.csi.storage.gke.io", + Type: ProvisionerTypeFile, + Features: FeatureSupport{}, + EncryptionKeyName: "", + ManagedStorageClassNames: []string{"filestore-csi"}, + }, + }, + DefaultStorageClass: "standard-csi", + }, + "azure": { + Provisioners: []ProvisionerInfo{ + { + Name: "disk.csi.azure.com", + Type: ProvisionerTypeBlock, + Features: FeatureSupport{ + SupportsBYOK: true, + }, + EncryptionKeyName: "diskEncryptionSetID", + ManagedStorageClassNames: []string{"managed-csi"}, + }, + { + Name: "file.csi.azure.com", + Type: ProvisionerTypeFile, + Features: FeatureSupport{}, + EncryptionKeyName: "", + ManagedStorageClassNames: []string{"azurefile-csi"}, + }, + }, + DefaultStorageClass: "managed-csi", + }, + "ibmcloud": { + Provisioners: []ProvisionerInfo{ + { + Name: "vpc.block.csi.ibm.io", + Type: ProvisionerTypeBlock, + Features: FeatureSupport{ + SupportsBYOK: true, + }, + EncryptionKeyName: "encryptionKey", + ManagedStorageClassNames: []string{ + "ibmc-vpc-block-10iops-tier", + "ibmc-vpc-block-5iops-tier", + "ibmc-vpc-block-custom"}, + }, + }, + DefaultStorageClass: "ibmc-vpc-block-10iops-tier", + }, +} + +// GetProvisionersByPlatform returns all provisioners for a given platform +func GetProvisionersByPlatform(platform string) []ProvisionerInfo { + if config, ok := Platforms[platform]; ok { + return config.Provisioners + } + return nil +} + +// GetProvisionerByName finds a provisioner by its name across all platforms +func GetProvisionerByName(provisioner string) *ProvisionerInfo { + for _, config := range Platforms { + for _, p := range config.Provisioners { + if p.Name == provisioner { + return &p + } + } + } + return nil +} + +// GetBYOKProvisioners returns provisioners that support BYOK for a given platform +func GetBYOKProvisioners(platform string) []ProvisionerInfo { + var result []ProvisionerInfo + provisioners := GetProvisionersByPlatform(platform) + for _, prov := range provisioners { + if prov.Features.SupportsBYOK { + result = append(result, prov) + } + } + return result +} + +// GetProvisionersByType returns provisioners of a specific type for a given platform +func GetProvisionersByType(platform string, provType ProvisionerType) []ProvisionerInfo { + var result []ProvisionerInfo + provisioners := GetProvisionersByPlatform(platform) + for _, prov := range provisioners { + if prov.Type == provType { + result = append(result, prov) + } + } + return result +} + +// GetProvisionerNames returns just the names of provisioners for a given platform +func GetProvisionerNames(platform string) []string { + provisioners := GetProvisionersByPlatform(platform) + names := make([]string, len(provisioners)) + for i, p := range provisioners { + names[i] = p.Name + } + return names +} + +// GetBYOKProvisionerNames returns names of provisioners that support BYOK for a given platform +func GetBYOKProvisionerNames(platform string) []string { + byokProvisioners := GetBYOKProvisioners(platform) + names := make([]string, len(byokProvisioners)) + for i, p := range byokProvisioners { + names[i] = p.Name + } + return names +} + +// GetDefaultStorageClass returns the default storage class for a given platform +func GetDefaultStorageClass(platform string) string { + if config, ok := Platforms[platform]; ok { + return config.DefaultStorageClass + } + return "" +} diff --git a/test/extended/storage/csi_byok.go b/test/extended/storage/csi_byok.go new file mode 100644 index 000000000000..eb4754271a93 --- /dev/null +++ b/test/extended/storage/csi_byok.go @@ -0,0 +1,139 @@ +package storage + +import ( + "context" + "fmt" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2e "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" + + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe(`[sig-storage][Feature:BYOK][Jira:"Storage"]`, func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLIWithPodSecurityLevel("csi-byok", admissionapi.LevelPrivileged) + cloudProvider = "" + featureProviderSupportProvisioners = []string{} + ) + + g.BeforeEach(func() { + // Detect cloud provider and set supported provisioners + cloudProvider = e2e.TestContext.Provider + featureProviderSupportProvisioners = GetBYOKProvisionerNames(cloudProvider) + + if len(featureProviderSupportProvisioners) == 0 { + g.Skip("Skip for scenario non-supported provisioner!!!") + } + + }) + + g.It("managed storage classes should be set with the specified encryption key", func() { + + for _, provisioner := range featureProviderSupportProvisioners { + + // Get provisioner info from const + provisionerInfo := GetProvisionerByName(provisioner) + if provisionerInfo == nil { + g.Skip("Provisioner not found in configuration: " + provisioner) + } + + // Skipped for test clusters not installed with the BYOK + byokKeyID := getByokKeyIDFromClusterCSIDriver(oc, provisioner) + if len(byokKeyID) == 0 { + g.Skip("Skipped: the cluster is not byok cluster, no key settings in clustercsidriver/" + provisioner) + } + + g.By("Get preset storageClass names from configuration") + presetStorageClassNames := getPresetStorageClassNamesByProvisioner(provisioner) + if len(presetStorageClassNames) == 0 { + g.Skip(fmt.Sprintf("No preset storage classes configured for provisioner %s. Expected one of: %v", + provisioner, provisionerInfo.ManagedStorageClassNames)) + } + + g.By("Verify all preset storage classes have encryption key configured") + for _, scName := range presetStorageClassNames { + g.By(fmt.Sprintf("Verifying storage class: %s", scName)) + + sc, err := oc.AdminKubeClient().StorageV1().StorageClasses().Get(context.Background(), scName, metav1.GetOptions{}) + if err != nil { + g.Skip(fmt.Sprintf("Storage class %s not found in cluster: %v", scName, err)) + } + + // Double check the storage class is properly configured + o.Expect(sc.Provisioner).Should(o.Equal(provisioner), + fmt.Sprintf("Storage class %s provisioner mismatch", sc.Name)) + + // Verify BYOK key parameter exists + storedKeyID, exists := sc.Parameters[provisionerInfo.EncryptionKeyName] + if !exists { + g.Fail(fmt.Sprintf("Storage class %s does not have BYOK key parameter %s configured", + sc.Name, provisionerInfo.EncryptionKeyName)) + } + o.Expect(storedKeyID).Should(o.Equal(byokKeyID), + fmt.Sprintf("Storage class %s has different BYOK key than ClusterCSIDriver", sc.Name)) + } + } + }) +}) + +func getByokKeyIDFromClusterCSIDriver(oc *exutil.CLI, provisioner string) string { + clusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), provisioner, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Failed to get ClusterCSIDriver %s: %v", provisioner, err) + return "" + } + + driverConfig := clusterCSIDriver.Spec.DriverConfig + if driverConfig.DriverType == "" { + return "" + } + + // Extract key ID based on driver type using struct fields + switch driverConfig.DriverType { + case "AWS": + if driverConfig.AWS != nil { + return driverConfig.AWS.KMSKeyARN + } + case "Azure": + if driverConfig.Azure != nil && driverConfig.Azure.DiskEncryptionSet != nil { + // Build the full disk encryption set ID + des := driverConfig.Azure.DiskEncryptionSet + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/diskEncryptionSets/%s", + des.SubscriptionID, des.ResourceGroup, des.Name) + } + case "GCP": + if driverConfig.GCP != nil && driverConfig.GCP.KMSKey != nil { + // For GCP, return the full KMS key reference or just the key ring based on what's needed + kmsKey := driverConfig.GCP.KMSKey + location := kmsKey.Location + if location == "" { + location = "global" + } + return fmt.Sprintf("projects/%s/locations/%s/keyRings/%s/cryptoKeys/%s", + kmsKey.ProjectID, location, kmsKey.KeyRing, kmsKey.Name) + } + case "IBMCloud": + if driverConfig.IBMCloud != nil { + return driverConfig.IBMCloud.EncryptionKeyCRN + } + } + + return "" +} + +func getPresetStorageClassNamesByProvisioner(provisioner string) []string { + // Get provisioner info to get managed storage class names from static config + provisionerInfo := GetProvisionerByName(provisioner) + if provisionerInfo == nil { + e2e.Logf("Provisioner not found in configuration: %s", provisioner) + return []string{} + } + return provisionerInfo.ManagedStorageClassNames +}