From 1a2367bb78fd740e1db3204bbca6c6e0f08bb2b5 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Thu, 13 Aug 2026 14:20:20 +0200 Subject: [PATCH 1/8] wip Signed-off-by: Felix Breuer --- .gitignore | 4 ++ go.mod | 3 + go.sum | 6 ++ test/kubetest2.go | 11 +++ test/kubetest2/deployer.go | 88 +++++++++++++++++++++++ test/kubetest2/env.go | 36 ++++++++++ test/kubetest2/flags.go | 88 +++++++++++++++++++++++ test/kubetest2/lifecycle.go | 121 ++++++++++++++++++++++++++++++++ test/kubetest2/naming.go | 15 ++++ test/kubetest2/payload.go | 131 +++++++++++++++++++++++++++++++++++ test/kubetest2/ske_client.go | 96 +++++++++++++++++++++++++ 11 files changed, 599 insertions(+) create mode 100644 test/kubetest2.go create mode 100644 test/kubetest2/deployer.go create mode 100644 test/kubetest2/env.go create mode 100644 test/kubetest2/flags.go create mode 100644 test/kubetest2/lifecycle.go create mode 100644 test/kubetest2/naming.go create mode 100644 test/kubetest2/payload.go create mode 100644 test/kubetest2/ske_client.go diff --git a/.gitignore b/.gitignore index 41e3565d..63550e4f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ __debug_* test/e2e/inventory* test/e2e/sa-key* test/e2e/kubeconfig* + +# kubetest2 +_artifacts +_rundir diff --git a/go.mod b/go.mod index 90348018..02c6a4f8 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/stackitcloud/stackit-sdk-go/core v0.26.0 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -31,6 +32,7 @@ require ( k8s.io/klog/v2 v2.140.0 k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -81,6 +83,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/term v0.5.2 // indirect diff --git a/go.sum b/go.sum index 85b2cbc1..c34c2bed 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= @@ -190,6 +192,8 @@ github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD8 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -369,6 +373,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/test/kubetest2.go b/test/kubetest2.go new file mode 100644 index 00000000..a8e9be55 --- /dev/null +++ b/test/kubetest2.go @@ -0,0 +1,11 @@ +package main + +import ( + "sigs.k8s.io/kubetest2/pkg/app" + + kubetest2stackit "github.com/stackitcloud/cloud-provider-stackit/test/kubetest2" +) + +func main() { + app.Main(kubetest2stackit.Name, kubetest2stackit.New) +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go new file mode 100644 index 00000000..497b8c11 --- /dev/null +++ b/test/kubetest2/deployer.go @@ -0,0 +1,88 @@ +package kubetest2 + +import ( + "flag" + + "github.com/spf13/pflag" + "github.com/stackitcloud/cloud-provider-stackit/pkg/version" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +const ( + Name = "stackit" + + defaultRegion = "eu01" + defaultAvailabilityZone = "eu01-1" + defaultNodeCount int64 = 2 + defaultNodepoolName = "default" + defaultVolumeSizeGiB int64 = 100 + defaultKubeconfigExpiration int64 = 3600 + minKubeconfigExpiration int64 = 600 + maxKubeconfigExpiration int64 = 15552000 +) + +type Deployer struct { + options types.Options + + region string + kubernetesVersion string + availabilityZone string + machineType string + nodeImageName string + nodeImageVersion string + nodeCount int64 + nodepoolName string + volumeSizeGiB int64 + volumeType string + kubeconfigExpiresIn int64 + + projectID string + serviceAccount string + kubeconfigPath string + + skeClient skeClient +} + +var _ types.NewDeployer = New +var _ types.Deployer = &Deployer{} +var _ types.DeployerWithInit = &Deployer{} +var _ types.DeployerWithKubeconfig = &Deployer{} +var _ types.DeployerWithProvider = &Deployer{} +var _ types.DeployerWithVersion = &Deployer{} + +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + d := &Deployer{ + options: opts, + region: defaultRegion, + availabilityZone: defaultAvailabilityZone, + nodeCount: defaultNodeCount, + nodepoolName: defaultNodepoolName, + volumeSizeGiB: defaultVolumeSizeGiB, + kubeconfigExpiresIn: defaultKubeconfigExpiration, + } + + fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) + bindFlags(fs, d) + + klog.InitFlags(nil) + fs.AddGoFlagSet(flag.CommandLine) + + return d, fs +} + +func (d *Deployer) Provider() string { + return Name +} + +func (d *Deployer) Version() string { + return version.Version +} + +func (d *Deployer) Build() error { + return nil +} + +func (d *Deployer) DumpClusterLogs() error { + return nil +} diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go new file mode 100644 index 00000000..a85c1fa1 --- /dev/null +++ b/test/kubetest2/env.go @@ -0,0 +1,36 @@ +package kubetest2 + +import ( + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +func (d *Deployer) loadEnvironment() error { + klog.Infof("Loading STACKIT environment variables") + + serviceAccount, ok := os.LookupEnv("STACKIT_SERVICE_ACCOUNT") + if !ok || strings.TrimSpace(serviceAccount) == "" { + return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") + } + + projectID, ok := os.LookupEnv("STACKIT_PROJECT_ID") + if !ok || strings.TrimSpace(projectID) == "" { + return incorrectUsagef("STACKIT_PROJECT_ID environment variable is required") + } + + d.serviceAccount = serviceAccount + d.projectID = projectID + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + + klog.Infof( + "Loaded STACKIT environment: project_id=%q service_account_bytes=%d kubeconfig_path=%q", + d.projectID, + len(d.serviceAccount), + d.kubeconfigPath, + ) + + return nil +} diff --git a/test/kubetest2/flags.go b/test/kubetest2/flags.go new file mode 100644 index 00000000..530610d9 --- /dev/null +++ b/test/kubetest2/flags.go @@ -0,0 +1,88 @@ +package kubetest2 + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func bindFlags(fs *pflag.FlagSet, d *Deployer) { + fs.StringVar(&d.region, "region", defaultRegion, "STACKIT region for the SKE cluster") + fs.StringVar(&d.kubernetesVersion, "kubernetes-version", "", "Kubernetes version for the SKE cluster") + fs.StringVar(&d.availabilityZone, "availability-zone", defaultAvailabilityZone, "Availability zone for the SKE nodepool") + fs.StringVar(&d.machineType, "machine-type", "", "Machine type for the SKE nodepool") + fs.StringVar(&d.nodeImageName, "node-image-name", "", "Node image name for the SKE nodepool") + fs.StringVar(&d.nodeImageVersion, "node-image-version", "", "Node image version for the SKE nodepool") + fs.Int64Var(&d.nodeCount, "node-count", defaultNodeCount, "Node count for the SKE nodepool") + fs.StringVar(&d.nodepoolName, "nodepool-name", defaultNodepoolName, "Nodepool name for the SKE cluster") + fs.Int64Var(&d.volumeSizeGiB, "volume-size", defaultVolumeSizeGiB, "Root volume size in GiB for the SKE nodepool") + fs.StringVar(&d.volumeType, "volume-type", "", "Root volume type for the SKE nodepool") + fs.Int64Var(&d.kubeconfigExpiresIn, "kubeconfig-expiration-seconds", defaultKubeconfigExpiration, "Admin kubeconfig expiration in seconds") +} + +func (d *Deployer) validate() error { + klog.Infof( + "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d", + d.options.RunID(), + d.region, + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + d.kubeconfigExpiresIn, + ) + + requiredFlags := map[string]string{ + "--region": d.region, + "--kubernetes-version": d.kubernetesVersion, + "--availability-zone": d.availabilityZone, + "--machine-type": d.machineType, + "--node-image-name": d.nodeImageName, + "--node-image-version": d.nodeImageVersion, + } + + for flagName, value := range requiredFlags { + if strings.TrimSpace(value) == "" { + return incorrectUsagef("%s is required", flagName) + } + } + + if strings.TrimSpace(d.nodepoolName) == "" { + return incorrectUsagef("--nodepool-name must not be empty") + } + if len(d.nodepoolName) > 15 { + return incorrectUsagef("--nodepool-name must be 15 characters or fewer") + } + if d.nodeCount < 1 { + return incorrectUsagef("--node-count must be greater than 0") + } + if d.volumeSizeGiB < 1 { + return incorrectUsagef("--volume-size must be greater than 0") + } + if d.kubeconfigExpiresIn < minKubeconfigExpiration || d.kubeconfigExpiresIn > maxKubeconfigExpiration { + return incorrectUsagef( + "--kubeconfig-expiration-seconds must be between %d and %d", + minKubeconfigExpiration, + maxKubeconfigExpiration, + ) + } + if strings.TrimSpace(d.options.RunID()) == "" { + return incorrectUsagef("kubetest2 run-id must not be empty") + } + + klog.Infof("Deployer configuration validation succeeded") + + return nil +} + +func incorrectUsagef(format string, args ...any) error { + return types.NewIncorrectUsage(fmt.Sprintf(format, args...)) +} diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go new file mode 100644 index 00000000..5c123505 --- /dev/null +++ b/test/kubetest2/lifecycle.go @@ -0,0 +1,121 @@ +package kubetest2 + +import ( + "context" + "fmt" + "os" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +func (d *Deployer) Init() error { + klog.Infof("Initializing STACKIT kubetest2 deployer") + + if err := d.validate(); err != nil { + return err + } + if err := d.loadEnvironment(); err != nil { + return err + } + + client, err := newSKEClient(d.region, d.serviceAccount) + if err != nil { + return err + } + d.skeClient = client + + klog.Infof("STACKIT kubetest2 deployer initialized successfully") + + return nil +} + +func (d *Deployer) Up() error { + klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) + + ctx := context.Background() + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + + klog.Infof("Cluster up flow completed successfully for cluster=%q", clusterName) + + return nil +} + +func (d *Deployer) Down() error { + ctx := context.Background() + clusterName := d.clusterName() + + klog.Infof("Starting cluster down flow for cluster=%q", clusterName) + + if err := d.skeClient.DeleteCluster(ctx, d.projectID, d.region, clusterName); err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("delete SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Cluster=%q already absent, treating delete as success", clusterName) + return nil + } + + if err := d.skeClient.WaitForClusterDeleted(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q deletion: %w", clusterName, err) + } + + klog.Infof("Cluster down flow completed successfully for cluster=%q", clusterName) + + return nil +} + +func (d *Deployer) IsUp() (bool, error) { + ctx := context.Background() + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) + cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) + if err != nil { + if stackiterrors.IsNotFound(err) { + klog.Infof("Cluster=%q not found during IsUp check", d.clusterName()) + return false, nil + } + return false, fmt.Errorf("get SKE cluster %q: %w", d.clusterName(), err) + } + + if cluster.Status == nil || cluster.Status.Aggregated == nil { + klog.Infof("Cluster=%q has no aggregated status yet", d.clusterName()) + return false, nil + } + + state := cluster.Status.GetAggregated() + klog.Infof("Cluster=%q current aggregated state=%q", d.clusterName(), state) + return state == ske.CLUSTERSTATUSSTATE_HEALTHY || state == ske.CLUSTERSTATUSSTATE_HIBERNATED, nil +} + +func (d *Deployer) Kubeconfig() (string, error) { + if d.kubeconfigPath == "" { + d.kubeconfigPath = d.options.RunDir() + "/kubeconfig" + } + klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) + return d.kubeconfigPath, nil +} diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go new file mode 100644 index 00000000..0c8136ca --- /dev/null +++ b/test/kubetest2/naming.go @@ -0,0 +1,15 @@ +package kubetest2 + +import ( + "crypto/sha256" + "encoding/hex" +) + +func clusterNameForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return "kt2" + hex.EncodeToString(sum[:4]) +} + +func (d *Deployer) clusterName() string { + return clusterNameForRun(d.options.RunID()) +} diff --git a/test/kubetest2/payload.go b/test/kubetest2/payload.go new file mode 100644 index 00000000..c629c54e --- /dev/null +++ b/test/kubetest2/payload.go @@ -0,0 +1,131 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +func (d *Deployer) validateProviderOptions(ctx context.Context) error { + klog.Infof("Validating SKE provider options for region=%q", d.region) + + options, err := d.skeClient.ListProviderOptions(ctx, d.region) + if err != nil { + return fmt.Errorf("list SKE provider options: %w", err) + } + + if !containsKubernetesVersion(options, d.kubernetesVersion) { + return incorrectUsagef("unsupported --kubernetes-version %q for region %q", d.kubernetesVersion, d.region) + } + if !containsAvailabilityZone(options, d.availabilityZone) { + return incorrectUsagef("unsupported --availability-zone %q for region %q", d.availabilityZone, d.region) + } + if !containsMachineType(options, d.machineType) { + return incorrectUsagef("unsupported --machine-type %q for region %q", d.machineType, d.region) + } + if !containsMachineImage(options, d.nodeImageName, d.nodeImageVersion) { + return incorrectUsagef( + "unsupported node image %q version %q for region %q", + d.nodeImageName, + d.nodeImageVersion, + d.region, + ) + } + if d.volumeType != "" && !containsVolumeType(options, d.volumeType) { + return incorrectUsagef("unsupported --volume-type %q for region %q", d.volumeType, d.region) + } + + klog.Infof("SKE provider option validation succeeded for region=%q", d.region) + + return nil +} + +func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { + klog.Infof( + "Building SKE cluster payload: cluster=%q kubernetes_version=%q availability_zone=%q machine_type=%q image=%q/%q node_count=%d nodepool=%q volume_size=%d volume_type=%q", + d.clusterName(), + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + ) + + clusterKubernetes := ske.NewKubernetes(d.kubernetesVersion) + nodeImage := ske.NewImage(d.nodeImageName, d.nodeImageVersion) + nodeMachine := ske.NewMachine(*nodeImage, d.machineType) + nodeVolume := ske.NewVolume(d.volumeSizeGiB) + if d.volumeType != "" { + nodeVolume.SetType(d.volumeType) + } + + nodepool := ske.NewNodepool( + []string{d.availabilityZone}, + *nodeMachine, + d.nodeCount, + d.nodeCount, + d.nodepoolName, + *nodeVolume, + ) + nodepool.SetAllowSystemComponents(true) + + payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) + klog.Infof("Built SKE cluster payload for cluster=%q", d.clusterName()) + return *payload +} + +func containsKubernetesVersion(options *ske.ProviderOptions, version string) bool { + for _, item := range options.GetKubernetesVersions() { + if item.GetVersion() == version { + return true + } + } + return false +} + +func containsAvailabilityZone(options *ske.ProviderOptions, zone string) bool { + for _, item := range options.GetAvailabilityZones() { + if item.GetName() == zone { + return true + } + } + return false +} + +func containsMachineType(options *ske.ProviderOptions, machineType string) bool { + for _, item := range options.GetMachineTypes() { + if item.GetName() == machineType { + return true + } + } + return false +} + +func containsMachineImage(options *ske.ProviderOptions, imageName, imageVersion string) bool { + for _, image := range options.GetMachineImages() { + if image.GetName() != imageName { + continue + } + for _, version := range image.GetVersions() { + if version.GetVersion() == imageVersion { + return true + } + } + } + return false +} + +func containsVolumeType(options *ske.ProviderOptions, volumeType string) bool { + for _, item := range options.GetVolumeTypes() { + if item.GetName() == volumeType { + return true + } + } + return false +} diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go new file mode 100644 index 00000000..83104ebc --- /dev/null +++ b/test/kubetest2/ske_client.go @@ -0,0 +1,96 @@ +package kubetest2 + +import ( + "context" + "fmt" + "strconv" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" + "k8s.io/klog/v2" +) + +type skeClient interface { + GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) + CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) + WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) + DeleteCluster(ctx context.Context, projectID, region, name string) error + WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error +} + +type sdkSKEClient struct { + api ske.DefaultApi +} + +func newSKEClient(region, serviceAccount string) (skeClient, error) { + klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) + + httpClient := metrics.NewInstrumentedHTTPClient("ske") + apiClient, err := ske.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccount), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create SKE client: %w", err) + } + + klog.Infof("Created SKE API client successfully") + + return &sdkSKEClient{api: apiClient}, nil +} + +func (c *sdkSKEClient) GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("SKE GetCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.GetCluster(ctx, projectID, region, name).Execute() +} + +func (c *sdkSKEClient) ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) { + klog.Infof("SKE ListProviderOptions: region=%q", region) + return c.api.ListProviderOptions(ctx, region).Execute() +} + +func (c *sdkSKEClient) CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + klog.Infof("SKE CreateOrUpdateCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.CreateOrUpdateCluster(ctx, projectID, region, name).CreateOrUpdateClusterPayload(payload).Execute() +} + +func (c *sdkSKEClient) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("Waiting for SKE cluster to become ready: project_id=%q region=%q cluster=%q", projectID, region, name) + cluster, err := skewait.CreateOrUpdateClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + if err != nil { + return nil, err + } + if cluster != nil && cluster.Status != nil && cluster.Status.Aggregated != nil { + klog.Infof("SKE cluster is ready: cluster=%q state=%q", name, cluster.Status.GetAggregated()) + } + return cluster, nil +} + +func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) { + klog.Infof( + "SKE CreateKubeconfig: project_id=%q region=%q cluster=%q expiration_seconds=%d", + projectID, + region, + name, + expirationSeconds, + ) + payload := ske.NewCreateKubeconfigPayload() + payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) + return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() +} + +func (c *sdkSKEClient) DeleteCluster(ctx context.Context, projectID, region, name string) error { + klog.Infof("SKE DeleteCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + _, err := c.api.DeleteCluster(ctx, projectID, region, name).Execute() + return err +} + +func (c *sdkSKEClient) WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error { + klog.Infof("Waiting for SKE cluster deletion: project_id=%q region=%q cluster=%q", projectID, region, name) + _, err := skewait.DeleteClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + return err +} From c1273c4b840e23a9be00f23ec241433206e75373 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Mon, 17 Aug 2026 15:10:57 +0200 Subject: [PATCH 2/8] add project wrapper Signed-off-by: Felix Breuer --- go.mod | 4 +- go.sum | 4 + test/kubetest2/bootstrap_clients.go | 168 ++++++ test/kubetest2/bootstrap_resources.go | 353 ++++++++++++ test/kubetest2/bootstrap_test.go | 761 ++++++++++++++++++++++++++ test/kubetest2/deployer.go | 19 +- test/kubetest2/env.go | 21 +- test/kubetest2/lifecycle.go | 57 +- test/kubetest2/naming.go | 25 + 9 files changed, 1389 insertions(+), 23 deletions(-) create mode 100644 test/kubetest2/bootstrap_clients.go create mode 100644 test/kubetest2/bootstrap_resources.go create mode 100644 test/kubetest2/bootstrap_test.go diff --git a/go.mod b/go.mod index 02c6a4f8..4fa03e25 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,11 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 + github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 + github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 @@ -93,7 +96,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index c34c2bed..dd7e9949 100644 --- a/go.sum +++ b/go.sum @@ -186,12 +186,16 @@ github.com/stackitcloud/cloud-provider v0.36.0-ske-2 h1:OCDooD0twsMDdcGbcEEofRa6 github.com/stackitcloud/cloud-provider v0.36.0-ske-2/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= github.com/stackitcloud/stackit-sdk-go/core v0.26.0 h1:jQEb9gkehfp6VCP6TcYk7BI10cz4l0KM2L6hqYBH2QA= github.com/stackitcloud/stackit-sdk-go/core v0.26.0/go.mod h1:WU1hhxnjXw2EV7CYa1nlEvNpMiRY6CvmIOaHuL3pOaA= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 h1:TtpVXxgCJ17AqSOIzooQru/H3QnC4Z2VIF7HbaeKeJs= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1/go.mod h1:c9JimEdshtcIHAI2j2N388ybW+InO1fvYPvL6rxb2uY= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 h1:P8XhOi8TLA7NbLHhxU8dCJjgQpha/65agos3pI59+vw= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1/go.mod h1:/QpOsaCOQjwGgQqG8y9xXqryp25AfkorPjInX9xIeOU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD80TlFNDTKSgaEvV/o7LevTrATqcDL0r4nuhU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go new file mode 100644 index 00000000..69f27656 --- /dev/null +++ b/test/kubetest2/bootstrap_clients.go @@ -0,0 +1,168 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/wait" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" +) + +const ( + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectRole = "ske.admin" + projectListPageSize float32 = 100 +) + +type projectClient interface { + ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) + CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) + WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) + DeleteProject(ctx context.Context, projectID string) error + WaitForProjectDeleted(ctx context.Context, projectID string) error +} + +type serviceAccountClient interface { + ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) + CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) + CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) +} + +type authorizationClient interface { + ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) + AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error +} + +type sdkProjectClient struct { + api *resourcemanager.APIClient +} + +type sdkServiceAccountClient struct { + api *serviceaccount.APIClient +} + +type sdkAuthorizationClient struct { + api *authorization.APIClient +} + +func newProjectClient(serviceAccountKey string) (projectClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("resourcemanager") + apiClient, err := resourcemanager.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Resource Manager client: %w", err) + } + return &sdkProjectClient{api: apiClient}, nil +} + +func newServiceAccountClient(serviceAccountKey string) (serviceAccountClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("serviceaccount") + apiClient, err := serviceaccount.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Service Account client: %w", err) + } + return &sdkServiceAccountClient{api: apiClient}, nil +} + +func newAuthorizationClient(serviceAccountKey string) (authorizationClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("authorization") + apiClient, err := authorization.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Authorization client: %w", err) + } + return &sdkAuthorizationClient{api: apiClient}, nil +} + +func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + projects := make([]resourcemanager.Project, 0) + var offset float32 + for { + resp, err := c.api.ListProjects(ctx). + ContainerParentId(parentContainerID). + Offset(offset). + Limit(projectListPageSize). + Execute() + if err != nil { + return nil, err + } + items := resp.GetItems() + projects = append(projects, items...) + if len(items) < int(projectListPageSize) { + return projects, nil + } + offset += float32(len(items)) + } +} + +func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + payload := resourcemanager.NewCreateProjectPayload( + parentContainerID, + []resourcemanager.Member{*resourcemanager.NewMember(projectOwnerRole, ownerEmail)}, + name, + ) + payload.SetLabels(labels) + return c.api.CreateProject(ctx).CreateProjectPayload(*payload).Execute() +} + +func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api, containerID).WaitWithContext(ctx) +} + +func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { + return c.api.DeleteProject(ctx, projectID).Execute() +} + +func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api, projectID).WaitWithContext(ctx) + return err +} + +func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + resp, err := c.api.ListServiceAccounts(ctx, projectID).Execute() + if err != nil { + return nil, err + } + return resp.GetItems(), nil +} + +func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + payload := serviceaccount.NewCreateServiceAccountPayload(name) + return c.api.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() +} + +func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() + return c.api.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() +} + +func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + resp, err := c.api.ListMembers(ctx, resourceType, resourceID).Execute() + if err != nil { + return nil, err + } + return resp.GetMembers(), nil +} + +func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { + payload := authorization.NewAddMembersPayload(members, resourceType) + _, err := c.api.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + return err +} diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go new file mode 100644 index 00000000..6c040364 --- /dev/null +++ b/test/kubetest2/bootstrap_resources.go @@ -0,0 +1,353 @@ +package kubetest2 + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +type managedServiceAccount struct { + Email string + ProjectID string +} + +type serviceAccountKeyFile struct { + Active bool `json:"active"` + CreatedAt time.Time `json:"createdAt"` + Credentials serviceAccountKeyCredentialsFile `json:"credentials"` + ID string `json:"id"` + KeyAlgorithm string `json:"keyAlgorithm"` + KeyOrigin string `json:"keyOrigin"` + KeyType string `json:"keyType"` + PublicKey string `json:"publicKey"` + ValidUntil *time.Time `json:"validUntil,omitempty"` +} + +type serviceAccountKeyCredentialsFile struct { + Aud string `json:"aud"` + Iss string `json:"iss"` + Kid string `json:"kid"` + PrivateKey *string `json:"privateKey,omitempty"` + Sub string `json:"sub"` + TokenEndpoint string `json:"tokenEndpoint"` +} + +func (d *Deployer) initializeBootstrapClients() error { + if d.projectClient == nil { + client, err := newProjectClient(d.serviceAccount) + if err != nil { + return err + } + d.projectClient = client + } + if d.serviceAccountClient == nil { + client, err := newServiceAccountClient(d.serviceAccount) + if err != nil { + return err + } + d.serviceAccountClient = client + } + if d.authorizationClient == nil { + client, err := newAuthorizationClient(d.serviceAccount) + if err != nil { + return err + } + d.authorizationClient = client + } + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + if d.skeClientFactory == nil { + d.skeClientFactory = newSKEClient + } + client, err := d.skeClientFactory(d.region, serviceAccountKey) + if err != nil { + return err + } + d.skeClient = client + return nil +} + +func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) + if err != nil { + return err + } + d.childServiceAccountEmail = childServiceAccount.Email + + if err := d.ensureProjectServiceAccountRole(ctx, project.ProjectID, childServiceAccount.Email); err != nil { + return err + } + + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, project.ProjectID, childServiceAccount.Email) + if err != nil { + return err + } + return d.initializeSKEClient(serviceAccountKey) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for _, project := range projects { + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { + serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("list service accounts in STACKIT project %q: %w", projectID, err) + } + + matches := make([]managedServiceAccount, 0, 1) + for _, serviceAccount := range serviceAccounts { + if !d.matchesManagedServiceAccountEmail(serviceAccount.GetEmail()) { + continue + } + matches = append(matches, managedServiceAccount{ + Email: serviceAccount.GetEmail(), + ProjectID: serviceAccount.GetProjectId(), + }) + } + + switch len(matches) { + case 0: + klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) + createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) + if err != nil { + return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) + } + return &managedServiceAccount{ + Email: createdServiceAccount.GetEmail(), + ProjectID: createdServiceAccount.GetProjectId(), + }, nil + case 1: + klog.Infof("Reusing managed service account=%q in project_id=%q", matches[0].Email, projectID) + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed service accounts for run token %q in project %q", + len(matches), + d.runToken(), + projectID, + ) + } +} + +func (d *Deployer) ensureProjectServiceAccountRole(ctx context.Context, projectID, serviceAccountEmail string) error { + members, err := d.authorizationClient.ListMembers(ctx, projectResourceType, projectID) + if err != nil { + return fmt.Errorf("list members for STACKIT project %q: %w", projectID, err) + } + for _, member := range members { + if member.GetSubject() == serviceAccountEmail && member.GetRole() == childProjectRole { + klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, childProjectRole, projectID) + return nil + } + } + + klog.Infof("Adding role=%q for managed service account=%q in project_id=%q", childProjectRole, serviceAccountEmail, projectID) + if err := d.authorizationClient.AddMembers( + ctx, + projectID, + projectResourceType, + []authorization.Member{*authorization.NewMember(childProjectRole, serviceAccountEmail)}, + ); err != nil { + return fmt.Errorf("add role %q for service account %q in STACKIT project %q: %w", childProjectRole, serviceAccountEmail, projectID, err) + } + return nil +} + +func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (string, error) { + cachedKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return "", fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if ok { + klog.Infof("Reusing cached child service-account key %q", d.serviceAccountKeyPath) + return cachedKey, nil + } + + klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) + createdKey, err := d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + if err != nil { + return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) + } + keyJSON, err := serviceAccountKeyJSON(createdKey) + if err != nil { + return "", fmt.Errorf("serialize service-account key for %q: %w", serviceAccountEmail, err) + } + if err := d.writeCachedChildServiceAccountKey(keyJSON); err != nil { + return "", fmt.Errorf("write cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + return keyJSON, nil +} + +func (d *Deployer) readCachedChildServiceAccountKey() (string, bool, error) { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return "", false, nil + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + return string(keyBytes), true, nil +} + +func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { + if email == "" { + return false + } + localPart, _, found := strings.Cut(email, "@") + if !found { + return false + } + return strings.HasPrefix(localPart, d.serviceAccountName()) +} + +func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { + credentials := createdKey.GetCredentials() + privateKey, ok := credentials.GetPrivateKeyOk() + if !ok || strings.TrimSpace(privateKey) == "" { + return "", fmt.Errorf("service-account key response did not include a private key") + } + + serviceAccountKey := serviceAccountKeyFile{ + Active: createdKey.GetActive(), + CreatedAt: createdKey.GetCreatedAt(), + Credentials: serviceAccountKeyCredentialsFile{ + Aud: credentials.GetAud(), + Iss: credentials.GetIss(), + Kid: credentials.GetKid(), + PrivateKey: &privateKey, + Sub: credentials.GetSub(), + TokenEndpoint: credentials.GetTokenEndpoint(), + }, + ID: createdKey.GetId(), + KeyAlgorithm: string(createdKey.GetKeyAlgorithm()), + KeyOrigin: string(createdKey.GetKeyOrigin()), + KeyType: string(createdKey.GetKeyType()), + PublicKey: createdKey.GetPublicKey(), + } + if validUntil, ok := createdKey.GetValidUntilOk(); ok { + serviceAccountKey.ValidUntil = &validUntil + } + + keyJSON, err := json.Marshal(serviceAccountKey) + if err != nil { + return "", err + } + return string(keyJSON), nil +} diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go new file mode 100644 index 00000000..eeebab41 --- /dev/null +++ b/test/kubetest2/bootstrap_test.go @@ -0,0 +1,761 @@ +package kubetest2 + +import ( + "context" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" +) + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + deleteClusterCalled bool + waitDeletedCalled bool + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func (c *fakeSKEClient) DeleteCluster(_ context.Context, _, _, _ string) error { + c.deleteClusterCalled = true + return nil +} + +func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) error { + c.waitDeletedCalled = true + return nil +} + +func TestLoadEnvironmentValidation(t *testing.T) { + testCases := []struct { + name string + env map[string]string + wantErrContains string + }{ + { + name: "missing service account", + env: map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + wantErrContains: "STACKIT_SERVICE_ACCOUNT", + }, + { + name: "missing parent container", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + wantErrContains: "STACKIT_PARENT_CONTAINER_ID", + }, + { + name: "missing project member email", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, + wantErrContains: "STACKIT_PROJECT_MEMBER_EMAIL", + }, + { + name: "project id no longer required", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, key := range []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_MEMBER_EMAIL", + "STACKIT_PROJECT_ID", + } { + t.Setenv(key, "") + } + for key, value := range tc.env { + t.Setenv(key, value) + } + + runDir := t.TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if tc.wantErrContains != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) { + t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err) + } + return + } + if err != nil { + t.Fatalf("loadEnvironment() error = %v", err) + } + if d.parentContainerID != "parent-1" { + t.Fatalf("parentContainerID = %q, want %q", d.parentContainerID, "parent-1") + } + if d.projectMemberEmail != "owner@example.com" { + t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") + } + if d.projectID != "" { + t.Fatalf("projectID = %q, want empty", d.projectID) + } + if d.kubeconfigPath != filepath.Join(runDir, "kubeconfig") { + t.Fatalf("kubeconfigPath = %q", d.kubeconfigPath) + } + if d.serviceAccountKeyPath != filepath.Join(runDir, "service-account-key.json") { + t.Fatalf("serviceAccountKeyPath = %q", d.serviceAccountKeyPath) + } + }) + } +} + +func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { + d := newTestDeployer(t) + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + if err != nil { + t.Fatalf("resolveManagedProject() error = %v", err) + } + if project.ProjectID != "project-123" { + t.Fatalf("project id = %q, want %q", project.ProjectID, "project-123") + } + if projectClient.createCalls != 1 { + t.Fatalf("createCalls = %d, want 1", projectClient.createCalls) + } + if projectClient.waitActiveCalls != 1 { + t.Fatalf("waitActiveCalls = %d, want 1", projectClient.waitActiveCalls) + } + if projectClient.lastCreateParentContainerID != d.parentContainerID { + t.Fatalf("parent container = %q, want %q", projectClient.lastCreateParentContainerID, d.parentContainerID) + } + if projectClient.lastCreateName != d.projectName() { + t.Fatalf("created project name = %q, want %q", projectClient.lastCreateName, d.projectName()) + } + if projectClient.lastCreateOwnerEmail != d.projectMemberEmail { + t.Fatalf("created owner email = %q, want %q", projectClient.lastCreateOwnerEmail, d.projectMemberEmail) + } + if !reflect.DeepEqual(projectClient.lastCreateLabels, d.managedProjectLabels()) { + t.Fatalf("created labels = %#v, want %#v", projectClient.lastCreateLabels, d.managedProjectLabels()) + } +} + +func TestResolveManagedProjectErrorsOnMultipleMatches(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + if err == nil || !strings.Contains(err.Error(), "found 2 managed STACKIT projects") { + t.Fatalf("expected duplicate managed project error, got %v", err) + } +} + +func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *testing.T) { + d := newTestDeployer(t) + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + if err := d.ensureManagedClusterAccess(context.Background()); err != nil { + t.Fatalf("ensureManagedClusterAccess() error = %v", err) + } + if receivedKey != cachedKey { + t.Fatalf("received key = %q, want cached key", receivedKey) + } + if authorizationClient.addCalls != 0 { + t.Fatalf("addCalls = %d, want 0", authorizationClient.addCalls) + } + if serviceAccountClient.createKeyCalls != 0 { + t.Fatalf("createKeyCalls = %d, want 0", serviceAccountClient.createKeyCalls) + } + if d.projectID != "project-123" { + t.Fatalf("projectID = %q, want %q", d.projectID, "project-123") + } +} + +func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { + d := newTestDeployer(t) + + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + if err := d.ensureManagedClusterAccess(context.Background()); err != nil { + t.Fatalf("ensureManagedClusterAccess() error = %v", err) + } + if authorizationClient.addCalls != 1 { + t.Fatalf("addCalls = %d, want 1", authorizationClient.addCalls) + } + if serviceAccountClient.createKeyCalls != 1 { + t.Fatalf("createKeyCalls = %d, want 1", serviceAccountClient.createKeyCalls) + } + if authorizationClient.lastAddedType != projectResourceType || authorizationClient.lastAddedID != "project-123" { + t.Fatalf("AddMembers called with type=%q id=%q", authorizationClient.lastAddedType, authorizationClient.lastAddedID) + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + t.Fatalf("read cached key: %v", err) + } + if receivedKey != string(keyBytes) { + t.Fatalf("factory key mismatch") + } + if !strings.Contains(string(keyBytes), "\"privateKey\":\"PRIVATE\"") { + t.Fatalf("cached key did not contain serialized private key: %s", string(keyBytes)) + } + info, err := os.Stat(d.serviceAccountKeyPath) + if err != nil { + t.Fatalf("stat cached key: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("cached key mode = %o, want 600", info.Mode().Perm()) + } +} + +func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { + d := newTestDeployer(t) + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + if serviceAccount != cachedKey { + t.Fatalf("unexpected service account key passed to SKE client") + } + return fakeSKE, nil + } + + if err := d.Up(); err != nil { + t.Fatalf("Up() error = %v", err) + } + if fakeSKE.lastCreateProjectID != "project-123" { + t.Fatalf("CreateOrUpdate projectID = %q, want %q", fakeSKE.lastCreateProjectID, "project-123") + } + if fakeSKE.lastCreateClusterName != d.clusterName() { + t.Fatalf("cluster name = %q, want %q", fakeSKE.lastCreateClusterName, d.clusterName()) + } + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + if err != nil { + t.Fatalf("read kubeconfig: %v", err) + } + if string(kubeconfigBytes) != "apiVersion: v1\n" { + t.Fatalf("kubeconfig = %q", string(kubeconfigBytes)) + } +} + +func TestDownMissingProjectIsSuccess(t *testing.T) { + d := newTestDeployer(t) + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + if err := d.Down(); err != nil { + t.Fatalf("Down() error = %v", err) + } + if projectClient.deleteCalls != 0 { + t.Fatalf("deleteCalls = %d, want 0", projectClient.deleteCalls) + } +} + +func TestDownDeletesProjectWithoutTouchingClusterDelete(t *testing.T) { + d := newTestDeployer(t) + fakeSKE := &fakeSKEClient{} + d.skeClient = fakeSKE + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + if err := d.Down(); err != nil { + t.Fatalf("Down() error = %v", err) + } + if d.projectClient.(*fakeProjectClient).deleteCalls != 1 { + t.Fatalf("deleteCalls = %d, want 1", d.projectClient.(*fakeProjectClient).deleteCalls) + } + if d.projectClient.(*fakeProjectClient).waitDeletedCalls != 1 { + t.Fatalf("waitDeletedCalls = %d, want 1", d.projectClient.(*fakeProjectClient).waitDeletedCalls) + } + if fakeSKE.deleteClusterCalled || fakeSKE.waitDeletedCalled { + t.Fatalf("cluster delete path was invoked unexpectedly") + } +} + +func TestIsUpNoProjectReturnsFalse(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if isUp { + t.Fatalf("IsUp() = true, want false") + } +} + +func TestIsUpProjectWithoutCachedKeyReturnsError(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + isUp, err := d.IsUp() + if err == nil || !strings.Contains(err.Error(), "child service-account key cache") { + t.Fatalf("expected missing key cache error, got up=%v err=%v", isUp, err) + } +} + +func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + if serviceAccount != cachedKey { + t.Fatalf("unexpected cached key content") + } + return fakeSKE, nil + } + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if !isUp { + t.Fatalf("IsUp() = false, want true") + } + if fakeSKE.lastGetProjectID != "project-123" { + t.Fatalf("GetCluster projectID = %q, want %q", fakeSKE.lastGetProjectID, "project-123") + } +} + +func newTestDeployer(t *testing.T) *Deployer { + t.Helper() + runDir := t.TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ALGORITHM__2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_TYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if isUp { + t.Fatalf("IsUp() = true, want false") + } +} + +var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 497b8c11..b861e7e9 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -37,11 +37,19 @@ type Deployer struct { volumeType string kubeconfigExpiresIn int64 - projectID string - serviceAccount string - kubeconfigPath string - - skeClient skeClient + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + childServiceAccountEmail string + kubeconfigPath string + serviceAccountKeyPath string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + skeClient skeClient + skeClientFactory func(region, serviceAccount string) (skeClient, error) } var _ types.NewDeployer = New @@ -60,6 +68,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { nodepoolName: defaultNodepoolName, volumeSizeGiB: defaultVolumeSizeGiB, kubeconfigExpiresIn: defaultKubeconfigExpiration, + skeClientFactory: newSKEClient, } fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index a85c1fa1..12b2cae3 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -16,20 +16,29 @@ func (d *Deployer) loadEnvironment() error { return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") } - projectID, ok := os.LookupEnv("STACKIT_PROJECT_ID") - if !ok || strings.TrimSpace(projectID) == "" { - return incorrectUsagef("STACKIT_PROJECT_ID environment variable is required") + parentContainerID, ok := os.LookupEnv("STACKIT_PARENT_CONTAINER_ID") + if !ok || strings.TrimSpace(parentContainerID) == "" { + return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") + } + + projectMemberEmail, ok := os.LookupEnv("STACKIT_PROJECT_MEMBER_EMAIL") + if !ok || strings.TrimSpace(projectMemberEmail) == "" { + return incorrectUsagef("STACKIT_PROJECT_MEMBER_EMAIL environment variable is required") } d.serviceAccount = serviceAccount - d.projectID = projectID + d.parentContainerID = parentContainerID + d.projectMemberEmail = projectMemberEmail d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") klog.Infof( - "Loaded STACKIT environment: project_id=%q service_account_bytes=%d kubeconfig_path=%q", - d.projectID, + "Loaded STACKIT environment: parent_container_id=%q project_member_email=%q service_account_bytes=%d kubeconfig_path=%q service_account_key_path=%q", + d.parentContainerID, + d.projectMemberEmail, len(d.serviceAccount), d.kubeconfigPath, + d.serviceAccountKeyPath, ) return nil diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go index 5c123505..a30f4dd9 100644 --- a/test/kubetest2/lifecycle.go +++ b/test/kubetest2/lifecycle.go @@ -20,11 +20,9 @@ func (d *Deployer) Init() error { return err } - client, err := newSKEClient(d.region, d.serviceAccount) - if err != nil { + if err := d.initializeBootstrapClients(); err != nil { return err } - d.skeClient = client klog.Infof("STACKIT kubetest2 deployer initialized successfully") @@ -35,6 +33,9 @@ func (d *Deployer) Up() error { klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) ctx := context.Background() + if err := d.ensureManagedClusterAccess(ctx); err != nil { + return err + } if err := d.validateProviderOptions(ctx); err != nil { return err } @@ -69,29 +70,63 @@ func (d *Deployer) Up() error { func (d *Deployer) Down() error { ctx := context.Background() - clusterName := d.clusterName() - klog.Infof("Starting cluster down flow for cluster=%q", clusterName) + klog.Infof("Starting cluster down flow for cluster=%q", d.clusterName()) + + project, err := d.findManagedProject(ctx) + if err != nil { + return err + } + if project == nil { + klog.Infof("Managed project for run_id=%q is already absent", d.options.RunID()) + return nil + } - if err := d.skeClient.DeleteCluster(ctx, d.projectID, d.region, clusterName); err != nil { + d.projectID = project.ProjectID + if err := d.projectClient.DeleteProject(ctx, project.ProjectID); err != nil { if !stackiterrors.IsNotFound(err) { - return fmt.Errorf("delete SKE cluster %q: %w", clusterName, err) + return fmt.Errorf("delete STACKIT project %q: %w", project.ProjectID, err) } - klog.Infof("Cluster=%q already absent, treating delete as success", clusterName) + klog.Infof("Project=%q already absent, treating delete as success", project.ProjectID) return nil } - if err := d.skeClient.WaitForClusterDeleted(ctx, d.projectID, d.region, clusterName); err != nil { - return fmt.Errorf("wait for SKE cluster %q deletion: %w", clusterName, err) + if err := d.projectClient.WaitForProjectDeleted(ctx, project.ProjectID); err != nil { + return fmt.Errorf("wait for STACKIT project %q deletion: %w", project.ProjectID, err) } - klog.Infof("Cluster down flow completed successfully for cluster=%q", clusterName) + klog.Infof("Cluster down flow completed successfully for project=%q", project.ProjectID) return nil } func (d *Deployer) IsUp() (bool, error) { ctx := context.Background() + project, err := d.findManagedProject(ctx) + if err != nil { + return false, err + } + if project == nil { + klog.Infof("Managed project for run_id=%q not found during IsUp check", d.options.RunID()) + return false, nil + } + + d.projectID = project.ProjectID + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return false, fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok { + return false, fmt.Errorf( + "managed project %q exists but child service-account key cache %q is missing", + project.ProjectID, + d.serviceAccountKeyPath, + ) + } + if err := d.initializeSKEClient(serviceAccountKey); err != nil { + return false, err + } + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) if err != nil { diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go index 0c8136ca..749e76aa 100644 --- a/test/kubetest2/naming.go +++ b/test/kubetest2/naming.go @@ -10,6 +10,31 @@ func clusterNameForRun(runID string) string { return "kt2" + hex.EncodeToString(sum[:4]) } +func runTokenForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return hex.EncodeToString(sum[:4]) +} + +func projectNameForRun(runID string) string { + return "kt2-" + runTokenForRun(runID) +} + +func serviceAccountNameForRun(runID string) string { + return "kt2-" + runTokenForRun(runID) +} + func (d *Deployer) clusterName() string { return clusterNameForRun(d.options.RunID()) } + +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) +} + +func (d *Deployer) projectName() string { + return projectNameForRun(d.options.RunID()) +} + +func (d *Deployer) serviceAccountName() string { + return serviceAccountNameForRun(d.options.RunID()) +} From beb2bc7f5f865f438c5a117651525c15fa9db346 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 12:46:06 +0200 Subject: [PATCH 3/8] migrate deprecated --- test/kubetest2/bootstrap_clients.go | 28 +++++++++++++-------------- test/kubetest2/bootstrap_resources.go | 19 +++++++++--------- test/kubetest2/bootstrap_test.go | 12 ++++++------ 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index 69f27656..fe27169f 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -6,10 +6,10 @@ import ( "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/wait" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" ) const ( @@ -95,7 +95,7 @@ func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID s projects := make([]resourcemanager.Project, 0) var offset float32 for { - resp, err := c.api.ListProjects(ctx). + resp, err := c.api.DefaultAPI.ListProjects(ctx). ContainerParentId(parentContainerID). Offset(offset). Limit(projectListPageSize). @@ -119,24 +119,24 @@ func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ) payload.SetLabels(labels) - return c.api.CreateProject(ctx).CreateProjectPayload(*payload).Execute() + return c.api.DefaultAPI.CreateProject(ctx).CreateProjectPayload(*payload).Execute() } func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { - return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api, containerID).WaitWithContext(ctx) + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api.DefaultAPI, containerID).WaitWithContext(ctx) } func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { - return c.api.DeleteProject(ctx, projectID).Execute() + return c.api.DefaultAPI.DeleteProject(ctx, projectID).Execute() } func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { - _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api, projectID).WaitWithContext(ctx) + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api.DefaultAPI, projectID).WaitWithContext(ctx) return err } func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { - resp, err := c.api.ListServiceAccounts(ctx, projectID).Execute() + resp, err := c.api.DefaultAPI.ListServiceAccounts(ctx, projectID).Execute() if err != nil { return nil, err } @@ -145,16 +145,16 @@ func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, proje func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { payload := serviceaccount.NewCreateServiceAccountPayload(name) - return c.api.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() + return c.api.DefaultAPI.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() } func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() - return c.api.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + return c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() } func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { - resp, err := c.api.ListMembers(ctx, resourceType, resourceID).Execute() + resp, err := c.api.DefaultAPI.ListMembers(ctx, resourceType, resourceID).Execute() if err != nil { return nil, err } @@ -163,6 +163,6 @@ func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { payload := authorization.NewAddMembersPayload(members, resourceType) - _, err := c.api.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() return err } diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 6c040364..564b4feb 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" "k8s.io/klog/v2" ) @@ -115,7 +115,8 @@ func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, err } matches := make([]managedProject, 0, 1) - for _, project := range projects { + for i := range projects { + project := &projects[i] if !d.matchesManagedProject(project) { continue } @@ -267,7 +268,7 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje return keyJSON, nil } -func (d *Deployer) readCachedChildServiceAccountKey() (string, bool, error) { +func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { if strings.TrimSpace(d.serviceAccountKeyPath) == "" { return "", false, nil } @@ -293,7 +294,7 @@ func (d *Deployer) managedProjectLabels() map[string]string { } } -func (d *Deployer) matchesManagedProject(project resourcemanager.Project) bool { +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { if project.GetName() != d.projectName() { return false } @@ -320,7 +321,7 @@ func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { credentials := createdKey.GetCredentials() privateKey, ok := credentials.GetPrivateKeyOk() - if !ok || strings.TrimSpace(privateKey) == "" { + if !ok || strings.TrimSpace(*privateKey) == "" { return "", fmt.Errorf("service-account key response did not include a private key") } @@ -331,7 +332,7 @@ func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyRes Aud: credentials.GetAud(), Iss: credentials.GetIss(), Kid: credentials.GetKid(), - PrivateKey: &privateKey, + PrivateKey: privateKey, Sub: credentials.GetSub(), TokenEndpoint: credentials.GetTokenEndpoint(), }, @@ -342,7 +343,7 @@ func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyRes PublicKey: createdKey.GetPublicKey(), } if validUntil, ok := createdKey.GetValidUntilOk(); ok { - serviceAccountKey.ValidUntil = &validUntil + serviceAccountKey.ValidUntil = validUntil } keyJSON, err := json.Marshal(serviceAccountKey) diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index eeebab41..e9440208 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -11,9 +11,9 @@ import ( "time" oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" "github.com/stackitcloud/stackit-sdk-go/services/ske" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -724,9 +724,9 @@ func createServiceAccountKeyResponseFixture(email string) *serviceaccount.Create time.Unix(0, 0).UTC(), *credentials, "00000000-0000-0000-0000-000000000003", - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ALGORITHM__2048, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ORIGIN_GENERATED, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_TYPE_USER_MANAGED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, "PUBLIC KEY", ) } From 3b44cfa5f97a69c9595e35eb111c3976b4da9fa7 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 13:10:47 +0200 Subject: [PATCH 4/8] allow api endpoint overwrites --- test/kubetest2/bootstrap_clients.go | 33 ++++++++-------- test/kubetest2/bootstrap_resources.go | 8 ++-- test/kubetest2/bootstrap_test.go | 54 ++++++++++++++++++++++++--- test/kubetest2/deployer.go | 7 +++- test/kubetest2/env.go | 12 ++++++ test/kubetest2/ske_client.go | 10 +---- 6 files changed, 89 insertions(+), 35 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index fe27169f..fa50d5d5 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -55,36 +55,35 @@ type sdkAuthorizationClient struct { api *authorization.APIClient } -func newProjectClient(serviceAccountKey string) (projectClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("resourcemanager") - apiClient, err := resourcemanager.NewAPIClient( +func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { + opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) + sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(apiName)), + } + if endpoint != "" { + opts = append(opts, sdkconfig.WithEndpoint(endpoint)) + } + return opts +} + +func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) if err != nil { return nil, fmt.Errorf("create Resource Manager client: %w", err) } return &sdkProjectClient{api: apiClient}, nil } -func newServiceAccountClient(serviceAccountKey string) (serviceAccountClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("serviceaccount") - apiClient, err := serviceaccount.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) +func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceaccount")...) if err != nil { return nil, fmt.Errorf("create Service Account client: %w", err) } return &sdkServiceAccountClient{api: apiClient}, nil } -func newAuthorizationClient(serviceAccountKey string) (authorizationClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("authorization") - apiClient, err := authorization.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) +func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "authorization")...) if err != nil { return nil, fmt.Errorf("create Authorization client: %w", err) } diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 564b4feb..35ce4585 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -49,21 +49,21 @@ type serviceAccountKeyCredentialsFile struct { func (d *Deployer) initializeBootstrapClients() error { if d.projectClient == nil { - client, err := newProjectClient(d.serviceAccount) + client, err := newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) if err != nil { return err } d.projectClient = client } if d.serviceAccountClient == nil { - client, err := newServiceAccountClient(d.serviceAccount) + client, err := newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) if err != nil { return err } d.serviceAccountClient = client } if d.authorizationClient == nil { - client, err := newAuthorizationClient(d.serviceAccount) + client, err := newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) if err != nil { return err } @@ -76,7 +76,7 @@ func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { if d.skeClientFactory == nil { d.skeClientFactory = newSKEClient } - client, err := d.skeClientFactory(d.region, serviceAccountKey) + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) if err != nil { return err } diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index e9440208..aa609f4f 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -268,6 +268,10 @@ func TestLoadEnvironmentValidation(t *testing.T) { "STACKIT_PARENT_CONTAINER_ID", "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_PROJECT_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", } { t.Setenv(key, "") } @@ -306,6 +310,46 @@ func TestLoadEnvironmentValidation(t *testing.T) { } } +func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { + for _, key := range []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_MEMBER_EMAIL", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + } { + t.Setenv(key, "") + } + t.Setenv("STACKIT_SERVICE_ACCOUNT", "{}") + t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") + t.Setenv("STACKIT_PROJECT_MEMBER_EMAIL", "owner@example.com") + t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + t.Setenv("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := t.TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + if err := d.loadEnvironment(); err != nil { + t.Fatalf("loadEnvironment() error = %v", err) + } + + if d.resourceManagerEndpoint != "https://resource-manager.example.com" { + t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) + } + if d.serviceAccountEndpoint != "https://service-account.example.com" { + t.Fatalf("serviceAccountEndpoint = %q", d.serviceAccountEndpoint) + } + if d.authorizationEndpoint != "https://authorization.example.com" { + t.Fatalf("authorizationEndpoint = %q", d.authorizationEndpoint) + } + if d.skeEndpoint != "https://ske.example.com" { + t.Fatalf("skeEndpoint = %q", d.skeEndpoint) + } +} + func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { d := newTestDeployer(t) projectClient := &fakeProjectClient{ @@ -385,7 +429,7 @@ func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *tes d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return fakeSKE, nil } @@ -428,7 +472,7 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return &fakeSKEClient{}, nil } @@ -500,7 +544,7 @@ func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { return cfg }(), } - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { if serviceAccount != cachedKey { t.Fatalf("unexpected service account key passed to SKE client") } @@ -603,7 +647,7 @@ func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { fakeSKE := &fakeSKEClient{ getClusterResult: healthyClusterFixture(), } - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { if serviceAccount != cachedKey { t.Fatalf("unexpected cached key content") } @@ -745,7 +789,7 @@ func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { fakeSKE := &fakeSKEClient{ getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, } - d.skeClientFactory = func(_, _ string) (skeClient, error) { + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { return fakeSKE, nil } diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index b861e7e9..02825608 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -45,11 +45,16 @@ type Deployer struct { kubeconfigPath string serviceAccountKeyPath string + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + skeEndpoint string + projectClient projectClient serviceAccountClient serviceAccountClient authorizationClient authorizationClient skeClient skeClient - skeClientFactory func(region, serviceAccount string) (skeClient, error) + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) } var _ types.NewDeployer = New diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 12b2cae3..44f058a4 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -29,6 +29,10 @@ func (d *Deployer) loadEnvironment() error { d.serviceAccount = serviceAccount d.parentContainerID = parentContainerID d.projectMemberEmail = projectMemberEmail + d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) + d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) + d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") @@ -41,5 +45,13 @@ func (d *Deployer) loadEnvironment() error { d.serviceAccountKeyPath, ) + klog.Infof( + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q ske=%q", + d.resourceManagerEndpoint, + d.serviceAccountEndpoint, + d.authorizationEndpoint, + d.skeEndpoint, + ) + return nil } diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go index 83104ebc..d56060f6 100644 --- a/test/kubetest2/ske_client.go +++ b/test/kubetest2/ske_client.go @@ -5,8 +5,6 @@ import ( "fmt" "strconv" - "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" - sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" "github.com/stackitcloud/stackit-sdk-go/services/ske" skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" "k8s.io/klog/v2" @@ -26,14 +24,10 @@ type sdkSKEClient struct { api ske.DefaultApi } -func newSKEClient(region, serviceAccount string) (skeClient, error) { +func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) - httpClient := metrics.NewInstrumentedHTTPClient("ske") - apiClient, err := ske.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccount), - sdkconfig.WithHTTPClient(httpClient), - ) + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint, "ske")...) if err != nil { return nil, fmt.Errorf("create SKE client: %w", err) } From 8bd730bb4b19d30c40e49d92539bb96cab8bc9dd Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 14:33:34 +0200 Subject: [PATCH 5/8] use email from owner service account --- test/kubetest2/bootstrap_clients.go | 13 ++++++++++++- test/kubetest2/bootstrap_test.go | 25 ++++++++++++------------- test/kubetest2/env.go | 20 +++++++++++++++++--- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index fa50d5d5..c0e6115c 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -66,6 +66,13 @@ func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.C return opts } +func apiEndpointURL(cfg *sdkconfig.Configuration) string { + if cfg == nil || len(cfg.Servers) == 0 { + return "unknown" + } + return cfg.Servers[0].URL +} + func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) if err != nil { @@ -149,7 +156,11 @@ func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, proj func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() - return c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + resp, err := c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + if err != nil { + return nil, fmt.Errorf("%w (endpoint: %s)", err, apiEndpointURL(c.api.GetConfig())) + } + return resp, nil } func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index aa609f4f..69e0a490 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -18,6 +18,8 @@ import ( "sigs.k8s.io/kubetest2/pkg/types" ) +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + type fakeOptions struct { runID string runDir string @@ -230,33 +232,30 @@ func TestLoadEnvironmentValidation(t *testing.T) { { name: "missing service account", env: map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, wantErrContains: "STACKIT_SERVICE_ACCOUNT", }, { name: "missing parent container", env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, }, wantErrContains: "STACKIT_PARENT_CONTAINER_ID", }, { - name: "missing project member email", + name: "invalid service account key", env: map[string]string{ "STACKIT_SERVICE_ACCOUNT": "{}", "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, - wantErrContains: "STACKIT_PROJECT_MEMBER_EMAIL", + wantErrContains: "invalid STACKIT_SERVICE_ACCOUNT", }, { name: "project id no longer required", env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, }, } @@ -266,7 +265,6 @@ func TestLoadEnvironmentValidation(t *testing.T) { for _, key := range []string{ "STACKIT_SERVICE_ACCOUNT", "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_PROJECT_ID", "STACKIT_RESOURCE_MANAGER_ENDPOINT", "STACKIT_SERVICE_ACCOUNT_ENDPOINT", @@ -314,7 +312,6 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { for _, key := range []string{ "STACKIT_SERVICE_ACCOUNT", "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_RESOURCE_MANAGER_ENDPOINT", "STACKIT_SERVICE_ACCOUNT_ENDPOINT", "STACKIT_AUTHORIZATION_ENDPOINT", @@ -322,9 +319,8 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { } { t.Setenv(key, "") } - t.Setenv("STACKIT_SERVICE_ACCOUNT", "{}") + t.Setenv("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") - t.Setenv("STACKIT_PROJECT_MEMBER_EMAIL", "owner@example.com") t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") @@ -336,6 +332,9 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { t.Fatalf("loadEnvironment() error = %v", err) } + if d.projectMemberEmail != "owner@example.com" { + t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") + } if d.resourceManagerEndpoint != "https://resource-manager.example.com" { t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) } diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 44f058a4..271e1366 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -1,6 +1,8 @@ package kubetest2 import ( + "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -21,9 +23,9 @@ func (d *Deployer) loadEnvironment() error { return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") } - projectMemberEmail, ok := os.LookupEnv("STACKIT_PROJECT_MEMBER_EMAIL") - if !ok || strings.TrimSpace(projectMemberEmail) == "" { - return incorrectUsagef("STACKIT_PROJECT_MEMBER_EMAIL environment variable is required") + projectMemberEmail, err := extractServiceAccountEmail(serviceAccount) + if err != nil { + return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) } d.serviceAccount = serviceAccount @@ -55,3 +57,15 @@ func (d *Deployer) loadEnvironment() error { return nil } + +func extractServiceAccountEmail(serviceAccountKey string) (string, error) { + var key serviceAccountKeyFile + if err := json.Unmarshal([]byte(serviceAccountKey), &key); err != nil { + return "", fmt.Errorf("parse service account key: %w", err) + } + email := strings.TrimSpace(key.Credentials.Iss) + if email == "" { + return "", fmt.Errorf("service account key has no email in credentials.iss") + } + return email, nil +} From fb4c5daa6bc636c11d1a1107cfda72566e3c445e Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 16:16:02 +0200 Subject: [PATCH 6/8] add ske sevice enablement --- go.mod | 1 + go.sum | 2 + test/kubetest2/bootstrap_clients.go | 34 ++++++ test/kubetest2/bootstrap_resources.go | 79 ++++++++++++- test/kubetest2/bootstrap_test.go | 163 ++++++++++++++++++++++++++ test/kubetest2/deployer.go | 22 ++-- test/kubetest2/env.go | 4 +- 7 files changed, 293 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 4fa03e25..8974e3e3 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index dd7e9949..56f24f10 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJ github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 h1:TNZHrunhsXRbuqZcucLs2Gqy1sEyvabufM7pB5Tscmo= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0/go.mod h1:fXq3TmVLb4JMSve989NFFViMFoYa83s7M3hJWgN6mdQ= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index c0e6115c..d70eea6c 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -10,6 +10,8 @@ import ( resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + serviceenablementwait "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api/wait" ) const ( @@ -22,6 +24,7 @@ const ( projectResourceType = "project" childProjectRole = "ske.admin" projectListPageSize float32 = 100 + skeServiceID = "cloud.stackit.ske" ) type projectClient interface { @@ -43,6 +46,12 @@ type authorizationClient interface { AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error } +type serviceEnablementClient interface { + GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) + EnableService(ctx context.Context, region, projectID, serviceID string) error + WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error +} + type sdkProjectClient struct { api *resourcemanager.APIClient } @@ -55,6 +64,10 @@ type sdkAuthorizationClient struct { api *authorization.APIClient } +type sdkServiceEnablementClient struct { + api *serviceenablement.APIClient +} + func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), @@ -97,6 +110,14 @@ func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationCl return &sdkAuthorizationClient{api: apiClient}, nil } +func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceenablement")...) + if err != nil { + return nil, fmt.Errorf("create Service Enablement client: %w", err) + } + return &sdkServiceEnablementClient{api: apiClient}, nil +} + func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { projects := make([]resourcemanager.Project, 0) var offset float32 @@ -176,3 +197,16 @@ func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, res _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() return err } + +func (c *sdkServiceEnablementClient) GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + return c.api.DefaultAPI.GetServiceStatusRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) EnableService(ctx context.Context, region, projectID, serviceID string) error { + return c.api.DefaultAPI.EnableServiceRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error { + _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) + return err +} diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 35ce4585..e8f0b64a 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -3,17 +3,28 @@ package kubetest2 import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "os" "strings" "time" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) +var childKeyCreationRetryBackoff = wait.Backoff{ + Duration: 3 * time.Second, + Factor: 2.0, + Steps: 5, +} + type managedProject struct { ContainerID string ProjectID string @@ -69,6 +80,13 @@ func (d *Deployer) initializeBootstrapClients() error { } d.authorizationClient = client } + if d.serviceEnablementClient == nil { + client, err := newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + if err != nil { + return err + } + d.serviceEnablementClient = client + } return nil } @@ -91,6 +109,10 @@ func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { } d.projectID = project.ProjectID + if err := d.ensureSKEServiceEnabled(ctx, project.ProjectID); err != nil { + return err + } + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) if err != nil { return err @@ -108,6 +130,36 @@ func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { return d.initializeSKEClient(serviceAccountKey) } +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !isNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} + +func isNotFound(err error) bool { + var oapiErr *oapierror.GenericOpenAPIError + return errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound +} + func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) if err != nil { @@ -254,7 +306,9 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje } klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) - createdKey, err := d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + createdKey, err := retryWithBackoff(ctx, childKeyCreationRetryBackoff, func() (*serviceaccount.CreateServiceAccountKeyResponse, error) { + return d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + }) if err != nil { return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) } @@ -268,6 +322,29 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje return keyJSON, nil } +// retryWithBackoff retries fn until it succeeds or the backoff is exhausted. +// It is used for operations that may fail transiently, e.g. while a freshly +// created service account is still propagating through the STACKIT IAM and is +// not yet ready to have a key created for it. +func retryWithBackoff[T any](ctx context.Context, backoff wait.Backoff, fn func() (T, error)) (T, error) { + var result T + var lastErr error + + waitErr := wait.ExponentialBackoffWithContext(ctx, backoff, func(_ context.Context) (bool, error) { + val, err := fn() + if err != nil { + lastErr = err + return false, nil + } + result = val + return true, nil + }) + if waitErr != nil { + return result, fmt.Errorf("backoff failed: %w, last error: %v", waitErr, lastErr) + } + return result, nil +} + func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { if strings.TrimSpace(d.serviceAccountKeyPath) == "" { return "", false, nil diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index 69e0a490..264ff045 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -2,6 +2,7 @@ package kubetest2 import ( "context" + "errors" "net/http" "os" "path/filepath" @@ -14,7 +15,9 @@ import ( authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -163,6 +166,49 @@ func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, reso return c.addMembersErr } +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + type fakeSKEClient struct { providerOptions *ske.ProviderOptions providerOptionsErr error @@ -428,6 +474,7 @@ func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *tes d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return fakeSKE, nil @@ -471,6 +518,7 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return &fakeSKEClient{}, nil @@ -507,6 +555,120 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { } } +func TestEnsureSKEServiceEnabledSkipsWhenAlreadyEnabled(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 0 { + t.Fatalf("enableCalls = %d, want 0", client.enableCalls) + } + if client.waitCalls != 0 { + t.Fatalf("waitCalls = %d, want 0", client.waitCalls) + } + if client.lastGetStatusServiceID != skeServiceID { + t.Fatalf("service ID = %q, want %q", client.lastGetStatusServiceID, skeServiceID) + } +} + +func TestEnsureSKEServiceEnabledEnablesWhenNotFound(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 1 { + t.Fatalf("enableCalls = %d, want 1", client.enableCalls) + } + if client.waitCalls != 1 { + t.Fatalf("waitCalls = %d, want 1", client.waitCalls) + } + if client.lastEnableProjectID != "project-123" || client.lastEnableServiceID != skeServiceID { + t.Fatalf("EnableService called with project_id=%q service_id=%q", client.lastEnableProjectID, client.lastEnableServiceID) + } +} + +func TestEnsureSKEServiceEnabledEnablesWhenDisabled(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 1 { + t.Fatalf("enableCalls = %d, want 1", client.enableCalls) + } + if client.waitCalls != 1 { + t.Fatalf("waitCalls = %d, want 1", client.waitCalls) + } +} + +func TestEnsureSKEServiceEnabledFailsOnGetStatusError(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err == nil { + t.Fatal("ensureSKEServiceEnabled() error = nil, want non-nil") + } + if client.enableCalls != 0 { + t.Fatalf("enableCalls = %d, want 0", client.enableCalls) + } +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func TestRetryWithBackoffRetriesUntilSuccess(t *testing.T) { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + if err != nil { + t.Fatalf("retryWithBackoff() error = %v", err) + } + if result != "done" { + t.Fatalf("result = %q, want %q", result, "done") + } + if calls != 2 { + t.Fatalf("calls = %d, want 2", calls) + } +} + +func TestRetryWithBackoffReturnsLastErrorWhenExhausted(t *testing.T) { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + if err == nil { + t.Fatal("retryWithBackoff() error = nil, want non-nil") + } + if calls != 3 { + t.Fatalf("calls = %d, want 3", calls) + } +} + func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { d := newTestDeployer(t) configureValidUpInputs(d) @@ -532,6 +694,7 @@ func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { *authorization.NewMember(childProjectRole, serviceAccountEmail), }, } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} fakeSKE := &fakeSKEClient{ providerOptions: providerOptionsFixture(), diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 02825608..9725dae6 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -45,16 +45,18 @@ type Deployer struct { kubeconfigPath string serviceAccountKeyPath string - resourceManagerEndpoint string - serviceAccountEndpoint string - authorizationEndpoint string - skeEndpoint string - - projectClient projectClient - serviceAccountClient serviceAccountClient - authorizationClient authorizationClient - skeClient skeClient - skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + serviceEnablementEndpoint string + skeEndpoint string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + serviceEnablementClient serviceEnablementClient + skeClient skeClient + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) } var _ types.NewDeployer = New diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 271e1366..51bfbcfb 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -34,6 +34,7 @@ func (d *Deployer) loadEnvironment() error { d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.serviceEnablementEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ENABLEMENT_ENDPOINT")) d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") @@ -48,10 +49,11 @@ func (d *Deployer) loadEnvironment() error { ) klog.Infof( - "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q ske=%q", + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q", d.resourceManagerEndpoint, d.serviceAccountEndpoint, d.authorizationEndpoint, + d.serviceEnablementEndpoint, d.skeEndpoint, ) From e8f1e5974731d8cc6cb692d19f7413b62920fceb Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 17:11:41 +0200 Subject: [PATCH 7/8] migrate tests to ginkgo --- Makefile | 2 +- test/kubetest2/bootstrap_resources_test.go | 65 ++ test/kubetest2/bootstrap_test.go | 926 +++++++++------------ test/kubetest2/env_test.go | 24 + test/kubetest2/flags_test.go | 42 + test/kubetest2/payload_test.go | 87 ++ test/kubetest2/suite_test.go | 13 + 7 files changed, 616 insertions(+), 543 deletions(-) create mode 100644 test/kubetest2/bootstrap_resources_test.go create mode 100644 test/kubetest2/env_test.go create mode 100644 test/kubetest2/flags_test.go create mode 100644 test/kubetest2/payload_test.go create mode 100644 test/kubetest2/suite_test.go diff --git a/Makefile b/Makefile index 18f94467..4b1bc2d9 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ modules: ## Runs go mod to ensure modules are up to date. .PHONY: test test: ## Run tests. - ./hack/test.sh ./cmd/... ./pkg/... + ./hack/test.sh ./cmd/... ./pkg/... ./test/... .PHONY: test-cover test-cover: ## Run tests with coverage. diff --git a/test/kubetest2/bootstrap_resources_test.go b/test/kubetest2/bootstrap_resources_test.go new file mode 100644 index 00000000..ca1f0597 --- /dev/null +++ b/test/kubetest2/bootstrap_resources_test.go @@ -0,0 +1,65 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index 264ff045..44192f0d 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -6,11 +6,10 @@ import ( "net/http" "os" "path/filepath" - "reflect" - "strings" - "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" @@ -269,568 +268,426 @@ func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) return nil } -func TestLoadEnvironmentValidation(t *testing.T) { - testCases := []struct { - name string - env map[string]string - wantErrContains string - }{ - { - name: "missing service account", - env: map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - wantErrContains: "STACKIT_SERVICE_ACCOUNT", - }, - { - name: "missing parent container", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - }, - wantErrContains: "STACKIT_PARENT_CONTAINER_ID", - }, - { - name: "invalid service account key", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - wantErrContains: "invalid STACKIT_SERVICE_ACCOUNT", - }, - { - name: "project id no longer required", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - }, +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for _, key := range []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } { - t.Setenv(key, "") + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") } - for key, value := range tc.env { - t.Setenv(key, value) + for key, value := range env { + setEnvVar(key, value) } - runDir := t.TempDir() + runDir := GinkgoT().TempDir() d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} err := d.loadEnvironment() - if tc.wantErrContains != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) { - t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err) - } + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) return } - if err != nil { - t.Fatalf("loadEnvironment() error = %v", err) - } - if d.parentContainerID != "parent-1" { - t.Fatalf("parentContainerID = %q, want %q", d.parentContainerID, "parent-1") - } - if d.projectMemberEmail != "owner@example.com" { - t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") - } - if d.projectID != "" { - t.Fatalf("projectID = %q, want empty", d.projectID) - } - if d.kubeconfigPath != filepath.Join(runDir, "kubeconfig") { - t.Fatalf("kubeconfigPath = %q", d.kubeconfigPath) - } - if d.serviceAccountKeyPath != filepath.Join(runDir, "service-account-key.json") { - t.Fatalf("serviceAccountKeyPath = %q", d.serviceAccountKeyPath) - } - }) - } -} - -func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { - for _, key := range []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } { - t.Setenv(key, "") - } - t.Setenv("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) - t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") - t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") - t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") - t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") - t.Setenv("STACKIT_SKE_ENDPOINT", "https://ske.example.com") - - runDir := t.TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - if err := d.loadEnvironment(); err != nil { - t.Fatalf("loadEnvironment() error = %v", err) - } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing service account", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "STACKIT_SERVICE_ACCOUNT"), + Entry("missing parent container", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("project id no longer required", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) - if d.projectMemberEmail != "owner@example.com" { - t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") - } - if d.resourceManagerEndpoint != "https://resource-manager.example.com" { - t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) - } - if d.serviceAccountEndpoint != "https://service-account.example.com" { - t.Fatalf("serviceAccountEndpoint = %q", d.serviceAccountEndpoint) - } - if d.authorizationEndpoint != "https://authorization.example.com" { - t.Fatalf("authorizationEndpoint = %q", d.authorizationEndpoint) - } - if d.skeEndpoint != "https://ske.example.com" { - t.Fatalf("skeEndpoint = %q", d.skeEndpoint) - } -} + It("reads optional endpoints", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + }) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) -func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { - d := newTestDeployer(t) - projectClient := &fakeProjectClient{ - createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - } - d.projectClient = projectClient + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } - project, err := d.resolveManagedProject(context.Background()) - if err != nil { - t.Fatalf("resolveManagedProject() error = %v", err) - } - if project.ProjectID != "project-123" { - t.Fatalf("project id = %q, want %q", project.ProjectID, "project-123") - } - if projectClient.createCalls != 1 { - t.Fatalf("createCalls = %d, want 1", projectClient.createCalls) - } - if projectClient.waitActiveCalls != 1 { - t.Fatalf("waitActiveCalls = %d, want 1", projectClient.waitActiveCalls) - } - if projectClient.lastCreateParentContainerID != d.parentContainerID { - t.Fatalf("parent container = %q, want %q", projectClient.lastCreateParentContainerID, d.parentContainerID) - } - if projectClient.lastCreateName != d.projectName() { - t.Fatalf("created project name = %q, want %q", projectClient.lastCreateName, d.projectName()) - } - if projectClient.lastCreateOwnerEmail != d.projectMemberEmail { - t.Fatalf("created owner email = %q, want %q", projectClient.lastCreateOwnerEmail, d.projectMemberEmail) - } - if !reflect.DeepEqual(projectClient.lastCreateLabels, d.managedProjectLabels()) { - t.Fatalf("created labels = %#v, want %#v", projectClient.lastCreateLabels, d.managedProjectLabels()) - } -} + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) -func TestResolveManagedProjectErrorsOnMultipleMatches(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), - *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), - }, - } +var _ = Describe("ensureManagedClusterAccess", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - _, err := d.findManagedProject(context.Background()) - if err == nil || !strings.Contains(err.Error(), "found 2 managed STACKIT projects") { - t.Fatalf("expected duplicate managed project error, got %v", err) - } -} + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } -func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *testing.T) { - d := newTestDeployer(t) - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } + Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + Expect(d.projectID).To(Equal("project-123")) + }) - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - authorizationClient := &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - fakeSKE := &fakeSKEClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return fakeSKE, nil - } + It("creates key and adds membership", func() { + d := newTestDeployer() - if err := d.ensureManagedClusterAccess(context.Background()); err != nil { - t.Fatalf("ensureManagedClusterAccess() error = %v", err) - } - if receivedKey != cachedKey { - t.Fatalf("received key = %q, want cached key", receivedKey) - } - if authorizationClient.addCalls != 0 { - t.Fatalf("addCalls = %d, want 0", authorizationClient.addCalls) - } - if serviceAccountClient.createKeyCalls != 0 { - t.Fatalf("createKeyCalls = %d, want 0", serviceAccountClient.createKeyCalls) - } - if d.projectID != "project-123" { - t.Fatalf("projectID = %q, want %q", d.projectID, "project-123") - } -} + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } -func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { - d := newTestDeployer(t) + Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), - } - authorizationClient := &fakeAuthorizationClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return &fakeSKEClient{}, nil - } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) - if err := d.ensureManagedClusterAccess(context.Background()); err != nil { - t.Fatalf("ensureManagedClusterAccess() error = %v", err) - } - if authorizationClient.addCalls != 1 { - t.Fatalf("addCalls = %d, want 1", authorizationClient.addCalls) - } - if serviceAccountClient.createKeyCalls != 1 { - t.Fatalf("createKeyCalls = %d, want 1", serviceAccountClient.createKeyCalls) - } - if authorizationClient.lastAddedType != projectResourceType || authorizationClient.lastAddedID != "project-123" { - t.Fatalf("AddMembers called with type=%q id=%q", authorizationClient.lastAddedType, authorizationClient.lastAddedID) - } - keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) - if err != nil { - t.Fatalf("read cached key: %v", err) - } - if receivedKey != string(keyBytes) { - t.Fatalf("factory key mismatch") - } - if !strings.Contains(string(keyBytes), "\"privateKey\":\"PRIVATE\"") { - t.Fatalf("cached key did not contain serialized private key: %s", string(keyBytes)) - } - info, err := os.Stat(d.serviceAccountKeyPath) - if err != nil { - t.Fatalf("stat cached key: %v", err) - } - if info.Mode().Perm() != 0o600 { - t.Fatalf("cached key mode = %o, want 600", info.Mode().Perm()) - } -} + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) -func TestEnsureSKEServiceEnabledSkipsWhenAlreadyEnabled(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.serviceEnablementClient = client + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 0 { - t.Fatalf("enableCalls = %d, want 0", client.enableCalls) - } - if client.waitCalls != 0 { - t.Fatalf("waitCalls = %d, want 0", client.waitCalls) - } - if client.lastGetStatusServiceID != skeServiceID { - t.Fatalf("service ID = %q, want %q", client.lastGetStatusServiceID, skeServiceID) - } -} + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) -func TestEnsureSKEServiceEnabledEnablesWhenNotFound(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.serviceEnablementClient = client + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 1 { - t.Fatalf("enableCalls = %d, want 1", client.enableCalls) - } - if client.waitCalls != 1 { - t.Fatalf("waitCalls = %d, want 1", client.waitCalls) - } - if client.lastEnableProjectID != "project-123" || client.lastEnableServiceID != skeServiceID { - t.Fatalf("EnableService called with project_id=%q service_id=%q", client.lastEnableProjectID, client.lastEnableServiceID) - } -} + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) -func TestEnsureSKEServiceEnabledEnablesWhenDisabled(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusResult: disabledServiceStatusFixture(), - } - d.serviceEnablementClient = client + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 1 { - t.Fatalf("enableCalls = %d, want 1", client.enableCalls) - } - if client.waitCalls != 1 { - t.Fatalf("waitCalls = %d, want 1", client.waitCalls) - } -} + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) -func TestEnsureSKEServiceEnabledFailsOnGetStatusError(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, - } - d.serviceEnablementClient = client + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err == nil { - t.Fatal("ensureSKEServiceEnabled() error = nil, want non-nil") - } - if client.enableCalls != 0 { - t.Fatalf("enableCalls = %d, want 0", client.enableCalls) - } -} +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) -func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { - status := serviceenablement.NewServiceStatus() - state := serviceenablement.SERVICESTATUSSTATE_DISABLED - status.State = &state - return status -} + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) -func TestRetryWithBackoffRetriesUntilSuccess(t *testing.T) { - calls := 0 - result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { - calls++ - if calls < 2 { - return "", errors.New("transient") + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, } - return "done", nil - }) - if err != nil { - t.Fatalf("retryWithBackoff() error = %v", err) - } - if result != "done" { - t.Fatalf("result = %q, want %q", result, "done") - } - if calls != 2 { - t.Fatalf("calls = %d, want 2", calls) - } -} + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) -func TestRetryWithBackoffReturnsLastErrorWhenExhausted(t *testing.T) { - calls := 0 - _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { - calls++ - return 0, errors.New("always fails") + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) }) - if err == nil { - t.Fatal("retryWithBackoff() error = nil, want non-nil") - } - if calls != 3 { - t.Fatalf("calls = %d, want 3", calls) - } -} +}) -func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { - d := newTestDeployer(t) - configureValidUpInputs(d) +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - d.serviceAccountClient = &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - d.authorizationClient = &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - - fakeSKE := &fakeSKEClient{ - providerOptions: providerOptionsFixture(), - createOrUpdateResult: ske.NewClusterWithDefaults(), - waitReadyResult: ske.NewClusterWithDefaults(), - kubeconfigResult: func() *ske.Kubeconfig { - cfg := ske.NewKubeconfig() - cfg.SetKubeconfig("apiVersion: v1\n") - return cfg - }(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - if serviceAccount != cachedKey { - t.Fatalf("unexpected service account key passed to SKE client") + It("deletes project without touching cluster delete", func() { + d := newTestDeployer() + fakeSKE := &fakeSKEClient{} + d.skeClient = fakeSKE + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, } - return fakeSKE, nil - } - if err := d.Up(); err != nil { - t.Fatalf("Up() error = %v", err) - } - if fakeSKE.lastCreateProjectID != "project-123" { - t.Fatalf("CreateOrUpdate projectID = %q, want %q", fakeSKE.lastCreateProjectID, "project-123") - } - if fakeSKE.lastCreateClusterName != d.clusterName() { - t.Fatalf("cluster name = %q, want %q", fakeSKE.lastCreateClusterName, d.clusterName()) - } - kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) - if err != nil { - t.Fatalf("read kubeconfig: %v", err) - } - if string(kubeconfigBytes) != "apiVersion: v1\n" { - t.Fatalf("kubeconfig = %q", string(kubeconfigBytes)) - } -} + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + Expect(fakeSKE.deleteClusterCalled).To(BeFalse()) + Expect(fakeSKE.waitDeletedCalled).To(BeFalse()) + }) +}) -func TestDownMissingProjectIsSuccess(t *testing.T) { - d := newTestDeployer(t) - projectClient := &fakeProjectClient{} - d.projectClient = projectClient +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} - if err := d.Down(); err != nil { - t.Fatalf("Down() error = %v", err) - } - if projectClient.deleteCalls != 0 { - t.Fatalf("deleteCalls = %d, want 0", projectClient.deleteCalls) - } -} + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) -func TestDownDeletesProjectWithoutTouchingClusterDelete(t *testing.T) { - d := newTestDeployer(t) - fakeSKE := &fakeSKEClient{} - d.skeClient = fakeSKE - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } - if err := d.Down(); err != nil { - t.Fatalf("Down() error = %v", err) - } - if d.projectClient.(*fakeProjectClient).deleteCalls != 1 { - t.Fatalf("deleteCalls = %d, want 1", d.projectClient.(*fakeProjectClient).deleteCalls) - } - if d.projectClient.(*fakeProjectClient).waitDeletedCalls != 1 { - t.Fatalf("waitDeletedCalls = %d, want 1", d.projectClient.(*fakeProjectClient).waitDeletedCalls) - } - if fakeSKE.deleteClusterCalled || fakeSKE.waitDeletedCalled { - t.Fatalf("cluster delete path was invoked unexpectedly") - } -} + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) -func TestIsUpNoProjectReturnsFalse(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{} + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if isUp { - t.Fatalf("IsUp() = true, want false") - } -} + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } -func TestIsUpProjectWithoutCachedKeyReturnsError(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) - isUp, err := d.IsUp() - if err == nil || !strings.Contains(err.Error(), "child service-account key cache") { - t.Fatalf("expected missing key cache error, got up=%v err=%v", isUp, err) - } -} + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) -func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } - fakeSKE := &fakeSKEClient{ - getClusterResult: healthyClusterFixture(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - if serviceAccount != cachedKey { - t.Fatalf("unexpected cached key content") + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil } - return fakeSKE, nil - } - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if !isUp { - t.Fatalf("IsUp() = false, want true") - } - if fakeSKE.lastGetProjectID != "project-123" { - t.Fatalf("GetCluster projectID = %q, want %q", fakeSKE.lastGetProjectID, "project-123") - } -} + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) -func newTestDeployer(t *testing.T) *Deployer { - t.Helper() - runDir := t.TempDir() +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() return &Deployer{ options: fakeOptions{runID: "run-123", runDir: runDir}, region: defaultRegion, @@ -851,6 +708,11 @@ func configureValidUpInputs(d *Deployer) { d.volumeType = "storage" } +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + func providerOptionsFixture() *ske.ProviderOptions { kubernetesVersion := ske.NewKubernetesVersion() kubernetesVersion.SetVersion("1.31.0") @@ -888,6 +750,13 @@ func healthyClusterFixture() *ske.Cluster { return cluster } +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { project := resourcemanager.NewProjectWithDefaults() project.SetName(name) @@ -937,31 +806,4 @@ func createServiceAccountKeyResponseFixture(email string) *serviceaccount.Create ) } -func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } - fakeSKE := &fakeSKEClient{ - getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.skeClientFactory = func(_, _, _ string) (skeClient, error) { - return fakeSKE, nil - } - - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if isUp { - t.Fatalf("IsUp() = true, want false") - } -} - var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go new file mode 100644 index 00000000..267acdae --- /dev/null +++ b/test/kubetest2/env_test.go @@ -0,0 +1,24 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extractServiceAccountEmail", func() { + It("extracts the email from the credentials.iss field", func() { + email, err := extractServiceAccountEmail(validServiceAccountKey) + Expect(err).NotTo(HaveOccurred()) + Expect(email).To(Equal("owner@example.com")) + }) + + DescribeTable("rejects invalid service account keys", + func(key string, wantErrContains string) { + _, err := extractServiceAccountEmail(key) + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("malformed JSON", "not-json", "parse service account key"), + Entry("missing credentials.iss", `{"credentials":{"aud":"x"}}`, "no email in credentials.iss"), + Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), + ) +}) diff --git a/test/kubetest2/flags_test.go b/test/kubetest2/flags_test.go new file mode 100644 index 00000000..b0e9f083 --- /dev/null +++ b/test/kubetest2/flags_test.go @@ -0,0 +1,42 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func newValidTestDeployer() *Deployer { + d := newTestDeployer() + configureValidUpInputs(d) + d.nodeCount = defaultNodeCount + d.volumeSizeGiB = defaultVolumeSizeGiB + d.kubeconfigExpiresIn = defaultKubeconfigExpiration + return d +} + +var _ = Describe("validate", func() { + It("accepts a valid configuration", func() { + Expect(newValidTestDeployer().validate()).To(Succeed()) + }) + + DescribeTable("rejects invalid configuration", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + mutate(d) + Expect(d.validate()).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("missing region", func(d *Deployer) { d.region = "" }, "--region is required"), + Entry("missing kubernetes version", func(d *Deployer) { d.kubernetesVersion = "" }, "--kubernetes-version is required"), + Entry("missing availability zone", func(d *Deployer) { d.availabilityZone = "" }, "--availability-zone is required"), + Entry("missing machine type", func(d *Deployer) { d.machineType = "" }, "--machine-type is required"), + Entry("missing node image name", func(d *Deployer) { d.nodeImageName = "" }, "--node-image-name is required"), + Entry("missing node image version", func(d *Deployer) { d.nodeImageVersion = "" }, "--node-image-version is required"), + Entry("empty nodepool name", func(d *Deployer) { d.nodepoolName = "" }, "--nodepool-name must not be empty"), + Entry("nodepool name too long", func(d *Deployer) { d.nodepoolName = "this-name-is-way-too-long" }, "--nodepool-name must be 15 characters or fewer"), + Entry("zero node count", func(d *Deployer) { d.nodeCount = 0 }, "--node-count must be greater than 0"), + Entry("zero volume size", func(d *Deployer) { d.volumeSizeGiB = 0 }, "--volume-size must be greater than 0"), + Entry("kubeconfig expiration too small", func(d *Deployer) { d.kubeconfigExpiresIn = minKubeconfigExpiration - 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("kubeconfig expiration too large", func(d *Deployer) { d.kubeconfigExpiresIn = maxKubeconfigExpiration + 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("empty run id", func(d *Deployer) { d.options = fakeOptions{runID: "", runDir: d.options.RunDir()} }, "run-id must not be empty"), + ) +}) diff --git a/test/kubetest2/payload_test.go b/test/kubetest2/payload_test.go new file mode 100644 index 00000000..32b84a68 --- /dev/null +++ b/test/kubetest2/payload_test.go @@ -0,0 +1,87 @@ +package kubetest2 + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("validateProviderOptions", func() { + It("accepts supported options", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) + + DescribeTable("rejects unsupported options", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + mutate(d) + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("unsupported kubernetes version", func(d *Deployer) { d.kubernetesVersion = "9.9.9" }, "unsupported --kubernetes-version"), + Entry("unsupported availability zone", func(d *Deployer) { d.availabilityZone = "eu99-9" }, "unsupported --availability-zone"), + Entry("unsupported machine type", func(d *Deployer) { d.machineType = "x9.9" }, "unsupported --machine-type"), + Entry("unsupported node image name", func(d *Deployer) { d.nodeImageName = "fedora" }, "unsupported node image"), + Entry("unsupported node image version", func(d *Deployer) { d.nodeImageVersion = "v99" }, "unsupported node image"), + Entry("unsupported volume type", func(d *Deployer) { d.volumeType = "nvme" }, "unsupported --volume-type"), + ) + + It("fails when listing provider options errors", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptionsErr: errors.New("boom")} + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring("list SKE provider options"))) + }) + + It("allows an empty volume type", func() { + d := newValidTestDeployer() + d.volumeType = "" + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) +}) + +var _ = Describe("clusterPayload", func() { + It("builds the expected payload", func() { + d := newValidTestDeployer() + payload := d.clusterPayload() + + kubernetes := payload.GetKubernetes() + Expect(kubernetes.GetVersion()).To(Equal("1.31.0")) + + nodepools := payload.GetNodepools() + Expect(nodepools).To(HaveLen(1)) + + nodepool := nodepools[0] + Expect(nodepool.GetName()).To(Equal(defaultNodepoolName)) + Expect(nodepool.GetAvailabilityZones()).To(ConsistOf(defaultAvailabilityZone)) + + machine := nodepool.GetMachine() + Expect(machine.GetType()).To(Equal("g1.2")) + image := machine.GetImage() + Expect(image.GetName()).To(Equal("ubuntu")) + Expect(image.GetVersion()).To(Equal("v1")) + + Expect(nodepool.GetMinimum()).To(Equal(d.nodeCount)) + Expect(nodepool.GetMaximum()).To(Equal(d.nodeCount)) + + volume := nodepool.GetVolume() + Expect(volume.GetSize()).To(Equal(d.volumeSizeGiB)) + Expect(volume.GetType()).To(Equal("storage")) + + Expect(nodepool.GetAllowSystemComponents()).To(BeTrue()) + }) + + It("omits the volume type when unset", func() { + d := newValidTestDeployer() + d.volumeType = "" + payload := d.clusterPayload() + + Expect(payload.GetNodepools()).To(HaveLen(1)) + volume := payload.GetNodepools()[0].GetVolume() + Expect(volume.GetType()).To(BeEmpty()) + }) +}) diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go new file mode 100644 index 00000000..52d96754 --- /dev/null +++ b/test/kubetest2/suite_test.go @@ -0,0 +1,13 @@ +package kubetest2 + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKubetest2(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kubetest2 Suite") +} From dcf6dcc753cf5cf28f79dbc97286d149265ff9ea Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Wed, 19 Aug 2026 16:58:21 +0200 Subject: [PATCH 8/8] refactor file structure --- go.mod | 2 +- test/kubetest2/bootstrap_csi.go | 15 + test/kubetest2/bootstrap_project.go | 146 ++++ test/kubetest2/bootstrap_project_test.go | 157 ++++ test/kubetest2/bootstrap_resources_test.go | 65 -- ...sources.go => bootstrap_serviceaccount.go} | 204 +---- .../bootstrap_serviceaccount_test.go | 128 +++ .../{payload.go => bootstrap_ske.go} | 39 +- ...{payload_test.go => bootstrap_ske_test.go} | 0 test/kubetest2/bootstrap_test.go | 809 ------------------ .../{bootstrap_clients.go => clients.go} | 89 +- test/kubetest2/deployer.go | 19 +- test/kubetest2/env_test.go | 75 ++ test/kubetest2/lifecycle.go | 50 +- test/kubetest2/lifecycle_test.go | 160 ++++ test/kubetest2/naming.go | 23 +- test/kubetest2/ske_client.go | 16 +- test/kubetest2/suite_test.go | 368 ++++++++ 18 files changed, 1199 insertions(+), 1166 deletions(-) create mode 100644 test/kubetest2/bootstrap_csi.go create mode 100644 test/kubetest2/bootstrap_project.go create mode 100644 test/kubetest2/bootstrap_project_test.go delete mode 100644 test/kubetest2/bootstrap_resources_test.go rename test/kubetest2/{bootstrap_resources.go => bootstrap_serviceaccount.go} (57%) create mode 100644 test/kubetest2/bootstrap_serviceaccount_test.go rename test/kubetest2/{payload.go => bootstrap_ske.go} (68%) rename test/kubetest2/{payload_test.go => bootstrap_ske_test.go} (100%) delete mode 100644 test/kubetest2/bootstrap_test.go rename test/kubetest2/{bootstrap_clients.go => clients.go} (77%) create mode 100644 test/kubetest2/lifecycle_test.go diff --git a/go.mod b/go.mod index 8974e3e3..52e5256b 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 @@ -96,7 +97,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/test/kubetest2/bootstrap_csi.go b/test/kubetest2/bootstrap_csi.go new file mode 100644 index 00000000..728257ac --- /dev/null +++ b/test/kubetest2/bootstrap_csi.go @@ -0,0 +1,15 @@ +package kubetest2 + +import ( + "context" + + "k8s.io/klog/v2" +) + +// ensureCSI deploys the STACKIT CSI driver into the freshly provisioned +// cluster as the last bootstrap step. Not yet implemented; will install the +// CSI Helm chart here. +func (d *Deployer) ensureCSI(_ context.Context) error { + klog.Infof("CSI deployment not yet implemented, skipping") + return nil +} diff --git a/test/kubetest2/bootstrap_project.go b/test/kubetest2/bootstrap_project.go new file mode 100644 index 00000000..cf3a568e --- /dev/null +++ b/test/kubetest2/bootstrap_project.go @@ -0,0 +1,146 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +// ensureProject idempotently resolves (or creates) the managed STACKIT project +// and enables the SKE service for it. Sets d.projectID on success. +func (d *Deployer) ensureProject(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + return d.ensureSKEServiceEnabled(ctx, project.ProjectID) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for i := range projects { + project := &projects[i] + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} diff --git a/test/kubetest2/bootstrap_project_test.go b/test/kubetest2/bootstrap_project_test.go new file mode 100644 index 00000000..901a6aa9 --- /dev/null +++ b/test/kubetest2/bootstrap_project_test.go @@ -0,0 +1,157 @@ +package kubetest2 + +import ( + "context" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) + + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) + +var _ = Describe("ensureProject", func() { + It("resolves the managed project and enables the SKE service", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceEnablementClient := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = serviceEnablementClient + + Expect(d.ensureProject(context.Background())).To(Succeed()) + Expect(d.projectID).To(Equal("project-123")) + Expect(serviceEnablementClient.lastGetStatusProjectID).To(Equal("project-123")) + Expect(serviceEnablementClient.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) + + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) + + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) + + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) diff --git a/test/kubetest2/bootstrap_resources_test.go b/test/kubetest2/bootstrap_resources_test.go deleted file mode 100644 index ca1f0597..00000000 --- a/test/kubetest2/bootstrap_resources_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package kubetest2 - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("matchesManagedProject", func() { - It("matches a project with the expected name and labels", func() { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) - Expect(d.matchesManagedProject(project)).To(BeTrue()) - }) - - It("rejects a project with a different name", func() { - d := newTestDeployer() - project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }) - - It("rejects a project without labels", func() { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", nil) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }) - - DescribeTable("rejects a project with mismatched labels", - func(labels map[string]string) { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", labels) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }, - Entry("wrong scope", map[string]string{ - projectLabelScopeKey: "PRIVATE", - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: runTokenForRun("run-123"), - }), - Entry("missing managed label", map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelRunIDKey: runTokenForRun("run-123"), - }), - Entry("wrong run id", map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: "deadbeef", - }), - ) -}) - -var _ = Describe("matchesManagedServiceAccountEmail", func() { - It("matches the managed service account prefix", func() { - d := newTestDeployer() - Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) - }) - - DescribeTable("rejects non-matching emails", - func(email string) { - d := newTestDeployer() - Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) - }, - Entry("empty email", ""), - Entry("missing at sign", "kt2-no-separator"), - Entry("different local prefix", "other-account@sa.stackit.cloud"), - ) -}) diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_serviceaccount.go similarity index 57% rename from test/kubetest2/bootstrap_resources.go rename to test/kubetest2/bootstrap_serviceaccount.go index e8f0b64a..36683e23 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_serviceaccount.go @@ -3,18 +3,13 @@ package kubetest2 import ( "context" "encoding/json" - "errors" "fmt" - "net/http" "os" "strings" "time" - oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" - resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" - serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) @@ -25,13 +20,6 @@ var childKeyCreationRetryBackoff = wait.Backoff{ Steps: 5, } -type managedProject struct { - ContainerID string - ProjectID string - Name string - Labels map[string]string -} - type managedServiceAccount struct { Email string ProjectID string @@ -58,178 +46,26 @@ type serviceAccountKeyCredentialsFile struct { TokenEndpoint string `json:"tokenEndpoint"` } -func (d *Deployer) initializeBootstrapClients() error { - if d.projectClient == nil { - client, err := newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) - if err != nil { - return err - } - d.projectClient = client - } - if d.serviceAccountClient == nil { - client, err := newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) - if err != nil { - return err - } - d.serviceAccountClient = client - } - if d.authorizationClient == nil { - client, err := newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) - if err != nil { - return err - } - d.authorizationClient = client - } - if d.serviceEnablementClient == nil { - client, err := newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) - if err != nil { - return err - } - d.serviceEnablementClient = client - } - return nil -} - -func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { - if d.skeClientFactory == nil { - d.skeClientFactory = newSKEClient - } - client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) - if err != nil { - return err - } - d.skeClient = client - return nil -} - -func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { - project, err := d.resolveManagedProject(ctx) +// ensureServiceAccount idempotently resolves (or creates) the managed child +// service account, grants it the SKE admin role, caches a service-account key +// and initializes the SKE client from it. Depends on d.projectID being set. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, d.projectID) if err != nil { return err } - d.projectID = project.ProjectID - if err := d.ensureSKEServiceEnabled(ctx, project.ProjectID); err != nil { + if err := d.ensureProjectServiceAccountRole(ctx, d.projectID, childServiceAccount.Email); err != nil { return err } - childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) - if err != nil { - return err - } - d.childServiceAccountEmail = childServiceAccount.Email - - if err := d.ensureProjectServiceAccountRole(ctx, project.ProjectID, childServiceAccount.Email); err != nil { - return err - } - - serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, project.ProjectID, childServiceAccount.Email) + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, d.projectID, childServiceAccount.Email) if err != nil { return err } return d.initializeSKEClient(serviceAccountKey) } -// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) -// service for the managed project and waits until it is enabled. -func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { - status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) - if err != nil { - if !isNotFound(err) { - return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) - } - klog.Infof("SKE service not yet enabled for project_id=%q", projectID) - } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { - klog.Infof("SKE service already enabled for project_id=%q", projectID) - return nil - } else { - klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) - } - - if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { - return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) - } - if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { - return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) - } - return nil -} - -func isNotFound(err error) bool { - var oapiErr *oapierror.GenericOpenAPIError - return errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound -} - -func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { - projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) - if err != nil { - return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) - } - - matches := make([]managedProject, 0, 1) - for i := range projects { - project := &projects[i] - if !d.matchesManagedProject(project) { - continue - } - matches = append(matches, managedProject{ - ContainerID: project.GetContainerId(), - ProjectID: project.GetProjectId(), - Name: project.GetName(), - Labels: project.GetLabels(), - }) - } - - switch len(matches) { - case 0: - return nil, nil - case 1: - return &matches[0], nil - default: - return nil, fmt.Errorf( - "found %d managed STACKIT projects for run token %q under parent container %q", - len(matches), - d.runToken(), - d.parentContainerID, - ) - } -} - -func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { - project, err := d.findManagedProject(ctx) - if err != nil { - return nil, err - } - if project != nil { - klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) - return project, nil - } - - klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) - createdProject, err := d.projectClient.CreateProject( - ctx, - d.parentContainerID, - d.projectName(), - d.projectMemberEmail, - d.managedProjectLabels(), - ) - if err != nil { - return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) - } - - activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) - if err != nil { - return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) - } - - return &managedProject{ - ContainerID: activeProject.GetContainerId(), - ProjectID: activeProject.GetProjectId(), - Name: activeProject.GetName(), - Labels: activeProject.GetLabels(), - }, nil -} - func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) if err != nil { @@ -360,28 +196,10 @@ func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err } func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { - return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) -} - -func (d *Deployer) managedProjectLabels() map[string]string { - return map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: d.runToken(), - } -} - -func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { - if project.GetName() != d.projectName() { - return false - } - labels := project.GetLabels() - if labels == nil { - return false + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return nil } - return labels[projectLabelScopeKey] == projectLabelScopeValue && - labels[projectLabelManagedKey] == projectLabelManagedValue && - labels[projectLabelRunIDKey] == d.runToken() + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) } func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { @@ -392,7 +210,7 @@ func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { if !found { return false } - return strings.HasPrefix(localPart, d.serviceAccountName()) + return localPart == d.serviceAccountName() } func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { diff --git a/test/kubetest2/bootstrap_serviceaccount_test.go b/test/kubetest2/bootstrap_serviceaccount_test.go new file mode 100644 index 00000000..99c23d25 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount_test.go @@ -0,0 +1,128 @@ +package kubetest2 + +import ( + "context" + "errors" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" +) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) + +var _ = Describe("ensureServiceAccount", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + d.projectID = "project-123" + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + }) + + It("creates key and adds membership", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) + + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) + + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) diff --git a/test/kubetest2/payload.go b/test/kubetest2/bootstrap_ske.go similarity index 68% rename from test/kubetest2/payload.go rename to test/kubetest2/bootstrap_ske.go index c629c54e..e0d96e2c 100644 --- a/test/kubetest2/payload.go +++ b/test/kubetest2/bootstrap_ske.go @@ -3,11 +3,49 @@ package kubetest2 import ( "context" "fmt" + "os" "github.com/stackitcloud/stackit-sdk-go/services/ske" "k8s.io/klog/v2" ) +// ensureCluster idempotently validates provider options, then creates or +// updates the SKE cluster, waits until it is ready and writes its kubeconfig. +// Depends on d.projectID and d.skeClient being set. +func (d *Deployer) ensureCluster(ctx context.Context) error { + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + return d.writeKubeconfig(ctx, clusterName) +} + +func (d *Deployer) writeKubeconfig(ctx context.Context, clusterName string) error { + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + return nil +} + func (d *Deployer) validateProviderOptions(ctx context.Context) error { klog.Infof("Validating SKE provider options for region=%q", d.region) @@ -76,7 +114,6 @@ func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { nodepool.SetAllowSystemComponents(true) payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) - klog.Infof("Built SKE cluster payload for cluster=%q", d.clusterName()) return *payload } diff --git a/test/kubetest2/payload_test.go b/test/kubetest2/bootstrap_ske_test.go similarity index 100% rename from test/kubetest2/payload_test.go rename to test/kubetest2/bootstrap_ske_test.go diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go deleted file mode 100644 index 44192f0d..00000000 --- a/test/kubetest2/bootstrap_test.go +++ /dev/null @@ -1,809 +0,0 @@ -package kubetest2 - -import ( - "context" - "errors" - "net/http" - "os" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" - authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" - resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" - serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" - serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" - "github.com/stackitcloud/stackit-sdk-go/services/ske" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/kubetest2/pkg/types" -) - -const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` - -type fakeOptions struct { - runID string - runDir string -} - -func (o fakeOptions) HelpRequested() bool { return false } -func (o fakeOptions) ShouldBuild() bool { return false } -func (o fakeOptions) ShouldUp() bool { return false } -func (o fakeOptions) ShouldDown() bool { return false } -func (o fakeOptions) ShouldTest() bool { return false } -func (o fakeOptions) SkipTestJUnitReport() bool { return false } -func (o fakeOptions) RunID() string { return o.runID } -func (o fakeOptions) RunDir() string { return o.runDir } -func (o fakeOptions) RundirInArtifacts() bool { return false } -func (o fakeOptions) PostTestCmd() []string { return nil } -func (o fakeOptions) PreTestCmd() []string { return nil } - -type fakeProjectClient struct { - listProjectsResult []resourcemanager.Project - listProjectsErr error - - createProjectResult *resourcemanager.Project - createProjectErr error - waitActiveResult *resourcemanager.GetProjectResponse - waitActiveErr error - - deleteErr error - waitDeletedErr error - - createCalls int - waitActiveCalls int - deleteCalls int - waitDeletedCalls int - - lastListParentContainerID string - lastCreateParentContainerID string - lastCreateName string - lastCreateOwnerEmail string - lastCreateLabels map[string]string - lastWaitActiveContainerID string - lastDeletedProjectID string - lastWaitDeletedProjectID string -} - -func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { - c.lastListParentContainerID = parentContainerID - return c.listProjectsResult, c.listProjectsErr -} - -func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { - c.createCalls++ - c.lastCreateParentContainerID = parentContainerID - c.lastCreateName = name - c.lastCreateOwnerEmail = ownerEmail - c.lastCreateLabels = labels - return c.createProjectResult, c.createProjectErr -} - -func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { - c.waitActiveCalls++ - c.lastWaitActiveContainerID = containerID - return c.waitActiveResult, c.waitActiveErr -} - -func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { - c.deleteCalls++ - c.lastDeletedProjectID = projectID - return c.deleteErr -} - -func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { - c.waitDeletedCalls++ - c.lastWaitDeletedProjectID = projectID - return c.waitDeletedErr -} - -type fakeServiceAccountClient struct { - listResult []serviceaccount.ServiceAccount - listErr error - - createResult *serviceaccount.ServiceAccount - createErr error - - createKeyResult *serviceaccount.CreateServiceAccountKeyResponse - createKeyErr error - - createCalls int - createKeyCalls int - - lastProjectIDForList string - lastProjectIDForCreate string - lastCreatedName string - lastProjectIDForCreateKey string - lastCreateKeyEmail string -} - -func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { - c.lastProjectIDForList = projectID - return c.listResult, c.listErr -} - -func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { - c.createCalls++ - c.lastProjectIDForCreate = projectID - c.lastCreatedName = name - return c.createResult, c.createErr -} - -func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { - c.createKeyCalls++ - c.lastProjectIDForCreateKey = projectID - c.lastCreateKeyEmail = serviceAccountEmail - return c.createKeyResult, c.createKeyErr -} - -type fakeAuthorizationClient struct { - listMembersResult []authorization.Member - listMembersErr error - addMembersErr error - - addCalls int - - lastResourceType string - lastResourceID string - lastAddedType string - lastAddedID string - lastAddedMembers []authorization.Member -} - -func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { - c.lastResourceType = resourceType - c.lastResourceID = resourceID - return c.listMembersResult, c.listMembersErr -} - -func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { - c.addCalls++ - c.lastAddedID = resourceID - c.lastAddedType = resourceType - c.lastAddedMembers = members - return c.addMembersErr -} - -type fakeServiceEnablementClient struct { - getStatusResult *serviceenablement.ServiceStatus - getStatusErr error - enableErr error - waitErr error - - enableCalls int - waitCalls int - - lastGetStatusRegion string - lastGetStatusProjectID string - lastGetStatusServiceID string - lastEnableRegion string - lastEnableProjectID string - lastEnableServiceID string - lastWaitRegion string - lastWaitProjectID string - lastWaitServiceID string -} - -func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { - c.lastGetStatusRegion = region - c.lastGetStatusProjectID = projectID - c.lastGetStatusServiceID = serviceID - return c.getStatusResult, c.getStatusErr -} - -func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { - c.enableCalls++ - c.lastEnableRegion = region - c.lastEnableProjectID = projectID - c.lastEnableServiceID = serviceID - return c.enableErr -} - -func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { - c.waitCalls++ - c.lastWaitRegion = region - c.lastWaitProjectID = projectID - c.lastWaitServiceID = serviceID - return c.waitErr -} - -type fakeSKEClient struct { - providerOptions *ske.ProviderOptions - providerOptionsErr error - getClusterResult *ske.Cluster - getClusterErr error - - createOrUpdateResult *ske.Cluster - createOrUpdateErr error - waitReadyResult *ske.Cluster - waitReadyErr error - kubeconfigResult *ske.Kubeconfig - kubeconfigErr error - - deleteClusterCalled bool - waitDeletedCalled bool - - lastGetProjectID string - lastCreateProjectID string - lastCreateRegion string - lastCreateClusterName string - lastKubeconfigProjectID string - lastExpirationSeconds int64 -} - -func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { - c.lastGetProjectID = projectID - return c.getClusterResult, c.getClusterErr -} - -func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { - return c.providerOptions, c.providerOptionsErr -} - -func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { - c.lastCreateProjectID = projectID - c.lastCreateRegion = region - c.lastCreateClusterName = name - return c.createOrUpdateResult, c.createOrUpdateErr -} - -func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { - return c.waitReadyResult, c.waitReadyErr -} - -func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { - c.lastKubeconfigProjectID = projectID - c.lastExpirationSeconds = expirationSeconds - return c.kubeconfigResult, c.kubeconfigErr -} - -func (c *fakeSKEClient) DeleteCluster(_ context.Context, _, _, _ string) error { - c.deleteClusterCalled = true - return nil -} - -func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) error { - c.waitDeletedCalled = true - return nil -} - -var _ = Describe("loadEnvironment", func() { - envVarKeys := []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } - - DescribeTable("validates required environment variables", - func(env map[string]string, wantErrContains string) { - for _, key := range envVarKeys { - setEnvVar(key, "") - } - for key, value := range env { - setEnvVar(key, value) - } - - runDir := GinkgoT().TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - err := d.loadEnvironment() - if wantErrContains != "" { - Expect(err).To(MatchError(ContainSubstring(wantErrContains))) - return - } - Expect(err).NotTo(HaveOccurred()) - Expect(d.parentContainerID).To(Equal("parent-1")) - Expect(d.projectMemberEmail).To(Equal("owner@example.com")) - Expect(d.projectID).To(BeEmpty()) - Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) - Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) - }, - Entry("missing service account", map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, "STACKIT_SERVICE_ACCOUNT"), - Entry("missing parent container", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - }, "STACKIT_PARENT_CONTAINER_ID"), - Entry("invalid service account key", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, "invalid STACKIT_SERVICE_ACCOUNT"), - Entry("project id no longer required", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, ""), - ) - - It("reads optional endpoints", func() { - for _, key := range envVarKeys { - setEnvVar(key, "") - } - setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) - setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") - setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") - setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") - setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") - setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") - - runDir := GinkgoT().TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - Expect(d.loadEnvironment()).To(Succeed()) - - Expect(d.projectMemberEmail).To(Equal("owner@example.com")) - Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) - Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) - Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) - Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) - }) -}) - -var _ = Describe("resolveManagedProject", func() { - It("creates a project when missing", func() { - d := newTestDeployer() - projectClient := &fakeProjectClient{ - createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - } - d.projectClient = projectClient - - project, err := d.resolveManagedProject(context.Background()) - Expect(err).NotTo(HaveOccurred()) - Expect(project.ProjectID).To(Equal("project-123")) - Expect(projectClient.createCalls).To(Equal(1)) - Expect(projectClient.waitActiveCalls).To(Equal(1)) - Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) - Expect(projectClient.lastCreateName).To(Equal(d.projectName())) - Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) - Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) - }) - - It("errors on multiple matches", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), - *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), - }, - } - - _, err := d.findManagedProject(context.Background()) - Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) - }) -}) - -var _ = Describe("ensureManagedClusterAccess", func() { - It("reuses cached key and skips membership write", func() { - d := newTestDeployer() - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - authorizationClient := &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - fakeSKE := &fakeSKEClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return fakeSKE, nil - } - - Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) - Expect(receivedKey).To(Equal(cachedKey)) - Expect(authorizationClient.addCalls).To(Equal(0)) - Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) - Expect(d.projectID).To(Equal("project-123")) - }) - - It("creates key and adds membership", func() { - d := newTestDeployer() - - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), - } - authorizationClient := &fakeAuthorizationClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return &fakeSKEClient{}, nil - } - - Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) - Expect(authorizationClient.addCalls).To(Equal(1)) - Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) - Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) - Expect(authorizationClient.lastAddedID).To(Equal("project-123")) - - keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) - Expect(err).NotTo(HaveOccurred()) - Expect(receivedKey).To(Equal(string(keyBytes))) - Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) - - info, err := os.Stat(d.serviceAccountKeyPath) - Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) - }) -}) - -var _ = Describe("ensureSKEServiceEnabled", func() { - It("skips when already enabled", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(0)) - Expect(client.waitCalls).To(Equal(0)) - Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) - }) - - It("enables when not found", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(1)) - Expect(client.waitCalls).To(Equal(1)) - Expect(client.lastEnableProjectID).To(Equal("project-123")) - Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) - }) - - It("enables when disabled", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusResult: disabledServiceStatusFixture(), - } - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(1)) - Expect(client.waitCalls).To(Equal(1)) - }) - - It("fails on get status error", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, - } - d.serviceEnablementClient = client - - err := d.ensureSKEServiceEnabled(context.Background(), "project-123") - Expect(err).To(HaveOccurred()) - Expect(client.enableCalls).To(Equal(0)) - }) -}) - -var _ = Describe("retryWithBackoff", func() { - It("retries until success", func() { - calls := 0 - result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { - calls++ - if calls < 2 { - return "", errors.New("transient") - } - return "done", nil - }) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal("done")) - Expect(calls).To(Equal(2)) - }) - - It("returns last error when exhausted", func() { - calls := 0 - _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { - calls++ - return 0, errors.New("always fails") - }) - Expect(err).To(HaveOccurred()) - Expect(calls).To(Equal(3)) - }) -}) - -var _ = Describe("Up", func() { - It("uses discovered project and writes kubeconfig", func() { - d := newTestDeployer() - configureValidUpInputs(d) - - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - d.serviceAccountClient = &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - d.authorizationClient = &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - - fakeSKE := &fakeSKEClient{ - providerOptions: providerOptionsFixture(), - createOrUpdateResult: ske.NewClusterWithDefaults(), - waitReadyResult: ske.NewClusterWithDefaults(), - kubeconfigResult: func() *ske.Kubeconfig { - cfg := ske.NewKubeconfig() - cfg.SetKubeconfig("apiVersion: v1\n") - return cfg - }(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - Expect(serviceAccount).To(Equal(cachedKey)) - return fakeSKE, nil - } - - Expect(d.Up()).To(Succeed()) - Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) - Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) - - kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) - Expect(err).NotTo(HaveOccurred()) - Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) - }) -}) - -var _ = Describe("Down", func() { - It("succeeds when project is missing", func() { - d := newTestDeployer() - projectClient := &fakeProjectClient{} - d.projectClient = projectClient - - Expect(d.Down()).To(Succeed()) - Expect(projectClient.deleteCalls).To(Equal(0)) - }) - - It("deletes project without touching cluster delete", func() { - d := newTestDeployer() - fakeSKE := &fakeSKEClient{} - d.skeClient = fakeSKE - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - - Expect(d.Down()).To(Succeed()) - projectClient := d.projectClient.(*fakeProjectClient) - Expect(projectClient.deleteCalls).To(Equal(1)) - Expect(projectClient.waitDeletedCalls).To(Equal(1)) - Expect(fakeSKE.deleteClusterCalled).To(BeFalse()) - Expect(fakeSKE.waitDeletedCalled).To(BeFalse()) - }) -}) - -var _ = Describe("IsUp", func() { - It("returns false when no project", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{} - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeFalse()) - }) - - It("returns error when project without cached key", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - - _, err := d.IsUp() - Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) - }) - - It("queries cluster when project with cached key", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - fakeSKE := &fakeSKEClient{ - getClusterResult: healthyClusterFixture(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - Expect(serviceAccount).To(Equal(cachedKey)) - return fakeSKE, nil - } - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeTrue()) - Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) - }) - - It("returns false when cluster missing", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - fakeSKE := &fakeSKEClient{ - getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.skeClientFactory = func(_, _, _ string) (skeClient, error) { - return fakeSKE, nil - } - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeFalse()) - }) -}) - -func newTestDeployer() *Deployer { - runDir := GinkgoT().TempDir() - return &Deployer{ - options: fakeOptions{runID: "run-123", runDir: runDir}, - region: defaultRegion, - parentContainerID: "parent-123", - projectMemberEmail: "owner@example.com", - kubeconfigPath: filepath.Join(runDir, "kubeconfig"), - serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), - } -} - -func configureValidUpInputs(d *Deployer) { - d.kubernetesVersion = "1.31.0" - d.availabilityZone = defaultAvailabilityZone - d.machineType = "g1.2" - d.nodeImageName = "ubuntu" - d.nodeImageVersion = "v1" - d.nodepoolName = defaultNodepoolName - d.volumeType = "storage" -} - -func setEnvVar(key, value string) { - Expect(os.Setenv(key, value)).To(Succeed()) - DeferCleanup(os.Unsetenv, key) -} - -func providerOptionsFixture() *ske.ProviderOptions { - kubernetesVersion := ske.NewKubernetesVersion() - kubernetesVersion.SetVersion("1.31.0") - - availabilityZone := ske.NewAvailabilityZone() - availabilityZone.SetName(defaultAvailabilityZone) - - machineType := ske.NewMachineType() - machineType.SetName("g1.2") - - imageVersion := ske.NewMachineImageVersion() - imageVersion.SetVersion("v1") - - machineImage := ske.NewMachineImage() - machineImage.SetName("ubuntu") - machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) - - volumeType := ske.NewVolumeType() - volumeType.SetName("storage") - - providerOptions := ske.NewProviderOptions() - providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) - providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) - providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) - providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) - providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) - return providerOptions -} - -func healthyClusterFixture() *ske.Cluster { - cluster := ske.NewClusterWithDefaults() - status := ske.NewClusterStatus() - status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) - cluster.SetStatus(*status) - return cluster -} - -func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { - status := serviceenablement.NewServiceStatus() - state := serviceenablement.SERVICESTATUSSTATE_DISABLED - status.State = &state - return status -} - -func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { - project := resourcemanager.NewProjectWithDefaults() - project.SetName(name) - project.SetProjectId(projectID) - project.SetContainerId(containerID) - project.SetLabels(labels) - return project -} - -func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { - project := resourcemanager.NewGetProjectResponseWithDefaults() - project.SetName(name) - project.SetProjectId(projectID) - project.SetContainerId(containerID) - project.SetLabels(labels) - return project -} - -func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { - serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() - serviceAccountObject.SetEmail(email) - serviceAccountObject.SetProjectId(projectID) - serviceAccountObject.SetId("service-account-id") - serviceAccountObject.SetInternal(false) - return serviceAccountObject -} - -func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { - credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( - "https://accounts.stackit.cloud", - email, - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ) - credentials.SetPrivateKey("PRIVATE") - credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") - - return serviceaccount.NewCreateServiceAccountKeyResponse( - true, - time.Unix(0, 0).UTC(), - *credentials, - "00000000-0000-0000-0000-000000000003", - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, - "PUBLIC KEY", - ) -} - -var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/clients.go similarity index 77% rename from test/kubetest2/bootstrap_clients.go rename to test/kubetest2/clients.go index d70eea6c..7ca272be 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/clients.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" @@ -15,16 +14,16 @@ import ( ) const ( - projectLabelScopeKey = "scope" - projectLabelScopeValue = "PUBLIC" - projectLabelManagedKey = "kt2_managed" - projectLabelManagedValue = "true" - projectLabelRunIDKey = "kt2_run_id" - projectOwnerRole = "owner" - projectResourceType = "project" - childProjectRole = "ske.admin" - projectListPageSize float32 = 100 - skeServiceID = "cloud.stackit.ske" + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectRole = "ske.admin" + projectListPageSize = 100 + skeServiceID = "cloud.stackit.ske" ) type projectClient interface { @@ -68,10 +67,9 @@ type sdkServiceEnablementClient struct { api *serviceenablement.APIClient } -func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { +func apiClientOptions(serviceAccountKey, endpoint string) []sdkconfig.ConfigurationOption { opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(apiName)), } if endpoint != "" { opts = append(opts, sdkconfig.WithEndpoint(endpoint)) @@ -87,7 +85,7 @@ func apiEndpointURL(cfg *sdkconfig.Configuration) string { } func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { - apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Resource Manager client: %w", err) } @@ -95,7 +93,7 @@ func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) } func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { - apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceaccount")...) + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Service Account client: %w", err) } @@ -103,7 +101,7 @@ func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccount } func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { - apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "authorization")...) + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Authorization client: %w", err) } @@ -111,7 +109,7 @@ func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationCl } func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { - apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceenablement")...) + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Service Enablement client: %w", err) } @@ -119,23 +117,23 @@ func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnab } func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { - projects := make([]resourcemanager.Project, 0) - var offset float32 + projects := make([]resourcemanager.Project, 0, projectListPageSize) + offset := 0 for { resp, err := c.api.DefaultAPI.ListProjects(ctx). ContainerParentId(parentContainerID). - Offset(offset). - Limit(projectListPageSize). + Offset(float32(offset)). + Limit(float32(projectListPageSize)). Execute() if err != nil { return nil, err } items := resp.GetItems() projects = append(projects, items...) - if len(items) < int(projectListPageSize) { + if len(items) < projectListPageSize { return projects, nil } - offset += float32(len(items)) + offset += len(items) } } @@ -210,3 +208,48 @@ func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) return err } + +func (d *Deployer) initializeBootstrapClients() error { + if err := initializeBootstrapClient(d.projectClient != nil, &d.projectClient, func() (projectClient, error) { + return newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceAccountClient != nil, &d.serviceAccountClient, func() (serviceAccountClient, error) { + return newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.authorizationClient != nil, &d.authorizationClient, func() (authorizationClient, error) { + return newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceEnablementClient != nil, &d.serviceEnablementClient, func() (serviceEnablementClient, error) { + return newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + }); err != nil { + return err + } + return nil +} + +func initializeBootstrapClient[T any](initialized bool, dst *T, build func() (T, error)) error { + if initialized { + return nil + } + client, err := build() + if err != nil { + return err + } + *dst = client + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) + if err != nil { + return err + } + d.skeClient = client + return nil +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 9725dae6..1d92bc2e 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -17,9 +17,9 @@ const ( defaultNodeCount int64 = 2 defaultNodepoolName = "default" defaultVolumeSizeGiB int64 = 100 - defaultKubeconfigExpiration int64 = 3600 - minKubeconfigExpiration int64 = 600 - maxKubeconfigExpiration int64 = 15552000 + defaultKubeconfigExpiration int64 = 6 * 60 * 60 // 6 hours + minKubeconfigExpiration int64 = 10 * 60 // 10 minutes + maxKubeconfigExpiration int64 = 180 * 24 * 60 * 60 // 180 days ) type Deployer struct { @@ -37,13 +37,12 @@ type Deployer struct { volumeType string kubeconfigExpiresIn int64 - projectID string - serviceAccount string - parentContainerID string - projectMemberEmail string - childServiceAccountEmail string - kubeconfigPath string - serviceAccountKeyPath string + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + kubeconfigPath string + serviceAccountKeyPath string resourceManagerEndpoint string serviceAccountEndpoint string diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go index 267acdae..212d2de7 100644 --- a/test/kubetest2/env_test.go +++ b/test/kubetest2/env_test.go @@ -1,6 +1,8 @@ package kubetest2 import ( + "path/filepath" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -22,3 +24,76 @@ var _ = Describe("extractServiceAccountEmail", func() { Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), ) }) + +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SERVICE_ENABLEMENT_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + } + + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + for key, value := range env { + setEnvVar(key, value) + } + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing service account", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "STACKIT_SERVICE_ACCOUNT"), + Entry("missing parent container", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("project id no longer required", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) + + It("reads optional endpoints", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + }) +}) diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go index a30f4dd9..8a35723d 100644 --- a/test/kubetest2/lifecycle.go +++ b/test/kubetest2/lifecycle.go @@ -3,13 +3,18 @@ package kubetest2 import ( "context" "fmt" - "os" + "path/filepath" "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" "github.com/stackitcloud/stackit-sdk-go/services/ske" "k8s.io/klog/v2" ) +type bootstrapStep struct { + name string + fn func(context.Context) error +} + func (d *Deployer) Init() error { klog.Infof("Initializing STACKIT kubetest2 deployer") @@ -33,37 +38,20 @@ func (d *Deployer) Up() error { klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) ctx := context.Background() - if err := d.ensureManagedClusterAccess(ctx); err != nil { - return err - } - if err := d.validateProviderOptions(ctx); err != nil { - return err - } - - clusterName := d.clusterName() - payload := d.clusterPayload() - - klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) - if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { - return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) - } - klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) - - if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { - return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) - } - - klog.Infof("Creating kubeconfig for cluster=%q", clusterName) - kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) - if err != nil { - return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) - } - klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) - if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { - return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + for _, step := range []bootstrapStep{ + {"project", d.ensureProject}, + {"service account", d.ensureServiceAccount}, + {"cluster", d.ensureCluster}, + {"csi", d.ensureCSI}, + } { + klog.Infof("Bootstrap step %q starting", step.name) + if err := step.fn(ctx); err != nil { + return fmt.Errorf("bootstrap step %q: %w", step.name, err) + } + klog.Infof("Bootstrap step %q completed", step.name) } - klog.Infof("Cluster up flow completed successfully for cluster=%q", clusterName) + klog.Infof("Cluster up flow completed successfully for cluster=%q", d.clusterName()) return nil } @@ -149,7 +137,7 @@ func (d *Deployer) IsUp() (bool, error) { func (d *Deployer) Kubeconfig() (string, error) { if d.kubeconfigPath == "" { - d.kubeconfigPath = d.options.RunDir() + "/kubeconfig" + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") } klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) return d.kubeconfigPath, nil diff --git a/test/kubetest2/lifecycle_test.go b/test/kubetest2/lifecycle_test.go new file mode 100644 index 00000000..599505ea --- /dev/null +++ b/test/kubetest2/lifecycle_test.go @@ -0,0 +1,160 @@ +package kubetest2 + +import ( + "net/http" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" +) + +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) + + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) + }) +}) + +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) + + It("deletes project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + }) +}) + +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) + + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) + + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) + + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go index 749e76aa..9a8c1c6b 100644 --- a/test/kubetest2/naming.go +++ b/test/kubetest2/naming.go @@ -5,36 +5,23 @@ import ( "encoding/hex" ) -func clusterNameForRun(runID string) string { - sum := sha256.Sum256([]byte(runID)) - return "kt2" + hex.EncodeToString(sum[:4]) -} - func runTokenForRun(runID string) string { sum := sha256.Sum256([]byte(runID)) return hex.EncodeToString(sum[:4]) } -func projectNameForRun(runID string) string { - return "kt2-" + runTokenForRun(runID) -} - -func serviceAccountNameForRun(runID string) string { - return "kt2-" + runTokenForRun(runID) +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) } func (d *Deployer) clusterName() string { - return clusterNameForRun(d.options.RunID()) -} - -func (d *Deployer) runToken() string { - return runTokenForRun(d.options.RunID()) + return "kt2" + d.runToken() } func (d *Deployer) projectName() string { - return projectNameForRun(d.options.RunID()) + return "kt2-" + d.runToken() } func (d *Deployer) serviceAccountName() string { - return serviceAccountNameForRun(d.options.RunID()) + return "kt2-" + d.runToken() } diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go index d56060f6..ae9d6a45 100644 --- a/test/kubetest2/ske_client.go +++ b/test/kubetest2/ske_client.go @@ -16,8 +16,6 @@ type skeClient interface { CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) - DeleteCluster(ctx context.Context, projectID, region, name string) error - WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error } type sdkSKEClient struct { @@ -27,7 +25,7 @@ type sdkSKEClient struct { func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) - apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint, "ske")...) + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint)...) if err != nil { return nil, fmt.Errorf("create SKE client: %w", err) } @@ -76,15 +74,3 @@ func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() } - -func (c *sdkSKEClient) DeleteCluster(ctx context.Context, projectID, region, name string) error { - klog.Infof("SKE DeleteCluster: project_id=%q region=%q cluster=%q", projectID, region, name) - _, err := c.api.DeleteCluster(ctx, projectID, region, name).Execute() - return err -} - -func (c *sdkSKEClient) WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error { - klog.Infof("Waiting for SKE cluster deletion: project_id=%q region=%q cluster=%q", projectID, region, name) - _, err := skewait.DeleteClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) - return err -} diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go index 52d96754..f16aed5f 100644 --- a/test/kubetest2/suite_test.go +++ b/test/kubetest2/suite_test.go @@ -1,13 +1,381 @@ package kubetest2 import ( + "context" + "os" + "path/filepath" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" ) func TestKubetest2(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Kubetest2 Suite") } + +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + skeClientFactory: newSKEClient, + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" +} + +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +var _ types.Options = fakeOptions{}