diff --git a/.gitignore b/.gitignore index 41e3565d..63550e4f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ __debug_* test/e2e/inventory* test/e2e/sa-key* test/e2e/kubeconfig* + +# kubetest2 +_artifacts +_rundir diff --git a/Makefile b/Makefile index 18f94467..4b1bc2d9 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ modules: ## Runs go mod to ensure modules are up to date. .PHONY: test test: ## Run tests. - ./hack/test.sh ./cmd/... ./pkg/... + ./hack/test.sh ./cmd/... ./pkg/... ./test/... .PHONY: test-cover test-cover: ## Run tests with coverage. diff --git a/go.mod b/go.mod index 90348018..52e5256b 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,13 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 + github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 + github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 + github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -31,6 +36,7 @@ require ( k8s.io/klog/v2 v2.140.0 k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -81,6 +87,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/term v0.5.2 // indirect @@ -90,7 +97,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 85b2cbc1..56f24f10 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= @@ -184,12 +186,20 @@ github.com/stackitcloud/cloud-provider v0.36.0-ske-2 h1:OCDooD0twsMDdcGbcEEofRa6 github.com/stackitcloud/cloud-provider v0.36.0-ske-2/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= github.com/stackitcloud/stackit-sdk-go/core v0.26.0 h1:jQEb9gkehfp6VCP6TcYk7BI10cz4l0KM2L6hqYBH2QA= github.com/stackitcloud/stackit-sdk-go/core v0.26.0/go.mod h1:WU1hhxnjXw2EV7CYa1nlEvNpMiRY6CvmIOaHuL3pOaA= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 h1:TtpVXxgCJ17AqSOIzooQru/H3QnC4Z2VIF7HbaeKeJs= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1/go.mod h1:c9JimEdshtcIHAI2j2N388ybW+InO1fvYPvL6rxb2uY= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 h1:P8XhOi8TLA7NbLHhxU8dCJjgQpha/65agos3pI59+vw= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1/go.mod h1:/QpOsaCOQjwGgQqG8y9xXqryp25AfkorPjInX9xIeOU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD80TlFNDTKSgaEvV/o7LevTrATqcDL0r4nuhU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 h1:TNZHrunhsXRbuqZcucLs2Gqy1sEyvabufM7pB5Tscmo= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0/go.mod h1:fXq3TmVLb4JMSve989NFFViMFoYa83s7M3hJWgN6mdQ= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -369,6 +379,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/test/kubetest2.go b/test/kubetest2.go new file mode 100644 index 00000000..a8e9be55 --- /dev/null +++ b/test/kubetest2.go @@ -0,0 +1,11 @@ +package main + +import ( + "sigs.k8s.io/kubetest2/pkg/app" + + kubetest2stackit "github.com/stackitcloud/cloud-provider-stackit/test/kubetest2" +) + +func main() { + app.Main(kubetest2stackit.Name, kubetest2stackit.New) +} diff --git a/test/kubetest2/bootstrap_csi.go b/test/kubetest2/bootstrap_csi.go new file mode 100644 index 00000000..728257ac --- /dev/null +++ b/test/kubetest2/bootstrap_csi.go @@ -0,0 +1,15 @@ +package kubetest2 + +import ( + "context" + + "k8s.io/klog/v2" +) + +// ensureCSI deploys the STACKIT CSI driver into the freshly provisioned +// cluster as the last bootstrap step. Not yet implemented; will install the +// CSI Helm chart here. +func (d *Deployer) ensureCSI(_ context.Context) error { + klog.Infof("CSI deployment not yet implemented, skipping") + return nil +} diff --git a/test/kubetest2/bootstrap_project.go b/test/kubetest2/bootstrap_project.go new file mode 100644 index 00000000..cf3a568e --- /dev/null +++ b/test/kubetest2/bootstrap_project.go @@ -0,0 +1,146 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +// ensureProject idempotently resolves (or creates) the managed STACKIT project +// and enables the SKE service for it. Sets d.projectID on success. +func (d *Deployer) ensureProject(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + return d.ensureSKEServiceEnabled(ctx, project.ProjectID) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for i := range projects { + project := &projects[i] + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} diff --git a/test/kubetest2/bootstrap_project_test.go b/test/kubetest2/bootstrap_project_test.go new file mode 100644 index 00000000..901a6aa9 --- /dev/null +++ b/test/kubetest2/bootstrap_project_test.go @@ -0,0 +1,157 @@ +package kubetest2 + +import ( + "context" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) + + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) + +var _ = Describe("ensureProject", func() { + It("resolves the managed project and enables the SKE service", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceEnablementClient := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = serviceEnablementClient + + Expect(d.ensureProject(context.Background())).To(Succeed()) + Expect(d.projectID).To(Equal("project-123")) + Expect(serviceEnablementClient.lastGetStatusProjectID).To(Equal("project-123")) + Expect(serviceEnablementClient.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) + + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) + + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) + + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) diff --git a/test/kubetest2/bootstrap_serviceaccount.go b/test/kubetest2/bootstrap_serviceaccount.go new file mode 100644 index 00000000..36683e23 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount.go @@ -0,0 +1,249 @@ +package kubetest2 + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" +) + +var childKeyCreationRetryBackoff = wait.Backoff{ + Duration: 3 * time.Second, + Factor: 2.0, + Steps: 5, +} + +type managedServiceAccount struct { + Email string + ProjectID string +} + +type serviceAccountKeyFile struct { + Active bool `json:"active"` + CreatedAt time.Time `json:"createdAt"` + Credentials serviceAccountKeyCredentialsFile `json:"credentials"` + ID string `json:"id"` + KeyAlgorithm string `json:"keyAlgorithm"` + KeyOrigin string `json:"keyOrigin"` + KeyType string `json:"keyType"` + PublicKey string `json:"publicKey"` + ValidUntil *time.Time `json:"validUntil,omitempty"` +} + +type serviceAccountKeyCredentialsFile struct { + Aud string `json:"aud"` + Iss string `json:"iss"` + Kid string `json:"kid"` + PrivateKey *string `json:"privateKey,omitempty"` + Sub string `json:"sub"` + TokenEndpoint string `json:"tokenEndpoint"` +} + +// ensureServiceAccount idempotently resolves (or creates) the managed child +// service account, grants it the SKE admin role, caches a service-account key +// and initializes the SKE client from it. Depends on d.projectID being set. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, d.projectID) + if err != nil { + return err + } + + if err := d.ensureProjectServiceAccountRole(ctx, d.projectID, childServiceAccount.Email); err != nil { + return err + } + + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, d.projectID, childServiceAccount.Email) + if err != nil { + return err + } + return d.initializeSKEClient(serviceAccountKey) +} + +func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { + serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("list service accounts in STACKIT project %q: %w", projectID, err) + } + + matches := make([]managedServiceAccount, 0, 1) + for _, serviceAccount := range serviceAccounts { + if !d.matchesManagedServiceAccountEmail(serviceAccount.GetEmail()) { + continue + } + matches = append(matches, managedServiceAccount{ + Email: serviceAccount.GetEmail(), + ProjectID: serviceAccount.GetProjectId(), + }) + } + + switch len(matches) { + case 0: + klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) + createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) + if err != nil { + return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) + } + return &managedServiceAccount{ + Email: createdServiceAccount.GetEmail(), + ProjectID: createdServiceAccount.GetProjectId(), + }, nil + case 1: + klog.Infof("Reusing managed service account=%q in project_id=%q", matches[0].Email, projectID) + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed service accounts for run token %q in project %q", + len(matches), + d.runToken(), + projectID, + ) + } +} + +func (d *Deployer) ensureProjectServiceAccountRole(ctx context.Context, projectID, serviceAccountEmail string) error { + members, err := d.authorizationClient.ListMembers(ctx, projectResourceType, projectID) + if err != nil { + return fmt.Errorf("list members for STACKIT project %q: %w", projectID, err) + } + for _, member := range members { + if member.GetSubject() == serviceAccountEmail && member.GetRole() == childProjectRole { + klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, childProjectRole, projectID) + return nil + } + } + + klog.Infof("Adding role=%q for managed service account=%q in project_id=%q", childProjectRole, serviceAccountEmail, projectID) + if err := d.authorizationClient.AddMembers( + ctx, + projectID, + projectResourceType, + []authorization.Member{*authorization.NewMember(childProjectRole, serviceAccountEmail)}, + ); err != nil { + return fmt.Errorf("add role %q for service account %q in STACKIT project %q: %w", childProjectRole, serviceAccountEmail, projectID, err) + } + return nil +} + +func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (string, error) { + cachedKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return "", fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if ok { + klog.Infof("Reusing cached child service-account key %q", d.serviceAccountKeyPath) + return cachedKey, nil + } + + klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) + createdKey, err := retryWithBackoff(ctx, childKeyCreationRetryBackoff, func() (*serviceaccount.CreateServiceAccountKeyResponse, error) { + return d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + }) + if err != nil { + return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) + } + keyJSON, err := serviceAccountKeyJSON(createdKey) + if err != nil { + return "", fmt.Errorf("serialize service-account key for %q: %w", serviceAccountEmail, err) + } + if err := d.writeCachedChildServiceAccountKey(keyJSON); err != nil { + return "", fmt.Errorf("write cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + return keyJSON, nil +} + +// retryWithBackoff retries fn until it succeeds or the backoff is exhausted. +// It is used for operations that may fail transiently, e.g. while a freshly +// created service account is still propagating through the STACKIT IAM and is +// not yet ready to have a key created for it. +func retryWithBackoff[T any](ctx context.Context, backoff wait.Backoff, fn func() (T, error)) (T, error) { + var result T + var lastErr error + + waitErr := wait.ExponentialBackoffWithContext(ctx, backoff, func(_ context.Context) (bool, error) { + val, err := fn() + if err != nil { + lastErr = err + return false, nil + } + result = val + return true, nil + }) + if waitErr != nil { + return result, fmt.Errorf("backoff failed: %w, last error: %v", waitErr, lastErr) + } + return result, nil +} + +func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return "", false, nil + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + return string(keyBytes), true, nil +} + +func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return nil + } + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) +} + +func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { + if email == "" { + return false + } + localPart, _, found := strings.Cut(email, "@") + if !found { + return false + } + return localPart == d.serviceAccountName() +} + +func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { + credentials := createdKey.GetCredentials() + privateKey, ok := credentials.GetPrivateKeyOk() + if !ok || strings.TrimSpace(*privateKey) == "" { + return "", fmt.Errorf("service-account key response did not include a private key") + } + + serviceAccountKey := serviceAccountKeyFile{ + Active: createdKey.GetActive(), + CreatedAt: createdKey.GetCreatedAt(), + Credentials: serviceAccountKeyCredentialsFile{ + Aud: credentials.GetAud(), + Iss: credentials.GetIss(), + Kid: credentials.GetKid(), + PrivateKey: privateKey, + Sub: credentials.GetSub(), + TokenEndpoint: credentials.GetTokenEndpoint(), + }, + ID: createdKey.GetId(), + KeyAlgorithm: string(createdKey.GetKeyAlgorithm()), + KeyOrigin: string(createdKey.GetKeyOrigin()), + KeyType: string(createdKey.GetKeyType()), + PublicKey: createdKey.GetPublicKey(), + } + if validUntil, ok := createdKey.GetValidUntilOk(); ok { + serviceAccountKey.ValidUntil = validUntil + } + + keyJSON, err := json.Marshal(serviceAccountKey) + if err != nil { + return "", err + } + return string(keyJSON), nil +} diff --git a/test/kubetest2/bootstrap_serviceaccount_test.go b/test/kubetest2/bootstrap_serviceaccount_test.go new file mode 100644 index 00000000..99c23d25 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount_test.go @@ -0,0 +1,128 @@ +package kubetest2 + +import ( + "context" + "errors" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" +) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) + +var _ = Describe("ensureServiceAccount", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + d.projectID = "project-123" + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + }) + + It("creates key and adds membership", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) + + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) + + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) diff --git a/test/kubetest2/bootstrap_ske.go b/test/kubetest2/bootstrap_ske.go new file mode 100644 index 00000000..e0d96e2c --- /dev/null +++ b/test/kubetest2/bootstrap_ske.go @@ -0,0 +1,168 @@ +package kubetest2 + +import ( + "context" + "fmt" + "os" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +// ensureCluster idempotently validates provider options, then creates or +// updates the SKE cluster, waits until it is ready and writes its kubeconfig. +// Depends on d.projectID and d.skeClient being set. +func (d *Deployer) ensureCluster(ctx context.Context) error { + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + return d.writeKubeconfig(ctx, clusterName) +} + +func (d *Deployer) writeKubeconfig(ctx context.Context, clusterName string) error { + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + return nil +} + +func (d *Deployer) validateProviderOptions(ctx context.Context) error { + klog.Infof("Validating SKE provider options for region=%q", d.region) + + options, err := d.skeClient.ListProviderOptions(ctx, d.region) + if err != nil { + return fmt.Errorf("list SKE provider options: %w", err) + } + + if !containsKubernetesVersion(options, d.kubernetesVersion) { + return incorrectUsagef("unsupported --kubernetes-version %q for region %q", d.kubernetesVersion, d.region) + } + if !containsAvailabilityZone(options, d.availabilityZone) { + return incorrectUsagef("unsupported --availability-zone %q for region %q", d.availabilityZone, d.region) + } + if !containsMachineType(options, d.machineType) { + return incorrectUsagef("unsupported --machine-type %q for region %q", d.machineType, d.region) + } + if !containsMachineImage(options, d.nodeImageName, d.nodeImageVersion) { + return incorrectUsagef( + "unsupported node image %q version %q for region %q", + d.nodeImageName, + d.nodeImageVersion, + d.region, + ) + } + if d.volumeType != "" && !containsVolumeType(options, d.volumeType) { + return incorrectUsagef("unsupported --volume-type %q for region %q", d.volumeType, d.region) + } + + klog.Infof("SKE provider option validation succeeded for region=%q", d.region) + + return nil +} + +func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { + klog.Infof( + "Building SKE cluster payload: cluster=%q kubernetes_version=%q availability_zone=%q machine_type=%q image=%q/%q node_count=%d nodepool=%q volume_size=%d volume_type=%q", + d.clusterName(), + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + ) + + clusterKubernetes := ske.NewKubernetes(d.kubernetesVersion) + nodeImage := ske.NewImage(d.nodeImageName, d.nodeImageVersion) + nodeMachine := ske.NewMachine(*nodeImage, d.machineType) + nodeVolume := ske.NewVolume(d.volumeSizeGiB) + if d.volumeType != "" { + nodeVolume.SetType(d.volumeType) + } + + nodepool := ske.NewNodepool( + []string{d.availabilityZone}, + *nodeMachine, + d.nodeCount, + d.nodeCount, + d.nodepoolName, + *nodeVolume, + ) + nodepool.SetAllowSystemComponents(true) + + payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) + return *payload +} + +func containsKubernetesVersion(options *ske.ProviderOptions, version string) bool { + for _, item := range options.GetKubernetesVersions() { + if item.GetVersion() == version { + return true + } + } + return false +} + +func containsAvailabilityZone(options *ske.ProviderOptions, zone string) bool { + for _, item := range options.GetAvailabilityZones() { + if item.GetName() == zone { + return true + } + } + return false +} + +func containsMachineType(options *ske.ProviderOptions, machineType string) bool { + for _, item := range options.GetMachineTypes() { + if item.GetName() == machineType { + return true + } + } + return false +} + +func containsMachineImage(options *ske.ProviderOptions, imageName, imageVersion string) bool { + for _, image := range options.GetMachineImages() { + if image.GetName() != imageName { + continue + } + for _, version := range image.GetVersions() { + if version.GetVersion() == imageVersion { + return true + } + } + } + return false +} + +func containsVolumeType(options *ske.ProviderOptions, volumeType string) bool { + for _, item := range options.GetVolumeTypes() { + if item.GetName() == volumeType { + return true + } + } + return false +} diff --git a/test/kubetest2/bootstrap_ske_test.go b/test/kubetest2/bootstrap_ske_test.go new file mode 100644 index 00000000..32b84a68 --- /dev/null +++ b/test/kubetest2/bootstrap_ske_test.go @@ -0,0 +1,87 @@ +package kubetest2 + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("validateProviderOptions", func() { + It("accepts supported options", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) + + DescribeTable("rejects unsupported options", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + mutate(d) + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("unsupported kubernetes version", func(d *Deployer) { d.kubernetesVersion = "9.9.9" }, "unsupported --kubernetes-version"), + Entry("unsupported availability zone", func(d *Deployer) { d.availabilityZone = "eu99-9" }, "unsupported --availability-zone"), + Entry("unsupported machine type", func(d *Deployer) { d.machineType = "x9.9" }, "unsupported --machine-type"), + Entry("unsupported node image name", func(d *Deployer) { d.nodeImageName = "fedora" }, "unsupported node image"), + Entry("unsupported node image version", func(d *Deployer) { d.nodeImageVersion = "v99" }, "unsupported node image"), + Entry("unsupported volume type", func(d *Deployer) { d.volumeType = "nvme" }, "unsupported --volume-type"), + ) + + It("fails when listing provider options errors", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptionsErr: errors.New("boom")} + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring("list SKE provider options"))) + }) + + It("allows an empty volume type", func() { + d := newValidTestDeployer() + d.volumeType = "" + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) +}) + +var _ = Describe("clusterPayload", func() { + It("builds the expected payload", func() { + d := newValidTestDeployer() + payload := d.clusterPayload() + + kubernetes := payload.GetKubernetes() + Expect(kubernetes.GetVersion()).To(Equal("1.31.0")) + + nodepools := payload.GetNodepools() + Expect(nodepools).To(HaveLen(1)) + + nodepool := nodepools[0] + Expect(nodepool.GetName()).To(Equal(defaultNodepoolName)) + Expect(nodepool.GetAvailabilityZones()).To(ConsistOf(defaultAvailabilityZone)) + + machine := nodepool.GetMachine() + Expect(machine.GetType()).To(Equal("g1.2")) + image := machine.GetImage() + Expect(image.GetName()).To(Equal("ubuntu")) + Expect(image.GetVersion()).To(Equal("v1")) + + Expect(nodepool.GetMinimum()).To(Equal(d.nodeCount)) + Expect(nodepool.GetMaximum()).To(Equal(d.nodeCount)) + + volume := nodepool.GetVolume() + Expect(volume.GetSize()).To(Equal(d.volumeSizeGiB)) + Expect(volume.GetType()).To(Equal("storage")) + + Expect(nodepool.GetAllowSystemComponents()).To(BeTrue()) + }) + + It("omits the volume type when unset", func() { + d := newValidTestDeployer() + d.volumeType = "" + payload := d.clusterPayload() + + Expect(payload.GetNodepools()).To(HaveLen(1)) + volume := payload.GetNodepools()[0].GetVolume() + Expect(volume.GetType()).To(BeEmpty()) + }) +}) diff --git a/test/kubetest2/clients.go b/test/kubetest2/clients.go new file mode 100644 index 00000000..7ca272be --- /dev/null +++ b/test/kubetest2/clients.go @@ -0,0 +1,255 @@ +package kubetest2 + +import ( + "context" + "fmt" + + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + serviceenablementwait "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api/wait" +) + +const ( + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectRole = "ske.admin" + projectListPageSize = 100 + skeServiceID = "cloud.stackit.ske" +) + +type projectClient interface { + ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) + CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) + WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) + DeleteProject(ctx context.Context, projectID string) error + WaitForProjectDeleted(ctx context.Context, projectID string) error +} + +type serviceAccountClient interface { + ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) + CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) + CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) +} + +type authorizationClient interface { + ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) + AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error +} + +type serviceEnablementClient interface { + GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) + EnableService(ctx context.Context, region, projectID, serviceID string) error + WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error +} + +type sdkProjectClient struct { + api *resourcemanager.APIClient +} + +type sdkServiceAccountClient struct { + api *serviceaccount.APIClient +} + +type sdkAuthorizationClient struct { + api *authorization.APIClient +} + +type sdkServiceEnablementClient struct { + api *serviceenablement.APIClient +} + +func apiClientOptions(serviceAccountKey, endpoint string) []sdkconfig.ConfigurationOption { + opts := []sdkconfig.ConfigurationOption{ + sdkconfig.WithServiceAccountKey(serviceAccountKey), + } + if endpoint != "" { + opts = append(opts, sdkconfig.WithEndpoint(endpoint)) + } + return opts +} + +func apiEndpointURL(cfg *sdkconfig.Configuration) string { + if cfg == nil || len(cfg.Servers) == 0 { + return "unknown" + } + return cfg.Servers[0].URL +} + +func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Resource Manager client: %w", err) + } + return &sdkProjectClient{api: apiClient}, nil +} + +func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Service Account client: %w", err) + } + return &sdkServiceAccountClient{api: apiClient}, nil +} + +func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Authorization client: %w", err) + } + return &sdkAuthorizationClient{api: apiClient}, nil +} + +func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create Service Enablement client: %w", err) + } + return &sdkServiceEnablementClient{api: apiClient}, nil +} + +func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + projects := make([]resourcemanager.Project, 0, projectListPageSize) + offset := 0 + for { + resp, err := c.api.DefaultAPI.ListProjects(ctx). + ContainerParentId(parentContainerID). + Offset(float32(offset)). + Limit(float32(projectListPageSize)). + Execute() + if err != nil { + return nil, err + } + items := resp.GetItems() + projects = append(projects, items...) + if len(items) < projectListPageSize { + return projects, nil + } + offset += len(items) + } +} + +func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + payload := resourcemanager.NewCreateProjectPayload( + parentContainerID, + []resourcemanager.Member{*resourcemanager.NewMember(projectOwnerRole, ownerEmail)}, + name, + ) + payload.SetLabels(labels) + return c.api.DefaultAPI.CreateProject(ctx).CreateProjectPayload(*payload).Execute() +} + +func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api.DefaultAPI, containerID).WaitWithContext(ctx) +} + +func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { + return c.api.DefaultAPI.DeleteProject(ctx, projectID).Execute() +} + +func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api.DefaultAPI, projectID).WaitWithContext(ctx) + return err +} + +func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + resp, err := c.api.DefaultAPI.ListServiceAccounts(ctx, projectID).Execute() + if err != nil { + return nil, err + } + return resp.GetItems(), nil +} + +func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + payload := serviceaccount.NewCreateServiceAccountPayload(name) + return c.api.DefaultAPI.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() +} + +func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() + resp, err := c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + if err != nil { + return nil, fmt.Errorf("%w (endpoint: %s)", err, apiEndpointURL(c.api.GetConfig())) + } + return resp, nil +} + +func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + resp, err := c.api.DefaultAPI.ListMembers(ctx, resourceType, resourceID).Execute() + if err != nil { + return nil, err + } + return resp.GetMembers(), nil +} + +func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { + payload := authorization.NewAddMembersPayload(members, resourceType) + _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + return err +} + +func (c *sdkServiceEnablementClient) GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + return c.api.DefaultAPI.GetServiceStatusRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) EnableService(ctx context.Context, region, projectID, serviceID string) error { + return c.api.DefaultAPI.EnableServiceRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error { + _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) + return err +} + +func (d *Deployer) initializeBootstrapClients() error { + if err := initializeBootstrapClient(d.projectClient != nil, &d.projectClient, func() (projectClient, error) { + return newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceAccountClient != nil, &d.serviceAccountClient, func() (serviceAccountClient, error) { + return newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.authorizationClient != nil, &d.authorizationClient, func() (authorizationClient, error) { + return newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceEnablementClient != nil, &d.serviceEnablementClient, func() (serviceEnablementClient, error) { + return newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + }); err != nil { + return err + } + return nil +} + +func initializeBootstrapClient[T any](initialized bool, dst *T, build func() (T, error)) error { + if initialized { + return nil + } + client, err := build() + if err != nil { + return err + } + *dst = client + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) + if err != nil { + return err + } + d.skeClient = client + return nil +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go new file mode 100644 index 00000000..1d92bc2e --- /dev/null +++ b/test/kubetest2/deployer.go @@ -0,0 +1,103 @@ +package kubetest2 + +import ( + "flag" + + "github.com/spf13/pflag" + "github.com/stackitcloud/cloud-provider-stackit/pkg/version" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +const ( + Name = "stackit" + + defaultRegion = "eu01" + defaultAvailabilityZone = "eu01-1" + defaultNodeCount int64 = 2 + defaultNodepoolName = "default" + defaultVolumeSizeGiB int64 = 100 + defaultKubeconfigExpiration int64 = 6 * 60 * 60 // 6 hours + minKubeconfigExpiration int64 = 10 * 60 // 10 minutes + maxKubeconfigExpiration int64 = 180 * 24 * 60 * 60 // 180 days +) + +type Deployer struct { + options types.Options + + region string + kubernetesVersion string + availabilityZone string + machineType string + nodeImageName string + nodeImageVersion string + nodeCount int64 + nodepoolName string + volumeSizeGiB int64 + volumeType string + kubeconfigExpiresIn int64 + + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + kubeconfigPath string + serviceAccountKeyPath string + + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + serviceEnablementEndpoint string + skeEndpoint string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + serviceEnablementClient serviceEnablementClient + skeClient skeClient + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) +} + +var _ types.NewDeployer = New +var _ types.Deployer = &Deployer{} +var _ types.DeployerWithInit = &Deployer{} +var _ types.DeployerWithKubeconfig = &Deployer{} +var _ types.DeployerWithProvider = &Deployer{} +var _ types.DeployerWithVersion = &Deployer{} + +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + d := &Deployer{ + options: opts, + region: defaultRegion, + availabilityZone: defaultAvailabilityZone, + nodeCount: defaultNodeCount, + nodepoolName: defaultNodepoolName, + volumeSizeGiB: defaultVolumeSizeGiB, + kubeconfigExpiresIn: defaultKubeconfigExpiration, + skeClientFactory: newSKEClient, + } + + fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) + bindFlags(fs, d) + + klog.InitFlags(nil) + fs.AddGoFlagSet(flag.CommandLine) + + return d, fs +} + +func (d *Deployer) Provider() string { + return Name +} + +func (d *Deployer) Version() string { + return version.Version +} + +func (d *Deployer) Build() error { + return nil +} + +func (d *Deployer) DumpClusterLogs() error { + return nil +} diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go new file mode 100644 index 00000000..51bfbcfb --- /dev/null +++ b/test/kubetest2/env.go @@ -0,0 +1,73 @@ +package kubetest2 + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +func (d *Deployer) loadEnvironment() error { + klog.Infof("Loading STACKIT environment variables") + + serviceAccount, ok := os.LookupEnv("STACKIT_SERVICE_ACCOUNT") + if !ok || strings.TrimSpace(serviceAccount) == "" { + return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") + } + + parentContainerID, ok := os.LookupEnv("STACKIT_PARENT_CONTAINER_ID") + if !ok || strings.TrimSpace(parentContainerID) == "" { + return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") + } + + projectMemberEmail, err := extractServiceAccountEmail(serviceAccount) + if err != nil { + return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) + } + + d.serviceAccount = serviceAccount + d.parentContainerID = parentContainerID + d.projectMemberEmail = projectMemberEmail + d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) + d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) + d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.serviceEnablementEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ENABLEMENT_ENDPOINT")) + d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") + + klog.Infof( + "Loaded STACKIT environment: parent_container_id=%q project_member_email=%q service_account_bytes=%d kubeconfig_path=%q service_account_key_path=%q", + d.parentContainerID, + d.projectMemberEmail, + len(d.serviceAccount), + d.kubeconfigPath, + d.serviceAccountKeyPath, + ) + + klog.Infof( + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q", + d.resourceManagerEndpoint, + d.serviceAccountEndpoint, + d.authorizationEndpoint, + d.serviceEnablementEndpoint, + d.skeEndpoint, + ) + + return nil +} + +func extractServiceAccountEmail(serviceAccountKey string) (string, error) { + var key serviceAccountKeyFile + if err := json.Unmarshal([]byte(serviceAccountKey), &key); err != nil { + return "", fmt.Errorf("parse service account key: %w", err) + } + email := strings.TrimSpace(key.Credentials.Iss) + if email == "" { + return "", fmt.Errorf("service account key has no email in credentials.iss") + } + return email, nil +} diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go new file mode 100644 index 00000000..212d2de7 --- /dev/null +++ b/test/kubetest2/env_test.go @@ -0,0 +1,99 @@ +package kubetest2 + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extractServiceAccountEmail", func() { + It("extracts the email from the credentials.iss field", func() { + email, err := extractServiceAccountEmail(validServiceAccountKey) + Expect(err).NotTo(HaveOccurred()) + Expect(email).To(Equal("owner@example.com")) + }) + + DescribeTable("rejects invalid service account keys", + func(key string, wantErrContains string) { + _, err := extractServiceAccountEmail(key) + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("malformed JSON", "not-json", "parse service account key"), + Entry("missing credentials.iss", `{"credentials":{"aud":"x"}}`, "no email in credentials.iss"), + Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), + ) +}) + +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SERVICE_ENABLEMENT_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + } + + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + for key, value := range env { + setEnvVar(key, value) + } + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing service account", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "STACKIT_SERVICE_ACCOUNT"), + Entry("missing parent container", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("project id no longer required", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) + + It("reads optional endpoints", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + }) +}) diff --git a/test/kubetest2/flags.go b/test/kubetest2/flags.go new file mode 100644 index 00000000..530610d9 --- /dev/null +++ b/test/kubetest2/flags.go @@ -0,0 +1,88 @@ +package kubetest2 + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func bindFlags(fs *pflag.FlagSet, d *Deployer) { + fs.StringVar(&d.region, "region", defaultRegion, "STACKIT region for the SKE cluster") + fs.StringVar(&d.kubernetesVersion, "kubernetes-version", "", "Kubernetes version for the SKE cluster") + fs.StringVar(&d.availabilityZone, "availability-zone", defaultAvailabilityZone, "Availability zone for the SKE nodepool") + fs.StringVar(&d.machineType, "machine-type", "", "Machine type for the SKE nodepool") + fs.StringVar(&d.nodeImageName, "node-image-name", "", "Node image name for the SKE nodepool") + fs.StringVar(&d.nodeImageVersion, "node-image-version", "", "Node image version for the SKE nodepool") + fs.Int64Var(&d.nodeCount, "node-count", defaultNodeCount, "Node count for the SKE nodepool") + fs.StringVar(&d.nodepoolName, "nodepool-name", defaultNodepoolName, "Nodepool name for the SKE cluster") + fs.Int64Var(&d.volumeSizeGiB, "volume-size", defaultVolumeSizeGiB, "Root volume size in GiB for the SKE nodepool") + fs.StringVar(&d.volumeType, "volume-type", "", "Root volume type for the SKE nodepool") + fs.Int64Var(&d.kubeconfigExpiresIn, "kubeconfig-expiration-seconds", defaultKubeconfigExpiration, "Admin kubeconfig expiration in seconds") +} + +func (d *Deployer) validate() error { + klog.Infof( + "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d", + d.options.RunID(), + d.region, + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + d.kubeconfigExpiresIn, + ) + + requiredFlags := map[string]string{ + "--region": d.region, + "--kubernetes-version": d.kubernetesVersion, + "--availability-zone": d.availabilityZone, + "--machine-type": d.machineType, + "--node-image-name": d.nodeImageName, + "--node-image-version": d.nodeImageVersion, + } + + for flagName, value := range requiredFlags { + if strings.TrimSpace(value) == "" { + return incorrectUsagef("%s is required", flagName) + } + } + + if strings.TrimSpace(d.nodepoolName) == "" { + return incorrectUsagef("--nodepool-name must not be empty") + } + if len(d.nodepoolName) > 15 { + return incorrectUsagef("--nodepool-name must be 15 characters or fewer") + } + if d.nodeCount < 1 { + return incorrectUsagef("--node-count must be greater than 0") + } + if d.volumeSizeGiB < 1 { + return incorrectUsagef("--volume-size must be greater than 0") + } + if d.kubeconfigExpiresIn < minKubeconfigExpiration || d.kubeconfigExpiresIn > maxKubeconfigExpiration { + return incorrectUsagef( + "--kubeconfig-expiration-seconds must be between %d and %d", + minKubeconfigExpiration, + maxKubeconfigExpiration, + ) + } + if strings.TrimSpace(d.options.RunID()) == "" { + return incorrectUsagef("kubetest2 run-id must not be empty") + } + + klog.Infof("Deployer configuration validation succeeded") + + return nil +} + +func incorrectUsagef(format string, args ...any) error { + return types.NewIncorrectUsage(fmt.Sprintf(format, args...)) +} diff --git a/test/kubetest2/flags_test.go b/test/kubetest2/flags_test.go new file mode 100644 index 00000000..b0e9f083 --- /dev/null +++ b/test/kubetest2/flags_test.go @@ -0,0 +1,42 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func newValidTestDeployer() *Deployer { + d := newTestDeployer() + configureValidUpInputs(d) + d.nodeCount = defaultNodeCount + d.volumeSizeGiB = defaultVolumeSizeGiB + d.kubeconfigExpiresIn = defaultKubeconfigExpiration + return d +} + +var _ = Describe("validate", func() { + It("accepts a valid configuration", func() { + Expect(newValidTestDeployer().validate()).To(Succeed()) + }) + + DescribeTable("rejects invalid configuration", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + mutate(d) + Expect(d.validate()).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("missing region", func(d *Deployer) { d.region = "" }, "--region is required"), + Entry("missing kubernetes version", func(d *Deployer) { d.kubernetesVersion = "" }, "--kubernetes-version is required"), + Entry("missing availability zone", func(d *Deployer) { d.availabilityZone = "" }, "--availability-zone is required"), + Entry("missing machine type", func(d *Deployer) { d.machineType = "" }, "--machine-type is required"), + Entry("missing node image name", func(d *Deployer) { d.nodeImageName = "" }, "--node-image-name is required"), + Entry("missing node image version", func(d *Deployer) { d.nodeImageVersion = "" }, "--node-image-version is required"), + Entry("empty nodepool name", func(d *Deployer) { d.nodepoolName = "" }, "--nodepool-name must not be empty"), + Entry("nodepool name too long", func(d *Deployer) { d.nodepoolName = "this-name-is-way-too-long" }, "--nodepool-name must be 15 characters or fewer"), + Entry("zero node count", func(d *Deployer) { d.nodeCount = 0 }, "--node-count must be greater than 0"), + Entry("zero volume size", func(d *Deployer) { d.volumeSizeGiB = 0 }, "--volume-size must be greater than 0"), + Entry("kubeconfig expiration too small", func(d *Deployer) { d.kubeconfigExpiresIn = minKubeconfigExpiration - 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("kubeconfig expiration too large", func(d *Deployer) { d.kubeconfigExpiresIn = maxKubeconfigExpiration + 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("empty run id", func(d *Deployer) { d.options = fakeOptions{runID: "", runDir: d.options.RunDir()} }, "run-id must not be empty"), + ) +}) diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go new file mode 100644 index 00000000..8a35723d --- /dev/null +++ b/test/kubetest2/lifecycle.go @@ -0,0 +1,144 @@ +package kubetest2 + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +type bootstrapStep struct { + name string + fn func(context.Context) error +} + +func (d *Deployer) Init() error { + klog.Infof("Initializing STACKIT kubetest2 deployer") + + if err := d.validate(); err != nil { + return err + } + if err := d.loadEnvironment(); err != nil { + return err + } + + if err := d.initializeBootstrapClients(); err != nil { + return err + } + + klog.Infof("STACKIT kubetest2 deployer initialized successfully") + + return nil +} + +func (d *Deployer) Up() error { + klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) + + ctx := context.Background() + for _, step := range []bootstrapStep{ + {"project", d.ensureProject}, + {"service account", d.ensureServiceAccount}, + {"cluster", d.ensureCluster}, + {"csi", d.ensureCSI}, + } { + klog.Infof("Bootstrap step %q starting", step.name) + if err := step.fn(ctx); err != nil { + return fmt.Errorf("bootstrap step %q: %w", step.name, err) + } + klog.Infof("Bootstrap step %q completed", step.name) + } + + klog.Infof("Cluster up flow completed successfully for cluster=%q", d.clusterName()) + + return nil +} + +func (d *Deployer) Down() error { + ctx := context.Background() + + klog.Infof("Starting cluster down flow for cluster=%q", d.clusterName()) + + project, err := d.findManagedProject(ctx) + if err != nil { + return err + } + if project == nil { + klog.Infof("Managed project for run_id=%q is already absent", d.options.RunID()) + return nil + } + + d.projectID = project.ProjectID + if err := d.projectClient.DeleteProject(ctx, project.ProjectID); err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("delete STACKIT project %q: %w", project.ProjectID, err) + } + klog.Infof("Project=%q already absent, treating delete as success", project.ProjectID) + return nil + } + + if err := d.projectClient.WaitForProjectDeleted(ctx, project.ProjectID); err != nil { + return fmt.Errorf("wait for STACKIT project %q deletion: %w", project.ProjectID, err) + } + + klog.Infof("Cluster down flow completed successfully for project=%q", project.ProjectID) + + return nil +} + +func (d *Deployer) IsUp() (bool, error) { + ctx := context.Background() + project, err := d.findManagedProject(ctx) + if err != nil { + return false, err + } + if project == nil { + klog.Infof("Managed project for run_id=%q not found during IsUp check", d.options.RunID()) + return false, nil + } + + d.projectID = project.ProjectID + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return false, fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok { + return false, fmt.Errorf( + "managed project %q exists but child service-account key cache %q is missing", + project.ProjectID, + d.serviceAccountKeyPath, + ) + } + if err := d.initializeSKEClient(serviceAccountKey); err != nil { + return false, err + } + + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) + cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) + if err != nil { + if stackiterrors.IsNotFound(err) { + klog.Infof("Cluster=%q not found during IsUp check", d.clusterName()) + return false, nil + } + return false, fmt.Errorf("get SKE cluster %q: %w", d.clusterName(), err) + } + + if cluster.Status == nil || cluster.Status.Aggregated == nil { + klog.Infof("Cluster=%q has no aggregated status yet", d.clusterName()) + return false, nil + } + + state := cluster.Status.GetAggregated() + klog.Infof("Cluster=%q current aggregated state=%q", d.clusterName(), state) + return state == ske.CLUSTERSTATUSSTATE_HEALTHY || state == ske.CLUSTERSTATUSSTATE_HIBERNATED, nil +} + +func (d *Deployer) Kubeconfig() (string, error) { + if d.kubeconfigPath == "" { + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + } + klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) + return d.kubeconfigPath, nil +} diff --git a/test/kubetest2/lifecycle_test.go b/test/kubetest2/lifecycle_test.go new file mode 100644 index 00000000..599505ea --- /dev/null +++ b/test/kubetest2/lifecycle_test.go @@ -0,0 +1,160 @@ +package kubetest2 + +import ( + "net/http" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" +) + +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) + + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) + }) +}) + +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) + + It("deletes project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + }) +}) + +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) + + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) + + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) + + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go new file mode 100644 index 00000000..9a8c1c6b --- /dev/null +++ b/test/kubetest2/naming.go @@ -0,0 +1,27 @@ +package kubetest2 + +import ( + "crypto/sha256" + "encoding/hex" +) + +func runTokenForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return hex.EncodeToString(sum[:4]) +} + +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) +} + +func (d *Deployer) clusterName() string { + return "kt2" + d.runToken() +} + +func (d *Deployer) projectName() string { + return "kt2-" + d.runToken() +} + +func (d *Deployer) serviceAccountName() string { + return "kt2-" + d.runToken() +} diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go new file mode 100644 index 00000000..ae9d6a45 --- /dev/null +++ b/test/kubetest2/ske_client.go @@ -0,0 +1,76 @@ +package kubetest2 + +import ( + "context" + "fmt" + "strconv" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" + "k8s.io/klog/v2" +) + +type skeClient interface { + GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) + CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) + WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) +} + +type sdkSKEClient struct { + api ske.DefaultApi +} + +func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { + klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) + + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint)...) + if err != nil { + return nil, fmt.Errorf("create SKE client: %w", err) + } + + klog.Infof("Created SKE API client successfully") + + return &sdkSKEClient{api: apiClient}, nil +} + +func (c *sdkSKEClient) GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("SKE GetCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.GetCluster(ctx, projectID, region, name).Execute() +} + +func (c *sdkSKEClient) ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) { + klog.Infof("SKE ListProviderOptions: region=%q", region) + return c.api.ListProviderOptions(ctx, region).Execute() +} + +func (c *sdkSKEClient) CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + klog.Infof("SKE CreateOrUpdateCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.CreateOrUpdateCluster(ctx, projectID, region, name).CreateOrUpdateClusterPayload(payload).Execute() +} + +func (c *sdkSKEClient) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("Waiting for SKE cluster to become ready: project_id=%q region=%q cluster=%q", projectID, region, name) + cluster, err := skewait.CreateOrUpdateClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + if err != nil { + return nil, err + } + if cluster != nil && cluster.Status != nil && cluster.Status.Aggregated != nil { + klog.Infof("SKE cluster is ready: cluster=%q state=%q", name, cluster.Status.GetAggregated()) + } + return cluster, nil +} + +func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) { + klog.Infof( + "SKE CreateKubeconfig: project_id=%q region=%q cluster=%q expiration_seconds=%d", + projectID, + region, + name, + expirationSeconds, + ) + payload := ske.NewCreateKubeconfigPayload() + payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) + return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() +} diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go new file mode 100644 index 00000000..f16aed5f --- /dev/null +++ b/test/kubetest2/suite_test.go @@ -0,0 +1,381 @@ +package kubetest2 + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func TestKubetest2(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kubetest2 Suite") +} + +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + skeClientFactory: newSKEClient, + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" +} + +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +var _ types.Options = fakeOptions{}