diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/databricks.yml.tmpl b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/databricks.yml.tmpl new file mode 100644 index 00000000000..cad38109b8e --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: lifecycle-started-terraform-error-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + sql_warehouses: + mywarehouse: + name: my-warehouse-$UNIQUE_NAME + cluster_size: "Medium" + lifecycle: + started: true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml new file mode 100644 index 00000000000..88a060050c3 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/output.txt new file mode 100644 index 00000000000..67d9a86b663 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle plan +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:13:18 + + +Exit code: 1 + +=== bundle deploy fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:13:18 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/script new file mode 100644 index 00000000000..4fb2038cd28 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/script @@ -0,0 +1,7 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "bundle plan fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/test.toml new file mode 100644 index 00000000000..a18d500f458 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = false + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/databricks.yml.tmpl b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/databricks.yml.tmpl new file mode 100644 index 00000000000..b13f77d0af5 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: lifecycle-started-toggle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + sql_warehouses: + mywarehouse: + name: $UNIQUE_NAME + cluster_size: "Medium" + lifecycle: + started: false diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml new file mode 100644 index 00000000000..4426d448667 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt new file mode 100644 index 00000000000..b670fd4dc87 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt @@ -0,0 +1,73 @@ + +=== Deploy with started=false: warehouse created and then stopped +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses", + "body": { + "auto_stop_mins": 120, + "cluster_size": "Medium", + "enable_photon": true, + "max_num_clusters": 1, + "name": "[UNIQUE_NAME]", + "spot_instance_policy": "COST_OPTIMIZED" + } +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/stop" +} + +>>> errcode [CLI] warehouses get [UUID] +"STOPPED" + +=== Toggle started=false -> started=true: only Start should be called, no Edit +>>> update_file.py databricks.yml started: false started: true + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/start" +} + +>>> errcode [CLI] warehouses get [UUID] +"RUNNING" + +=== Toggle started=true -> started=false: only Stop should be called, no Edit +>>> update_file.py databricks.yml started: true started: false + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/stop" +} + +>>> errcode [CLI] warehouses get [UUID] +"STOPPED" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.sql_warehouses.mywarehouse + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script new file mode 100644 index 00000000000..a32ce1e0eda --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy with started=false: warehouse created and then stopped" +trace errcode $CLI bundle deploy +WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +title "Toggle started=false -> started=true: only Start should be called, no Edit" +trace update_file.py databricks.yml "started: false" "started: true" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +title "Toggle started=true -> started=false: only Stop should be called, no Edit" +trace update_file.py databricks.yml "started: true" "started: false" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/test.toml new file mode 100644 index 00000000000..7e3d0d9c69a --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[[Repls]] +Old = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +New = "[UUID]" + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/databricks.yml.tmpl new file mode 100644 index 00000000000..6eb3ba981e2 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + sql_warehouses: + mywarehouse: + name: $UNIQUE_NAME + cluster_size: "Medium" + lifecycle: + started: true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml new file mode 100644 index 00000000000..4426d448667 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt new file mode 100644 index 00000000000..10a132b3de1 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt @@ -0,0 +1,101 @@ + +=== Deploy with started=true: warehouse created and running +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses", + "body": { + "auto_stop_mins": 120, + "cluster_size": "Medium", + "enable_photon": true, + "max_num_clusters": 1, + "name": "[UNIQUE_NAME]", + "spot_instance_policy": "COST_OPTIMIZED" + } +} + +>>> errcode [CLI] warehouses get [UUID] +"RUNNING" + +=== Stop warehouse externally while config says started=true: plan detects drift +>>> errcode [CLI] warehouses stop [UUID] +"STOPPED" + +>>> [CLI] bundle plan +update sql_warehouses.mywarehouse + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Deploy fixes the drift: warehouse restarted +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/stop" +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/start" +} + +>>> errcode [CLI] warehouses get [UUID] +"RUNNING" + +=== Deploy with started=false while warehouse is stopped: warehouse stays stopped +>>> errcode [CLI] warehouses stop [UUID] +"STOPPED" + +>>> update_file.py databricks.yml started: true started: false + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/stop" +} + +>>> errcode [CLI] warehouses get [UUID] +"STOPPED" + +=== Deploy with started=true: warehouse restarted +>>> update_file.py databricks.yml started: false started: true + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[UUID]/start" +} + +>>> errcode [CLI] warehouses get [UUID] +"RUNNING" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.sql_warehouses.mywarehouse + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script new file mode 100644 index 00000000000..c4e99e5c829 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy with started=true: warehouse created and running" +trace errcode $CLI bundle deploy +WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +title "Stop warehouse externally while config says started=true: plan detects drift" +{ trace errcode $CLI warehouses stop "$WAREHOUSE_ID" | jq '.state'; } || true +trace $CLI bundle plan + +title "Deploy fixes the drift: warehouse restarted" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +title "Deploy with started=false while warehouse is stopped: warehouse stays stopped" +{ trace errcode $CLI warehouses stop "$WAREHOUSE_ID" | jq '.state'; } || true +trace update_file.py databricks.yml "started: true" "started: false" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +title "Deploy with started=true: warehouse restarted" +trace update_file.py databricks.yml "started: false" "started: true" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/test.toml new file mode 100644 index 00000000000..7e3d0d9c69a --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[[Repls]] +Old = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +New = "[UUID]" + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/resources/sql_warehouses.go b/bundle/config/resources/sql_warehouses.go index 016526a80ef..56653d27c99 100644 --- a/bundle/config/resources/sql_warehouses.go +++ b/bundle/config/resources/sql_warehouses.go @@ -15,9 +15,20 @@ type SqlWarehouse struct { BaseResource sql.CreateWarehouseRequest + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` + Permissions []SqlWarehousePermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (sw *SqlWarehouse) GetLifecycle() LifecycleConfig { + if sw.Lifecycle == nil { + return LifecycleWithStarted{} + } + return *sw.Lifecycle +} + func (sw *SqlWarehouse) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, sw) } diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 704f9f66187..146dee5294d 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -2,14 +2,51 @@ package dresources import ( "context" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" ) +// SqlWarehouseState is the state type for SqlWarehouse resources. It extends sql.CreateWarehouseRequest +// with lifecycle settings. +type SqlWarehouseState struct { + sql.CreateWarehouseRequest + + Lifecycle *StateLifecycle `json:"lifecycle,omitempty"` +} + +// Custom marshalers needed because embedded sql.CreateWarehouseRequest has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *SqlWarehouseState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s SqlWarehouseState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +// SqlWarehouseRemote extends sql.GetWarehouseResponse with a synthetic Lifecycle field so that +// RemoteType satisfies TestRemoteSuperset (every field in SqlWarehouseState exists in SqlWarehouseRemote). +// Lifecycle.Started is populated by DoRead from the warehouse's running state. +type SqlWarehouseRemote struct { + sql.GetWarehouseResponse + + Lifecycle *StateLifecycle `json:"lifecycle,omitempty"` +} + +func (r *SqlWarehouseRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) +} + +func (r SqlWarehouseRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) +} + type ResourceSqlWarehouse struct { client *databricks.WorkspaceClient } @@ -20,75 +57,184 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq } // PrepareState converts bundle config to the SDK type. -func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.CreateWarehouseRequest { - return &input.CreateWarehouseRequest +func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { + s := &SqlWarehouseState{ + CreateWarehouseRequest: input.CreateWarehouseRequest, + Lifecycle: nil, + } + if input.Lifecycle != nil && input.Lifecycle.Started != nil { + s.Lifecycle = &StateLifecycle{Started: input.Lifecycle.Started} + } + return s } -func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest { - return &sql.CreateWarehouseRequest{ - AutoStopMins: warehouse.AutoStopMins, - Channel: warehouse.Channel, - ClusterSize: warehouse.ClusterSize, - CreatorName: warehouse.CreatorName, - EnablePhoton: warehouse.EnablePhoton, - EnableServerlessCompute: warehouse.EnableServerlessCompute, - InstanceProfileArn: warehouse.InstanceProfileArn, - MaxNumClusters: warehouse.MaxNumClusters, - MinNumClusters: warehouse.MinNumClusters, - Name: warehouse.Name, - SpotInstancePolicy: warehouse.SpotInstancePolicy, - Tags: warehouse.Tags, - WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), - ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), +// RemapState maps the remote SqlWarehouseRemote to SqlWarehouseState for diff comparison. +// Started is derived from warehouse state so the planner can detect start/stop changes. +func (*ResourceSqlWarehouse) RemapState(warehouse *SqlWarehouseRemote) *SqlWarehouseState { + started := warehouse.State == sql.StateRunning + return &SqlWarehouseState{ + CreateWarehouseRequest: sql.CreateWarehouseRequest{ + AutoStopMins: warehouse.AutoStopMins, + Channel: warehouse.Channel, + ClusterSize: warehouse.ClusterSize, + CreatorName: warehouse.CreatorName, + EnablePhoton: warehouse.EnablePhoton, + EnableServerlessCompute: warehouse.EnableServerlessCompute, + InstanceProfileArn: warehouse.InstanceProfileArn, + MaxNumClusters: warehouse.MaxNumClusters, + MinNumClusters: warehouse.MinNumClusters, + Name: warehouse.Name, + SpotInstancePolicy: warehouse.SpotInstancePolicy, + Tags: warehouse.Tags, + WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), + ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), + }, + Lifecycle: &StateLifecycle{Started: &started}, } } // DoRead reads the warehouse by id. -func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*sql.GetWarehouseResponse, error) { - return r.client.Warehouses.GetById(ctx, id) +func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*SqlWarehouseRemote, error) { + warehouse, err := r.client.Warehouses.GetById(ctx, id) + if err != nil { + return nil, err + } + remote := &SqlWarehouseRemote{ + GetWarehouseResponse: *warehouse, + Lifecycle: nil, + } + + switch warehouse.State { + case sql.StateRunning: + started := true + remote.Lifecycle = &StateLifecycle{Started: &started} + case sql.StateStopped: + started := false + remote.Lifecycle = &StateLifecycle{Started: &started} + default: + remote.Lifecycle = nil + } + return remote, nil } // DoCreate creates the warehouse and returns its id. -func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateWarehouseRequest) (string, *sql.GetWarehouseResponse, error) { - waiter, err := r.client.Warehouses.Create(ctx, *config) +func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehouseState) (string, *SqlWarehouseRemote, error) { + waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest) if err != nil { return "", nil, err } return waiter.Id, nil, nil } +// hasWarehouseChanges reports whether the plan entry contains any Update changes +// to fields that belong to the Warehouse Edit API (i.e., not lifecycle-only fields). +func hasWarehouseChanges(entry *PlanEntry) bool { + return entry.Changes.HasChangeExcept("lifecycle", "lifecycle.started") +} + // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ *PlanEntry) (*sql.GetWarehouseResponse, error) { - request := sql.EditWarehouseRequest{ - AutoStopMins: config.AutoStopMins, - Channel: config.Channel, - ClusterSize: config.ClusterSize, - CreatorName: config.CreatorName, - EnablePhoton: config.EnablePhoton, - EnableServerlessCompute: config.EnableServerlessCompute, - Id: id, - InstanceProfileArn: config.InstanceProfileArn, - MaxNumClusters: config.MaxNumClusters, - MinNumClusters: config.MinNumClusters, - Name: config.Name, - SpotInstancePolicy: config.SpotInstancePolicy, - Tags: config.Tags, - WarehouseType: sql.EditWarehouseRequestWarehouseType(config.WarehouseType), - ForceSendFields: utils.FilterFields[sql.EditWarehouseRequest](config.ForceSendFields), - } - - waiter, err := r.client.Warehouses.Edit(ctx, request) +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, entry *PlanEntry) (*SqlWarehouseRemote, error) { + if hasWarehouseChanges(entry) { + request := sql.EditWarehouseRequest{ + AutoStopMins: config.AutoStopMins, + Channel: config.Channel, + ClusterSize: config.ClusterSize, + CreatorName: config.CreatorName, + EnablePhoton: config.EnablePhoton, + EnableServerlessCompute: config.EnableServerlessCompute, + Id: id, + InstanceProfileArn: config.InstanceProfileArn, + MaxNumClusters: config.MaxNumClusters, + MinNumClusters: config.MinNumClusters, + Name: config.Name, + SpotInstancePolicy: config.SpotInstancePolicy, + Tags: config.Tags, + WarehouseType: sql.EditWarehouseRequestWarehouseType(config.WarehouseType), + ForceSendFields: utils.FilterFields[sql.EditWarehouseRequest](config.ForceSendFields), + } + + waiter, err := r.client.Warehouses.Edit(ctx, request) + if err != nil { + return nil, err + } + + if waiter.Id != id { + log.Warnf(ctx, "sql_warehouses: response contains unexpected id=%#v (expected %#v)", waiter.Id, id) + } + } + + if config.Lifecycle == nil || config.Lifecycle.Started == nil { + return nil, nil + } + + desiredStarted := *config.Lifecycle.Started + alreadyRunning := remoteWarehouseIsRunning(entry) + if desiredStarted && !alreadyRunning { + // lifecycle.started=true: fire Start; WaitAfterUpdate polls for RUNNING. + _, err := r.client.Warehouses.Start(ctx, sql.StartRequest{Id: id}) + return nil, err + } else if !desiredStarted && alreadyRunning { + // lifecycle.started=false: fire Stop; WaitAfterUpdate polls for STOPPED. + _, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: id}) + return nil, err + } + + return nil, nil +} + +// WaitAfterUpdate waits for the warehouse to reach the desired lifecycle state after DoUpdate. +func (r *ResourceSqlWarehouse) WaitAfterUpdate(ctx context.Context, id string, config *SqlWarehouseState) (*SqlWarehouseRemote, error) { + if config.Lifecycle == nil || config.Lifecycle.Started == nil { + return nil, nil + } + + if *config.Lifecycle.Started { + _, err := r.client.Warehouses.WaitGetWarehouseRunning(ctx, id, 20*time.Minute, nil) + return nil, err + } + + _, err := r.client.Warehouses.WaitGetWarehouseStopped(ctx, id, 20*time.Minute, nil) + return nil, err +} + +// WaitAfterCreate waits for the warehouse to be ready, then stops it if lifecycle.started=false. +// Warehouses are created in a starting state; WaitGetWarehouseRunning waits for them to be RUNNING. +func (r *ResourceSqlWarehouse) WaitAfterCreate(ctx context.Context, id string, config *SqlWarehouseState) (*SqlWarehouseRemote, error) { + if config.Lifecycle == nil || config.Lifecycle.Started == nil { + return nil, nil + } + + // Always wait for RUNNING first: warehouses start asynchronously. + _, err := r.client.Warehouses.WaitGetWarehouseRunning(ctx, id, 20*time.Minute, nil) if err != nil { return nil, err } - if waiter.Id != id { - log.Warnf(ctx, "sql_warehouses: response contains unexpected id=%#v (expected %#v)", waiter.Id, id) + if !*config.Lifecycle.Started { + // started=false: stop the warehouse after it reaches RUNNING. + stopWaiter, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: id}) + if err != nil { + return nil, err + } + _, err = stopWaiter.Get() + return nil, err } return nil, nil } -func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string, _ *sql.CreateWarehouseRequest) error { +func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string, _ *SqlWarehouseState) error { return r.client.Warehouses.DeleteById(ctx, oldID) } + +// remoteWarehouseIsRunning reads the warehouse running state from the plan entry's remote state. +func remoteWarehouseIsRunning(entry *PlanEntry) bool { + if entry.RemoteState == nil { + return false + } + remote, ok := entry.RemoteState.(*SqlWarehouseRemote) + if !ok { + return false + } + return remote.State == sql.StateRunning +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 86f739cb2ca..e6fee4547fa 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -76,6 +76,9 @@ var knownMissingInStateType = map[string][]string{ "clusters": { "lifecycle.prevent_destroy", }, + "sql_warehouses": { + "lifecycle.prevent_destroy", + }, "dashboards": { "file_path", }, diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index c3c0599789c..56e3a85b5c0 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -8,4 +8,5 @@ const ( PresetsNamePrefixIsSet = "presets_name_prefix_is_set" AppLifecycleStarted = "app_lifecycle_started" ClusterLifecycleStarted = "cluster_lifecycle_started" + SqlWarehouseLifecycleStarted = "sql_warehouse_lifecycle_started" ) diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 6496b45921d..e64b736fc72 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -128,6 +128,13 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { } } + for _, warehouse := range b.Config.Resources.SqlWarehouses { + if warehouse != nil && warehouse.Lifecycle != nil && warehouse.Lifecycle.Started != nil { + b.Metrics.SetBoolValue(metrics.SqlWarehouseLifecycleStarted, *warehouse.Lifecycle.Started) + break + } + } + // If the bundle UUID is not set, we use a default 0 value. bundleUuid := "00000000-0000-0000-0000-000000000000" if b.Config.Bundle.Uuid != "" { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index ba65d09cd97..1ee87af1af7 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -555,6 +555,14 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.SqlWarehousesUpsert(req, req.Vars["warehouse_id"]) }) + server.Handle("POST", "/api/2.0/sql/warehouses/{warehouse_id}/start", func(req Request) any { + return req.Workspace.SqlWarehousesStart(req, req.Vars["warehouse_id"]) + }) + + server.Handle("POST", "/api/2.0/sql/warehouses/{warehouse_id}/stop", func(req Request) any { + return req.Workspace.SqlWarehousesStop(req, req.Vars["warehouse_id"]) + }) + server.Handle("DELETE", "/api/2.0/sql/warehouses/{warehouse_id}", func(req Request) any { return MapDelete(req.Workspace, req.Workspace.SqlWarehouses, req.Vars["warehouse_id"]) }) diff --git a/libs/testserver/sql_warehouses.go b/libs/testserver/sql_warehouses.go index 6e7fd3a2f91..7f36ace72fc 100644 --- a/libs/testserver/sql_warehouses.go +++ b/libs/testserver/sql_warehouses.go @@ -59,6 +59,34 @@ func (s *FakeWorkspace) SqlWarehousesUpsert(req Request, warehouseId string) Res } } +func (s *FakeWorkspace) SqlWarehousesStart(req Request, warehouseId string) Response { + defer s.LockUnlock()() + + warehouse, ok := s.SqlWarehouses[warehouseId] + if !ok { + return Response{StatusCode: 404} + } + + warehouse.State = sql.StateRunning + s.SqlWarehouses[warehouseId] = warehouse + + return Response{} +} + +func (s *FakeWorkspace) SqlWarehousesStop(req Request, warehouseId string) Response { + defer s.LockUnlock()() + + warehouse, ok := s.SqlWarehouses[warehouseId] + if !ok { + return Response{StatusCode: 404} + } + + warehouse.State = sql.StateStopped + s.SqlWarehouses[warehouseId] = warehouse + + return Response{} +} + func (s *FakeWorkspace) SqlWarehousesList(req Request) Response { var warehouses []sql.EndpointInfo for _, warehouse := range s.SqlWarehouses {