diff --git a/.nextchanges/bundles/validate-serverless-pipeline-clusters.md b/.nextchanges/bundles/validate-serverless-pipeline-clusters.md new file mode 100644 index 00000000000..a99cf2f8668 --- /dev/null +++ b/.nextchanges/bundles/validate-serverless-pipeline-clusters.md @@ -0,0 +1 @@ +* Reject pipeline cluster settings when serverless compute is enabled during bundle validation, planning, and deployment. ([#6662](https://github.com/databricks/cli/pull/6662)) diff --git a/acceptance/bundle/validate/pipeline_serverless_clusters/databricks.yml b/acceptance/bundle/validate/pipeline_serverless_clusters/databricks.yml new file mode 100644 index 00000000000..626b5a91b7f --- /dev/null +++ b/acceptance/bundle/validate/pipeline_serverless_clusters/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: pipeline-serverless-clusters + +targets: + invalid: + resources: + pipelines: + my_pipeline: + name: my-pipeline + serverless: true + clusters: + - label: default + num_workers: 1 + + classic: + resources: + pipelines: + explicit: + name: explicit-classic + serverless: false + clusters: + - label: default + num_workers: 1 + implicit: + name: implicit-classic + clusters: + - label: default + num_workers: 1 + + serverless: + resources: + pipelines: + omitted_clusters: + name: omitted-clusters + serverless: true + empty_clusters: + name: empty-clusters + serverless: true + clusters: [] diff --git a/acceptance/bundle/validate/pipeline_serverless_clusters/out.test.toml b/acceptance/bundle/validate/pipeline_serverless_clusters/out.test.toml new file mode 100644 index 00000000000..e1af1a235ad --- /dev/null +++ b/acceptance/bundle/validate/pipeline_serverless_clusters/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/pipeline_serverless_clusters/output.txt b/acceptance/bundle/validate/pipeline_serverless_clusters/output.txt new file mode 100644 index 00000000000..afadc33d1db --- /dev/null +++ b/acceptance/bundle/validate/pipeline_serverless_clusters/output.txt @@ -0,0 +1,52 @@ + +=== Reject cluster settings with serverless compute before deployment +>>> [CLI] bundle validate -t invalid +Error: Cannot configure clusters for a serverless pipeline + at resources.pipelines.my_pipeline.clusters + in databricks.yml:12:13 + +Remove the clusters setting or set serverless to false. + +Name: pipeline-serverless-clusters +Target: invalid +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/invalid + +Found 1 error + +>>> [CLI] bundle plan -t invalid +Error: Cannot configure clusters for a serverless pipeline + at resources.pipelines.my_pipeline.clusters + in databricks.yml:12:13 + +Remove the clusters setting or set serverless to false. + + +>>> [CLI] bundle deploy -t invalid +Error: Cannot configure clusters for a serverless pipeline + at resources.pipelines.my_pipeline.clusters + in databricks.yml:12:13 + +Remove the clusters setting or set serverless to false. + + +=== Allow classic pipeline cluster settings +>>> [CLI] bundle validate -t classic +Name: pipeline-serverless-clusters +Target: classic +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/classic + +Validation OK! + +=== Allow serverless pipelines without cluster settings +>>> [CLI] bundle validate -t serverless +Name: pipeline-serverless-clusters +Target: serverless +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/serverless + +Validation OK! diff --git a/acceptance/bundle/validate/pipeline_serverless_clusters/script b/acceptance/bundle/validate/pipeline_serverless_clusters/script new file mode 100644 index 00000000000..40a7d9d4e1c --- /dev/null +++ b/acceptance/bundle/validate/pipeline_serverless_clusters/script @@ -0,0 +1,10 @@ +title "Reject cluster settings with serverless compute before deployment" +musterr trace $CLI bundle validate -t invalid +musterr trace $CLI bundle plan -t invalid +musterr trace $CLI bundle deploy -t invalid + +title "Allow classic pipeline cluster settings" +trace $CLI bundle validate -t classic + +title "Allow serverless pipelines without cluster settings" +trace $CLI bundle validate -t serverless diff --git a/acceptance/bundle/validate/pipeline_serverless_clusters/test.toml b/acceptance/bundle/validate/pipeline_serverless_clusters/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_serverless_clusters/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index d01eb8c1491..303f77cf238 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -29,6 +29,7 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), + PipelineClusterSpec(), // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), diff --git a/bundle/config/validate/pipeline_cluster_spec.go b/bundle/config/validate/pipeline_cluster_spec.go new file mode 100644 index 00000000000..36975d5e94a --- /dev/null +++ b/bundle/config/validate/pipeline_cluster_spec.go @@ -0,0 +1,41 @@ +package validate + +import ( + "context" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// PipelineClusterSpec rejects cluster settings when serverless compute is enabled. +func PipelineClusterSpec() bundle.ReadOnlyMutator { + return &pipelineClusterSpec{} +} + +type pipelineClusterSpec struct{ bundle.RO } + +func (v *pipelineClusterSpec) Name() string { + return "validate:pipeline_cluster_spec" +} + +func (v *pipelineClusterSpec) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.Pipelines)) { + pipeline := b.Config.Resources.Pipelines[name] + if !pipeline.Serverless || len(pipeline.Clusters) == 0 { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(name), dyn.Key("clusters")) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Cannot configure clusters for a serverless pipeline", + Detail: "Remove the clusters setting or set serverless to false.", + Paths: []dyn.Path{path}, + Locations: b.Config.GetLocations(path.String()), + }) + } + return diags +}