bundle: read and edit the typed config in five mutators

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle
+  cluster_id: existing-cluster
+
+variables:
+  cluster:
+    type: complex
+    default:
+      num_workers: 1
+
+resources:
+  clusters:
+    source:
+      num_workers: 2
+  jobs:
+    job:
+      name: job
+      tasks:
+        # bundle.cluster_id replaces new_cluster. The reference below it must go too.
+        - task_key: scalar_ref
+          new_cluster:
+            num_workers: ${resources.clusters.source.num_workers}
+        # A reference to the whole new_cluster must not bring it back.
+        - task_key: variable_ref
+          new_cluster: ${var.cluster}
+        # A reference to the new_cluster of another job is resolved after the override.
+        - task_key: resource_ref
+          new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster}
+    other:
+      name: other
+      job_clusters:
+        - job_cluster_key: shared
+          new_cluster:
+            num_workers: 3

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "resource_ref"
+  },
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "scalar_ref"
+  },
+  {
+    "existing_cluster_id": "existing-cluster",
+    "task_key": "variable_ref"
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created clusters.source
+Created jobs.job
+Created jobs.other
+Files: 1 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script

@@ -0,0 +1,3 @@
+# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference.
+trace $CLI bundle validate -o json | jq .resources.jobs.job.tasks
+trace $CLI bundle deploy

acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: test-bundle
+
+resources:
+  quality_monitors:
+    src:
+      table_name: main.default.src
+      assets_dir: /Workspace/monitors/src
+      output_schema_name: main.default
+      schedule:
+        quartz_cron_expression: "0 0 12 * * ?"
+        timezone_id: UTC
+    dst:
+      table_name: main.default.dst
+      assets_dir: /Workspace/monitors/dst
+      output_schema_name: main.default
+
+targets:
+  dev:
+    presets:
+      trigger_pause_status: PAUSED
+    resources:
+      quality_monitors:
+        dst:
+          schedule: ${resources.quality_monitors.src.schedule}

acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files...
+Created quality_monitors.dst
+Created quality_monitors.src
+Files: 1 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //tables/main.default.dst/monitor
+{
+  "assets_dir": "/Workspace/monitors/dst",
+  "output_schema_name": "main.default"
+}

acceptance/bundle/override/preset_clears_forced_schedule_reference/script

@@ -0,0 +1,4 @@
+# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst
+# with a reference to the schedule of src, which is removed too: the request carries no schedule.
+trace $CLI bundle deploy
+trace print_requests.py //tables/main.default.dst/monitor | jq .body

acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml

@@ -0,0 +1 @@
+RecordRequests = true

acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+presets:
+  name_prefix: "pre-"
+
+resources:
+  jobs:
+    base:
+      name: base
+    viaref:
+      name: ${resources.jobs.base.name}

acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/presets/name_prefix_reference_to_name/output.txt

@@ -0,0 +1,3 @@
+
+>>> [CLI] bundle plan -o json
+"pre-pre-base"

acceptance/bundle/presets/name_prefix_reference_to_name/script

@@ -0,0 +1,2 @@
+# name_prefix is added to a name that is a reference to the name of another job, which has the prefix too.
+trace $CLI bundle plan -o json | jq '.plan["resources.jobs.viaref"].new_state.value.name'

acceptance/bundle/presets/name_prefix_reference_to_name/test.toml

@@ -0,0 +1 @@
+Badness = "name_prefix is added a second time to a name that is a reference to the name of another job."

acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle
+
+presets:
+  tags:
+    preset_tag: ${resources.jobs.j.id}
+
+resources:
+  schemas:
+    s:
+      name: s
+      catalog_name: main
+      properties: {owner: fixed}
+  jobs:
+    j:
+      # tags is a reference to a whole map, and the preset tag above is not added to it.
+      tags: ${resources.schemas.s.properties}

acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt

@@ -0,0 +1,5 @@
+
+>>> [CLI] bundle plan -o json
+{
+  "owner": "fixed"
+}

acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script

@@ -0,0 +1,5 @@
+# The preset tag is not applied to a job whose tags is a reference to a whole map: the job gets the
+# tags of the referenced schema only.
+# The preset value references the job itself. If the preset tag were applied, this plan would fail
+# with a cycle error.
+trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"].new_state.value.tags'

acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml

@@ -0,0 +1 @@
+Badness = "presets.tags are silently not applied to a job whose tags is a reference to a whole map."

acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    base:
+      schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC}
+    viaref:
+      schedule: ${resources.jobs.base.schedule}

acceptance/bundle/validate/references/job_schedule_whole_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/references/job_schedule_whole_reference/output.txt

@@ -0,0 +1,11 @@
+
+>>> musterr [CLI] bundle validate
+Error: expected a map to index "pause_status", found string
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 1 error

acceptance/bundle/validate/references/job_schedule_whole_reference/script

@@ -0,0 +1,3 @@
+# The schedule of viaref is a reference to the whole schedule of another job.
+# The defaults set schedule.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference.
+trace musterr $CLI bundle validate

acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml

@@ -0,0 +1 @@
+Badness = "The default of schedule.pause_status is written by path, which fails with 'expected a map to index pause_status, found string' when the schedule is a reference to a whole schedule of another job; validate should accept it."

acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    base:
+      trigger: {file_arrival: {url: "s3://bucket/path"}}
+    viaref:
+      trigger: ${resources.jobs.base.trigger}

acceptance/bundle/validate/references/job_trigger_whole_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/references/job_trigger_whole_reference/output.txt

@@ -0,0 +1,11 @@
+
+>>> musterr [CLI] bundle validate
+Error: expected a map to index "pause_status", found string
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 1 error

acceptance/bundle/validate/references/job_trigger_whole_reference/script

@@ -0,0 +1,3 @@
+# The trigger of viaref is a reference to the whole trigger of another job.
+# The defaults set trigger.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference.
+trace musterr $CLI bundle validate

acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml

@@ -0,0 +1 @@
+Badness = "The default of trigger.pause_status is written by path, which fails with 'expected a map to index pause_status, found string' when the trigger is a reference to a whole trigger of another job; validate should accept it."

acceptance/bundle/validate/required_struct_reference/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  clusters:
+    source:
+      is_single_node: true
+    target:
+      # clients is a required field of workload_type. Both of its fields are references.
+      workload_type:
+        clients:
+          notebooks: ${resources.clusters.source.is_single_node}
+          jobs: ${resources.clusters.source.is_single_node}

acceptance/bundle/validate/required_struct_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/required_struct_reference/output.txt

@@ -0,0 +1,8 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "clients": {
+    "jobs": "${resources.clusters.source.is_single_node}",
+    "notebooks": "${resources.clusters.source.is_single_node}"
+  }
+}

acceptance/bundle/validate/required_struct_reference/script

@@ -0,0 +1,2 @@
+# workload_type.clients is required. A clients whose fields are all references is not reported as missing.
+trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type

acceptance/bundle/validate/secret_scope_invalid_permission_level/output.txt

@@ -2,9 +2,11 @@
 >>> [CLI] bundle validate
 Warning: invalid value "12" for enum field. Valid values are [READ WRITE MANAGE]
   at resources.secret_scopes.bad_levels.permissions[0].level
+  in databricks.yml:10:18
 
 Warning: invalid value "fuzz_badlevel" for enum field. Valid values are [READ WRITE MANAGE]
   at resources.secret_scopes.bad_levels.permissions[1].level
+  in databricks.yml:12:18
 
 Name: test-bundle
 Target: default
@@ -17,9 +19,11 @@ Found 2 warnings
 >>> [CLI] bundle validate --strict
 Warning: invalid value "12" for enum field. Valid values are [READ WRITE MANAGE]
   at resources.secret_scopes.bad_levels.permissions[0].level
+  in databricks.yml:10:18
 
 Warning: invalid value "fuzz_badlevel" for enum field. Valid values are [READ WRITE MANAGE]
   at resources.secret_scopes.bad_levels.permissions[1].level
+  in databricks.yml:12:18
 
 Name: test-bundle
 Target: default

bundle/config/loader/process_root_includes.go

@@ -10,7 +10,6 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
 )
 
 type processRootIncludes struct{}
@@ -42,7 +41,8 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 
 	// Preserve the raw include patterns before they are replaced below with the
 	// expanded list of loaded files, so IsFileIncluded can re-match against them.
-	b.SetIncludePatterns(b.Config.Include)
+	patterns := b.Config.Include
+	b.SetIncludePatterns(patterns)
 
 	// Map with files we've already seen to avoid loading them twice.
 	seen := map[string]bool{}
@@ -131,27 +131,6 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		}
 	}
 
-	// Swap out the original includes list with the expanded globs. This goes through
-	// Mutate so the dynamic tree is updated too: the includes below are applied without
-	// their own mutator scope, so nothing converts the typed field back into the dynamic
-	// tree afterwards, and the next ToTyped would otherwise restore the raw patterns.
-	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
-		// Include is omitempty in the typed configuration, so an empty list must stay
-		// absent from the dynamic tree rather than be written as [].
-		if len(files) == 0 {
-			return dyn.DropKeys(root, []string{"include"})
-		}
-
-		includeValues := make([]dyn.Value, 0, len(files))
-		for _, file := range files {
-			includeValues = append(includeValues, dyn.V(file))
-		}
-		return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations()))
-	})
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
 	// Track number of bundle YAML (or JSON) files in the configuration. The +1 is there
 	// to account for the root databricks.yaml file.
 	b.Metrics.ConfigurationFileCount = int64(len(files)) + 1
@@ -161,5 +140,12 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	// whole accumulated configuration per file, making load quadratic in the number of
 	// included files (~20 minutes for 6000 files).
 	bundle.ApplySeqInScopeContext(ctx, b, out...)
+
+	// Swap out the original include patterns with the expanded globs. This is done after
+	// the includes are merged: [config.Root.Merge] reloads the typed configuration from
+	// the dynamic tree, which still holds the patterns, followed by the include sections
+	// of included files (merging concatenates sequences). Those are kept as they are.
+	// Include is omitempty, so an empty list is absent from the configuration.
+	b.Config.Include = append(files, b.Config.Include[len(patterns):]...)
 	return nil
 }

bundle/config/loader/process_root_includes_test.go

@@ -1,6 +1,7 @@
 package loader_test
 
 import (
+	"path/filepath"
 	"runtime"
 	"testing"
 
@@ -100,8 +101,8 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
 }
 
 // The expanded include list must be visible in both the typed and the dynamic
-// configuration: the per-file includes are applied without their own mutator scope, so
-// nothing converts the typed field back into the dynamic tree afterwards.
+// configuration: merging the per-file includes reloads the typed configuration from the
+// dynamic tree, so the expanded list must not be overwritten by the raw patterns.
 func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: t.TempDir(),
@@ -142,6 +143,28 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) {
 	assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind())
 }
 
+// Include sections of included files are not applied, but they are merged into the
+// include list after the expanded globs.
+func TestProcessRootIncludesKeepsIncludesOfIncludedFiles(t *testing.T) {
+	b := &bundle.Bundle{
+		BundleRootPath: t.TempDir(),
+		Config: config.Root{
+			Include: []string{
+				"*.yml",
+			},
+		},
+	}
+
+	testutil.Touch(t, b.BundleRootPath, "databricks.yml")
+	testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), "include:\n  - c.yml\n")
+	testutil.Touch(t, b.BundleRootPath, "b.yml")
+
+	diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes())
+	require.NoError(t, diags.Error())
+	assert.Equal(t, []string{"a.yml", "b.yml", "c.yml"}, b.Config.Include)
+	assert.Equal(t, []any{"a.yml", "b.yml", "c.yml"}, b.Config.Value().Get("include").AsAny())
+}
+
 func TestProcessRootIncludesNotExists(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: t.TempDir(),