From 3317c14f359d39ed3fdd1d6d93536f787eb64ee9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 20 Oct 2026 15:42:06 +0200 Subject: [PATCH 01/14] acc: record main's behaviour for reference, override and required-field edge cases Co-authored-by: Isaac --- .../databricks.yml | 36 +++++++++++++++++ .../out.test.toml | 2 + .../output.txt | 21 ++++++++++ .../script | 2 + .../databricks.yml | 39 +++++++++++++++++++ .../out.test.toml | 2 + .../output.txt | 12 ++++++ .../script | 2 + .../test.toml | 1 + .../databricks.yml | 18 +++++++++ .../out.test.toml | 2 + .../output.txt | 6 +++ .../whole_map_reference_hides_entries/script | 1 + .../required_struct_reference/databricks.yml | 16 ++++++++ .../required_struct_reference/out.test.toml | 2 + .../required_struct_reference/output.txt | 9 +++++ .../validate/required_struct_reference/script | 1 + 17 files changed, 172 insertions(+) create mode 100644 acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml create mode 100644 acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml create mode 100644 acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt create mode 100644 acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script create mode 100644 acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml create mode 100644 acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml create mode 100644 acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt create mode 100644 acceptance/bundle/override/preset_clears_forced_schedule_reference/script create mode 100644 acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml create mode 100644 acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml create mode 100644 acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml create mode 100644 acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt create mode 100644 acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script create mode 100644 acceptance/bundle/validate/required_struct_reference/databricks.yml create mode 100644 acceptance/bundle/validate/required_struct_reference/out.test.toml create mode 100644 acceptance/bundle/validate/required_struct_reference/output.txt create mode 100644 acceptance/bundle/validate/required_struct_reference/script diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml new file mode 100644 index 00000000000..f6f8fbb574f --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml @@ -0,0 +1,36 @@ +bundle: + name: test-bundle + cluster_id: existing-cluster + +variables: + cluster: + type: complex + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 2 + jobs: + job: + name: job + tasks: + - task_key: task + notebook_task: + notebook_path: /Workspace/notebook + # Replaced by bundle.cluster_id: the reference below new_cluster must go too. + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: ${resources.clusters.source.num_workers} + - task_key: task_var + notebook_task: + notebook_path: /Workspace/notebook + # A reference at new_cluster must not bring it back either. + new_cluster: ${var.cluster} diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt new file mode 100644 index 00000000000..95a63c43cd6 --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle validate -o json +[ + { + "task_key": "task", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, + { + "task_key": "task_var", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created clusters.source +Created jobs.job +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script new file mode 100644 index 00000000000..403b93598f6 --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script @@ -0,0 +1,2 @@ +trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]' +trace $CLI bundle deploy diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml new file mode 100644 index 00000000000..25315e8078e --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: test-bundle + +resources: + quality_monitors: + src: + table_name: main.default.src + assets_dir: /Workspace/monitors/src + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + schedule: + quartz_cron_expression: "0 0 12 * * ?" + timezone_id: UTC + dst: + table_name: main.default.dst + assets_dir: /Workspace/monitors/dst + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + # Forced null: the target override replaces it with a reference. + schedule: null + +targets: + dev: + presets: + trigger_pause_status: PAUSED + resources: + quality_monitors: + dst: + schedule: ${resources.quality_monitors.src.schedule} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt new file mode 100644 index 00000000000..ab216e1ab9a --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files... +Created quality_monitors.dst +Created quality_monitors.src +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //tables/main.default.dst/monitor +{ + "schedule": null +} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script new file mode 100644 index 00000000000..c725099d07b --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -0,0 +1,2 @@ +trace $CLI bundle deploy +trace print_requests.py //tables/main.default.dst/monitor | jq '{schedule: .body.schedule}' diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml new file mode 100644 index 00000000000..643ef4196c1 --- /dev/null +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +presets: + tags: + owner: ${resources.jobs.j.id} + +resources: + schemas: + s: + name: s + catalog_name: main + properties: {owner: fixed} + jobs: + j: + name: j + # A whole-map reference: the preset's tags added to the typed map stay hidden. + tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt new file mode 100644 index 00000000000..0e54725cd03 --- /dev/null +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle plan +create jobs.j +create schemas.s + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script new file mode 100644 index 00000000000..e896f075ebf --- /dev/null +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script @@ -0,0 +1 @@ +trace $CLI bundle plan diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml new file mode 100644 index 00000000000..74d8baa9c19 --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + is_single_node: true + target: + cluster_name: target + spark_version: 15.4.x-scala2.12 + # clients holds only a reference, so its Go value is zero; it is still set. + workload_type: + clients: + jobs: ${resources.clusters.source.is_single_node} diff --git a/acceptance/bundle/validate/required_struct_reference/out.test.toml b/acceptance/bundle/validate/required_struct_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/required_struct_reference/output.txt b/acceptance/bundle/validate/required_struct_reference/output.txt new file mode 100644 index 00000000000..d44a21b582a --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! diff --git a/acceptance/bundle/validate/required_struct_reference/script b/acceptance/bundle/validate/required_struct_reference/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/script @@ -0,0 +1 @@ +trace $CLI bundle validate From c8ba99edfdaad691f0df9ed3a20b6c003d7e7eb8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 20 Oct 2026 18:51:44 +0200 Subject: [PATCH 02/14] acc: say what each test checks and print the relevant JSON --- .../script | 1 + .../script | 2 ++ .../databricks.yml | 2 +- .../output.txt | 17 ++++++++++++----- .../whole_map_reference_hides_entries/script | 4 +++- .../required_struct_reference/databricks.yml | 2 +- .../required_struct_reference/output.txt | 15 +++++++-------- .../validate/required_struct_reference/script | 3 ++- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script index 403b93598f6..2016268365d 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script @@ -1,2 +1,3 @@ +# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference. trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]' trace $CLI bundle deploy diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script index c725099d07b..b8336500d3c 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -1,2 +1,4 @@ +# A target override can replace a null schedule with a reference to another monitor's schedule. +# The deploy request has to carry the schedule of the referenced monitor. trace $CLI bundle deploy trace print_requests.py //tables/main.default.dst/monitor | jq '{schedule: .body.schedule}' diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml index 643ef4196c1..508eff00ec8 100644 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml @@ -14,5 +14,5 @@ resources: jobs: j: name: j - # A whole-map reference: the preset's tags added to the typed map stay hidden. + # The whole tags map is a reference, so the preset tag above is not added to it. tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt index 0e54725cd03..06316029ed1 100644 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt @@ -1,6 +1,13 @@ ->>> [CLI] bundle plan -create jobs.j -create schemas.s - -Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged +>>> [CLI] bundle plan -o json +{ + "tags": { + "owner": "fixed" + }, + "depends_on": [ + { + "node": "resources.schemas.s", + "label": "${resources.schemas.s.properties}" + } + ] +} diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script index e896f075ebf..5e421cc2150 100644 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script @@ -1 +1,3 @@ -trace $CLI bundle plan +# When a whole map is a reference, the entries a preset adds to it are not applied: the job gets the +# tags of the referenced schema only, and depends on the schema. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml index 74d8baa9c19..b75c2f4cbb1 100644 --- a/acceptance/bundle/validate/required_struct_reference/databricks.yml +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -10,7 +10,7 @@ resources: target: cluster_name: target spark_version: 15.4.x-scala2.12 - # clients holds only a reference, so its Go value is zero; it is still set. + # clients is a required field of workload_type. Its only field is a reference. workload_type: clients: jobs: ${resources.clusters.source.is_single_node} diff --git a/acceptance/bundle/validate/required_struct_reference/output.txt b/acceptance/bundle/validate/required_struct_reference/output.txt index d44a21b582a..36dd96afff2 100644 --- a/acceptance/bundle/validate/required_struct_reference/output.txt +++ b/acceptance/bundle/validate/required_struct_reference/output.txt @@ -1,9 +1,8 @@ ->>> [CLI] bundle validate -Name: test-bundle -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Validation OK! +>>> [CLI] bundle validate -o json +{ + "clients": { + "jobs": "${resources.clusters.source.is_single_node}", + "notebooks": true + } +} diff --git a/acceptance/bundle/validate/required_struct_reference/script b/acceptance/bundle/validate/required_struct_reference/script index 5350876150f..2975f06e053 100644 --- a/acceptance/bundle/validate/required_struct_reference/script +++ b/acceptance/bundle/validate/required_struct_reference/script @@ -1 +1,2 @@ -trace $CLI bundle validate +# workload_type.clients is required. A clients that only has a reference in it is not reported as missing. +trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type From 3e596c99081a5bbea037385e21c35935219d75a8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 21 Oct 2026 18:00:06 +0200 Subject: [PATCH 03/14] acc: make the recorded edge cases exercise what they claim Co-authored-by: Isaac --- .../databricks.yml | 13 +++++++++++++ .../output.txt | 8 +++++++- .../output.txt | 1 + .../preset_clears_forced_schedule_reference/script | 6 +++--- .../databricks.yml | 2 +- .../required_struct_reference/databricks.yml | 3 ++- .../validate/required_struct_reference/output.txt | 2 +- .../validate/required_struct_reference/script | 2 +- 8 files changed, 29 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml index f6f8fbb574f..d2e913f1abe 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml @@ -34,3 +34,16 @@ resources: notebook_path: /Workspace/notebook # A reference at new_cluster must not bring it back either. new_cluster: ${var.cluster} + - task_key: task_resource_ref + notebook_task: + notebook_path: /Workspace/notebook + # A reference to the new_cluster of another job is resolved after the override. + new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster} + other: + name: other + job_clusters: + - job_cluster_key: shared + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 3 diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt index 95a63c43cd6..14e1e4bbefd 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -6,6 +6,11 @@ "existing_cluster_id": "existing-cluster", "new_cluster": null }, + { + "task_key": "task_resource_ref", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, { "task_key": "task_var", "existing_cluster_id": "existing-cluster", @@ -17,5 +22,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created clusters.source Created jobs.job +Created jobs.other Files: 1 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt index ab216e1ab9a..eb07e4ec538 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt @@ -8,5 +8,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //tables/main.default.dst/monitor { + "has_schedule": false, "schedule": null } diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script index b8336500d3c..37d355db8c8 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -1,4 +1,4 @@ -# A target override can replace a null schedule with a reference to another monitor's schedule. -# The deploy request has to carry the schedule of the referenced monitor. +# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst +# with a reference to the schedule of src, which is removed too: the request carries no schedule. trace $CLI bundle deploy -trace print_requests.py //tables/main.default.dst/monitor | jq '{schedule: .body.schedule}' +trace print_requests.py //tables/main.default.dst/monitor | jq '.body | {has_schedule: has("schedule"), schedule}' diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml index 508eff00ec8..511a289e603 100644 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml +++ b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml @@ -3,7 +3,7 @@ bundle: presets: tags: - owner: ${resources.jobs.j.id} + preset_tag: ${resources.jobs.j.id} resources: schemas: diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml index b75c2f4cbb1..60e02ad20cb 100644 --- a/acceptance/bundle/validate/required_struct_reference/databricks.yml +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -10,7 +10,8 @@ resources: target: cluster_name: target spark_version: 15.4.x-scala2.12 - # clients is a required field of workload_type. Its only field is a reference. + # clients is a required field of workload_type. Both of its fields are references. workload_type: clients: + notebooks: ${resources.clusters.source.is_single_node} jobs: ${resources.clusters.source.is_single_node} diff --git a/acceptance/bundle/validate/required_struct_reference/output.txt b/acceptance/bundle/validate/required_struct_reference/output.txt index 36dd96afff2..2fc3e5bc32e 100644 --- a/acceptance/bundle/validate/required_struct_reference/output.txt +++ b/acceptance/bundle/validate/required_struct_reference/output.txt @@ -3,6 +3,6 @@ { "clients": { "jobs": "${resources.clusters.source.is_single_node}", - "notebooks": true + "notebooks": "${resources.clusters.source.is_single_node}" } } diff --git a/acceptance/bundle/validate/required_struct_reference/script b/acceptance/bundle/validate/required_struct_reference/script index 2975f06e053..5d37f63d12b 100644 --- a/acceptance/bundle/validate/required_struct_reference/script +++ b/acceptance/bundle/validate/required_struct_reference/script @@ -1,2 +1,2 @@ -# workload_type.clients is required. A clients that only has a reference in it is not reported as missing. +# workload_type.clients is required. A clients whose fields are all references is not reported as missing. trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type From b78834add79d98262fb2c6b12df199d94c527e7a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 22 Oct 2026 11:37:27 +0200 Subject: [PATCH 04/14] acc: record that a target can add an explicit null that keeps the default off --- .../target_adds_null_keeps_it/databricks.yml | 18 ++++++++++++++++++ .../nulls/target_adds_null_keeps_it/nb.py | 2 ++ .../target_adds_null_keeps_it/out.test.toml | 2 ++ .../nulls/target_adds_null_keeps_it/output.txt | 7 +++++++ .../nulls/target_adds_null_keeps_it/script | 2 ++ 5 files changed, 31 insertions(+) create mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py create mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt create mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml new file mode 100644 index 00000000000..b5a5c0a0aca --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +resources: + jobs: + j: + name: j + tasks: + - task_key: t + notebook_task: + notebook_path: ./nb.py + +targets: + dev: + resources: + jobs: + j: + queue: null diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt new file mode 100644 index 00000000000..3f0a6236b88 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt @@ -0,0 +1,7 @@ + +>>> [CLI] bundle validate -t dev -o json +Warning: required field "enabled" is not set + at resources.jobs.j.queue + in databricks.yml:18:18 + +null diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script new file mode 100644 index 00000000000..5b7d2ba4bdb --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script @@ -0,0 +1,2 @@ +# A target can add an explicit null that is kept, like a null in the base file: the default queue is not applied. +trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j.queue' From 98a4abbf44694c52ccee245c2606518f75d77c97 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 27 Oct 2026 12:25:08 +0100 Subject: [PATCH 05/14] acc: say plainly that the preset tag is not applied, mark it as Badness, and show has_queue --- .../databricks.yml | 2 +- .../out.test.toml | 0 .../output.txt | 0 .../preset_tags_not_applied_to_whole_map_reference/script | 5 +++++ .../preset_tags_not_applied_to_whole_map_reference/test.toml | 1 + .../resource_deps/whole_map_reference_hides_entries/script | 3 --- .../validate/nulls/target_adds_null_keeps_it/output.txt | 5 ++++- .../bundle/validate/nulls/target_adds_null_keeps_it/script | 4 ++-- 8 files changed, 13 insertions(+), 7 deletions(-) rename acceptance/bundle/resource_deps/{whole_map_reference_hides_entries => preset_tags_not_applied_to_whole_map_reference}/databricks.yml (74%) rename acceptance/bundle/resource_deps/{whole_map_reference_hides_entries => preset_tags_not_applied_to_whole_map_reference}/out.test.toml (100%) rename acceptance/bundle/resource_deps/{whole_map_reference_hides_entries => preset_tags_not_applied_to_whole_map_reference}/output.txt (100%) create mode 100644 acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script create mode 100644 acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml delete mode 100644 acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml similarity index 74% rename from acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml rename to acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml index 511a289e603..9b2e3df1ddf 100644 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml @@ -14,5 +14,5 @@ resources: jobs: j: name: j - # The whole tags map is a reference, so the preset tag above is not added to it. + # tags is a reference to a whole map, and the preset tag above is not added to it. tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml similarity index 100% rename from acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml rename to acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt similarity index 100% rename from acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt rename to acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script new file mode 100644 index 00000000000..a28b61a766a --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script @@ -0,0 +1,5 @@ +# The preset tag is not applied to a job whose tags is a reference to a whole map: the job gets the +# tags of the referenced schema only, and depends on the schema. +# The preset value references the job itself. If the preset tag were applied, this plan would fail +# with a cycle error. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml new file mode 100644 index 00000000000..e329f5af27f --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml @@ -0,0 +1 @@ +Badness = "presets.tags are silently not applied to a job whose tags is a reference to a whole map." diff --git a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script b/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script deleted file mode 100644 index 5e421cc2150..00000000000 --- a/acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script +++ /dev/null @@ -1,3 +0,0 @@ -# When a whole map is a reference, the entries a preset adds to it are not applied: the job gets the -# tags of the referenced schema only, and depends on the schema. -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt index 3f0a6236b88..f09872cb27b 100644 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt @@ -4,4 +4,7 @@ Warning: required field "enabled" is not set at resources.jobs.j.queue in databricks.yml:18:18 -null +{ + "has_queue": true, + "queue": null +} diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script index 5b7d2ba4bdb..5b46b7cab90 100644 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script @@ -1,2 +1,2 @@ -# A target can add an explicit null that is kept, like a null in the base file: the default queue is not applied. -trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j.queue' +# A target can add an explicit null that is kept, like a null in the base file: queue stays null and the default queue is not applied. +trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j | {has_queue: has("queue"), queue}' From 804651217f499cf0ddb698f911af1e475af2042a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 27 Oct 2026 12:26:22 +0100 Subject: [PATCH 06/14] acc: record the whole job in the null-merge test --- .../target_adds_null_keeps_it/output.txt | 19 +++++++++++++++++-- .../nulls/target_adds_null_keeps_it/script | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt index f09872cb27b..fa2f1e4fbb0 100644 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt @@ -5,6 +5,21 @@ Warning: required field "enabled" is not set in databricks.yml:18:18 { - "has_queue": true, - "queue": null + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "j", + "queue": null, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb" + }, + "task_key": "t" + } + ] } diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script index 5b46b7cab90..d97f60de2bd 100644 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script +++ b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script @@ -1,2 +1,2 @@ # A target can add an explicit null that is kept, like a null in the base file: queue stays null and the default queue is not applied. -trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j | {has_queue: has("queue"), queue}' +trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j' From df199b4d0b940705094fd79c3b4d8fb3e65eebea Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 27 Oct 2026 13:11:25 +0100 Subject: [PATCH 07/14] acc: drop the null-merge acceptance test; a unit test covers it --- .../target_adds_null_keeps_it/databricks.yml | 18 ------------- .../nulls/target_adds_null_keeps_it/nb.py | 2 -- .../target_adds_null_keeps_it/out.test.toml | 2 -- .../target_adds_null_keeps_it/output.txt | 25 ------------------- .../nulls/target_adds_null_keeps_it/script | 2 -- 5 files changed, 49 deletions(-) delete mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml delete mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py delete mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml delete mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt delete mode 100644 acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml deleted file mode 100644 index b5a5c0a0aca..00000000000 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml +++ /dev/null @@ -1,18 +0,0 @@ -bundle: - name: test-bundle - -resources: - jobs: - j: - name: j - tasks: - - task_key: t - notebook_task: - notebook_path: ./nb.py - -targets: - dev: - resources: - jobs: - j: - queue: null diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py deleted file mode 100644 index 7c87acfa794..00000000000 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print(1) diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt deleted file mode 100644 index fa2f1e4fbb0..00000000000 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt +++ /dev/null @@ -1,25 +0,0 @@ - ->>> [CLI] bundle validate -t dev -o json -Warning: required field "enabled" is not set - at resources.jobs.j.queue - in databricks.yml:18:18 - -{ - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "j", - "queue": null, - "tasks": [ - { - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb" - }, - "task_key": "t" - } - ] -} diff --git a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script b/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script deleted file mode 100644 index d97f60de2bd..00000000000 --- a/acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script +++ /dev/null @@ -1,2 +0,0 @@ -# A target can add an explicit null that is kept, like a null in the base file: queue stays null and the default queue is not applied. -trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j' From 6441ba005adc87584ba2a61f0e4320e0b4a2ecce Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 27 Oct 2026 21:33:14 +0100 Subject: [PATCH 08/14] acc: record how presets handle a whole-schedule or whole-trigger reference and a referenced name --- .../databricks.yml | 17 +++++++++++++++++ .../presets/name_prefix_reference_to_name/nb.py | 2 ++ .../name_prefix_reference_to_name/out.test.toml | 2 ++ .../name_prefix_reference_to_name/output.txt | 5 +++++ .../name_prefix_reference_to_name/script | 2 ++ .../name_prefix_reference_to_name/test.toml | 1 + .../databricks.yml | 17 +++++++++++++++++ .../nb.py | 2 ++ .../out.test.toml | 2 ++ .../output.txt | 11 +++++++++++ .../script | 3 +++ .../test.toml | 1 + .../databricks.yml | 17 +++++++++++++++++ .../nb.py | 2 ++ .../out.test.toml | 2 ++ .../output.txt | 11 +++++++++++ .../script | 2 ++ .../test.toml | 1 + 18 files changed, 100 insertions(+) create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/nb.py create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/output.txt create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/script create mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/test.toml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script create mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml new file mode 100644 index 00000000000..fa7ad2b0783 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + name_prefix: "pre-" + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + name: ${resources.jobs.base.name} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py b/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml b/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt new file mode 100644 index 00000000000..e07fab23d09 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle plan -o json +{ + "resources.jobs.viaref": "pre-pre-base" +} diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/script b/acceptance/bundle/presets/name_prefix_reference_to_name/script new file mode 100644 index 00000000000..f43b4a8b5aa --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/script @@ -0,0 +1,2 @@ +# name_prefix is added to a name that is a reference to the name of another job, which has the prefix too. +trace $CLI bundle plan -o json | jq '.plan | del(.["resources.jobs.base"]) | map_values(.new_state.value.name)' diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml b/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml new file mode 100644 index 00000000000..0f1ffaaa6c1 --- /dev/null +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/test.toml @@ -0,0 +1 @@ +Badness = "name_prefix is added a second time to a name that is a reference to the name of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml new file mode 100644 index 00000000000..0ce3005fef5 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + trigger_pause_status: PAUSED + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + schedule: ${resources.jobs.base.schedule} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt new file mode 100644 index 00000000000..3709ef8711a --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt @@ -0,0 +1,11 @@ + +>>> musterr [CLI] bundle validate +Error: expected a map to index "pause_status", found string + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script new file mode 100644 index 00000000000..a7d47981807 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script @@ -0,0 +1,3 @@ +# trigger_pause_status needs to set pause_status inside the schedule, but the schedule of viaref is a reference to the whole +# schedule of another job and is still unresolved at that point. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml new file mode 100644 index 00000000000..04077d22022 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml @@ -0,0 +1 @@ +Badness = "trigger_pause_status fails with a confusing error when the schedule is a reference to the whole schedule of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml new file mode 100644 index 00000000000..5c536a73738 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +presets: + trigger_pause_status: PAUSED + +resources: + jobs: + base: + name: base + schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} + trigger: {file_arrival: {url: "s3://bucket/path"}} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] + viaref: + name: viaref + trigger: ${resources.jobs.base.trigger} + tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py new file mode 100644 index 00000000000..7c87acfa794 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt new file mode 100644 index 00000000000..3709ef8711a --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt @@ -0,0 +1,11 @@ + +>>> musterr [CLI] bundle validate +Error: expected a map to index "pause_status", found string + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script new file mode 100644 index 00000000000..6646655bab8 --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script @@ -0,0 +1,2 @@ +# Same as with a schedule: pause_status cannot be set inside a trigger that is a reference to the whole trigger of another job. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml new file mode 100644 index 00000000000..b1915e567cd --- /dev/null +++ b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml @@ -0,0 +1 @@ +Badness = "trigger_pause_status fails with a confusing error when the trigger is a reference to the whole trigger of another job." From fa012c806c561740053f1dca308913f5945a1b04 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 28 Oct 2026 13:25:02 +0100 Subject: [PATCH 09/14] acc: the schedule and trigger reference tests do not involve a preset; say what is broken --- .../trigger_pause_status_whole_schedule_reference/script | 3 --- .../trigger_pause_status_whole_schedule_reference/test.toml | 1 - .../trigger_pause_status_whole_trigger_reference/script | 2 -- .../trigger_pause_status_whole_trigger_reference/test.toml | 1 - .../references/job_schedule_whole_reference}/databricks.yml | 3 --- .../references/job_schedule_whole_reference}/nb.py | 0 .../references/job_schedule_whole_reference}/out.test.toml | 0 .../references/job_schedule_whole_reference}/output.txt | 0 .../validate/references/job_schedule_whole_reference/script | 3 +++ .../validate/references/job_schedule_whole_reference/test.toml | 1 + .../references/job_trigger_whole_reference}/databricks.yml | 3 --- .../references/job_trigger_whole_reference}/nb.py | 0 .../references/job_trigger_whole_reference}/out.test.toml | 0 .../references/job_trigger_whole_reference}/output.txt | 0 .../validate/references/job_trigger_whole_reference/script | 3 +++ .../validate/references/job_trigger_whole_reference/test.toml | 1 + 16 files changed, 8 insertions(+), 13 deletions(-) delete mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script delete mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml delete mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script delete mode 100644 acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml rename acceptance/bundle/{presets/trigger_pause_status_whole_schedule_reference => validate/references/job_schedule_whole_reference}/databricks.yml (91%) rename acceptance/bundle/{presets/trigger_pause_status_whole_schedule_reference => validate/references/job_schedule_whole_reference}/nb.py (100%) rename acceptance/bundle/{presets/trigger_pause_status_whole_schedule_reference => validate/references/job_schedule_whole_reference}/out.test.toml (100%) rename acceptance/bundle/{presets/trigger_pause_status_whole_schedule_reference => validate/references/job_schedule_whole_reference}/output.txt (100%) create mode 100644 acceptance/bundle/validate/references/job_schedule_whole_reference/script create mode 100644 acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml rename acceptance/bundle/{presets/trigger_pause_status_whole_trigger_reference => validate/references/job_trigger_whole_reference}/databricks.yml (91%) rename acceptance/bundle/{presets/trigger_pause_status_whole_trigger_reference => validate/references/job_trigger_whole_reference}/nb.py (100%) rename acceptance/bundle/{presets/trigger_pause_status_whole_trigger_reference => validate/references/job_trigger_whole_reference}/out.test.toml (100%) rename acceptance/bundle/{presets/trigger_pause_status_whole_trigger_reference => validate/references/job_trigger_whole_reference}/output.txt (100%) create mode 100644 acceptance/bundle/validate/references/job_trigger_whole_reference/script create mode 100644 acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script deleted file mode 100644 index a7d47981807..00000000000 --- a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/script +++ /dev/null @@ -1,3 +0,0 @@ -# trigger_pause_status needs to set pause_status inside the schedule, but the schedule of viaref is a reference to the whole -# schedule of another job and is still unresolved at that point. -trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml deleted file mode 100644 index 04077d22022..00000000000 --- a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/test.toml +++ /dev/null @@ -1 +0,0 @@ -Badness = "trigger_pause_status fails with a confusing error when the schedule is a reference to the whole schedule of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script deleted file mode 100644 index 6646655bab8..00000000000 --- a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/script +++ /dev/null @@ -1,2 +0,0 @@ -# Same as with a schedule: pause_status cannot be set inside a trigger that is a reference to the whole trigger of another job. -trace musterr $CLI bundle validate diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml b/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml deleted file mode 100644 index b1915e567cd..00000000000 --- a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/test.toml +++ /dev/null @@ -1 +0,0 @@ -Badness = "trigger_pause_status fails with a confusing error when the trigger is a reference to the whole trigger of another job." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml b/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml similarity index 91% rename from acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml rename to acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml index 0ce3005fef5..7e2323f8cec 100644 --- a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/databricks.yml +++ b/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml @@ -1,9 +1,6 @@ bundle: name: test-bundle -presets: - trigger_pause_status: PAUSED - resources: jobs: base: diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py b/acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/nb.py rename to acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml b/acceptance/bundle/validate/references/job_schedule_whole_reference/out.test.toml similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/out.test.toml rename to acceptance/bundle/validate/references/job_schedule_whole_reference/out.test.toml diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt b/acceptance/bundle/validate/references/job_schedule_whole_reference/output.txt similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_schedule_reference/output.txt rename to acceptance/bundle/validate/references/job_schedule_whole_reference/output.txt diff --git a/acceptance/bundle/validate/references/job_schedule_whole_reference/script b/acceptance/bundle/validate/references/job_schedule_whole_reference/script new file mode 100644 index 00000000000..4c84b58bb8c --- /dev/null +++ b/acceptance/bundle/validate/references/job_schedule_whole_reference/script @@ -0,0 +1,3 @@ +# The schedule of viaref is a reference to the whole schedule of another job. +# The defaults set schedule.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml b/acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml new file mode 100644 index 00000000000..12ba67a70a9 --- /dev/null +++ b/acceptance/bundle/validate/references/job_schedule_whole_reference/test.toml @@ -0,0 +1 @@ +Badness = "The default of schedule.pause_status is written by path, which fails with 'expected a map to index pause_status, found string' when the schedule is a reference to a whole schedule of another job; validate should accept it." diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml b/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml similarity index 91% rename from acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml rename to acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml index 5c536a73738..6947f6e242a 100644 --- a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/databricks.yml +++ b/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml @@ -1,9 +1,6 @@ bundle: name: test-bundle -presets: - trigger_pause_status: PAUSED - resources: jobs: base: diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py b/acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/nb.py rename to acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml b/acceptance/bundle/validate/references/job_trigger_whole_reference/out.test.toml similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/out.test.toml rename to acceptance/bundle/validate/references/job_trigger_whole_reference/out.test.toml diff --git a/acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt b/acceptance/bundle/validate/references/job_trigger_whole_reference/output.txt similarity index 100% rename from acceptance/bundle/presets/trigger_pause_status_whole_trigger_reference/output.txt rename to acceptance/bundle/validate/references/job_trigger_whole_reference/output.txt diff --git a/acceptance/bundle/validate/references/job_trigger_whole_reference/script b/acceptance/bundle/validate/references/job_trigger_whole_reference/script new file mode 100644 index 00000000000..b02a17e8dff --- /dev/null +++ b/acceptance/bundle/validate/references/job_trigger_whole_reference/script @@ -0,0 +1,3 @@ +# The trigger of viaref is a reference to the whole trigger of another job. +# The defaults set trigger.pause_status (and trigger, continuous) of every job by path, which cannot be done in a reference. +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml b/acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml new file mode 100644 index 00000000000..069b11f08be --- /dev/null +++ b/acceptance/bundle/validate/references/job_trigger_whole_reference/test.toml @@ -0,0 +1 @@ +Badness = "The default of trigger.pause_status is written by path, which fails with 'expected a map to index pause_status, found string' when the trigger is a reference to a whole trigger of another job; validate should accept it." From 4bf7811f4b982c4231ee044940da9f089dd99dfa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 28 Oct 2026 14:10:13 +0100 Subject: [PATCH 10/14] acc: minimal configs for the recorded edge cases; fix the duplicate name key --- .../databricks.yml | 27 +++++-------------- .../output.txt | 6 ++--- .../databricks.yml | 14 ---------- .../databricks.yml | 5 ---- .../name_prefix_reference_to_name/nb.py | 2 -- .../databricks.yml | 1 - .../databricks.yml | 5 ---- .../job_schedule_whole_reference/nb.py | 2 -- .../databricks.yml | 5 ---- .../job_trigger_whole_reference/nb.py | 2 -- .../required_struct_reference/databricks.yml | 4 --- 11 files changed, 9 insertions(+), 64 deletions(-) delete mode 100644 acceptance/bundle/presets/name_prefix_reference_to_name/nb.py delete mode 100644 acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py delete mode 100644 acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml index d2e913f1abe..b4655b4f0de 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml @@ -6,44 +6,29 @@ variables: cluster: type: complex default: - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge num_workers: 1 resources: clusters: source: - cluster_name: source - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge num_workers: 2 jobs: job: name: job tasks: - - task_key: task - notebook_task: - notebook_path: /Workspace/notebook - # Replaced by bundle.cluster_id: the reference below new_cluster must go too. + # bundle.cluster_id replaces new_cluster. The reference below it must go too. + - task_key: scalar_ref new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge num_workers: ${resources.clusters.source.num_workers} - - task_key: task_var - notebook_task: - notebook_path: /Workspace/notebook - # A reference at new_cluster must not bring it back either. + # A reference to the whole new_cluster must not bring it back. + - task_key: variable_ref new_cluster: ${var.cluster} - - task_key: task_resource_ref - notebook_task: - notebook_path: /Workspace/notebook - # A reference to the new_cluster of another job is resolved after the override. + # A reference to the new_cluster of another job is resolved after the override. + - task_key: resource_ref new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster} other: name: other job_clusters: - job_cluster_key: shared new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge num_workers: 3 diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt index 14e1e4bbefd..974ad463b30 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -2,17 +2,17 @@ >>> [CLI] bundle validate -o json [ { - "task_key": "task", + "task_key": "resource_ref", "existing_cluster_id": "existing-cluster", "new_cluster": null }, { - "task_key": "task_resource_ref", + "task_key": "scalar_ref", "existing_cluster_id": "existing-cluster", "new_cluster": null }, { - "task_key": "task_var", + "task_key": "variable_ref", "existing_cluster_id": "existing-cluster", "new_cluster": null } diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml index 25315e8078e..b560fc14c1d 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml @@ -7,12 +7,6 @@ resources: table_name: main.default.src assets_dir: /Workspace/monitors/src output_schema_name: main.default - inference_log: - granularities: [1 day] - model_id_col: model_id - prediction_col: prediction - problem_type: PROBLEM_TYPE_CLASSIFICATION - timestamp_col: timestamp schedule: quartz_cron_expression: "0 0 12 * * ?" timezone_id: UTC @@ -20,14 +14,6 @@ resources: table_name: main.default.dst assets_dir: /Workspace/monitors/dst output_schema_name: main.default - inference_log: - granularities: [1 day] - model_id_col: model_id - prediction_col: prediction - problem_type: PROBLEM_TYPE_CLASSIFICATION - timestamp_col: timestamp - # Forced null: the target override replaces it with a reference. - schedule: null targets: dev: diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml index fa7ad2b0783..5f33a16ea97 100644 --- a/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/databricks.yml @@ -8,10 +8,5 @@ resources: jobs: base: name: base - schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} - trigger: {file_arrival: {url: "s3://bucket/path"}} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] viaref: - name: viaref name: ${resources.jobs.base.name} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py b/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py deleted file mode 100644 index 7c87acfa794..00000000000 --- a/acceptance/bundle/presets/name_prefix_reference_to_name/nb.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print(1) diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml index 9b2e3df1ddf..966651d418e 100644 --- a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml @@ -13,6 +13,5 @@ resources: properties: {owner: fixed} jobs: j: - name: j # tags is a reference to a whole map, and the preset tag above is not added to it. tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml b/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml index 7e2323f8cec..abc96128e58 100644 --- a/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml +++ b/acceptance/bundle/validate/references/job_schedule_whole_reference/databricks.yml @@ -4,11 +4,6 @@ bundle: resources: jobs: base: - name: base schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} - trigger: {file_arrival: {url: "s3://bucket/path"}} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] viaref: - name: viaref schedule: ${resources.jobs.base.schedule} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py b/acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py deleted file mode 100644 index 7c87acfa794..00000000000 --- a/acceptance/bundle/validate/references/job_schedule_whole_reference/nb.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print(1) diff --git a/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml b/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml index 6947f6e242a..2466b747ca1 100644 --- a/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml +++ b/acceptance/bundle/validate/references/job_trigger_whole_reference/databricks.yml @@ -4,11 +4,6 @@ bundle: resources: jobs: base: - name: base - schedule: {quartz_cron_expression: "0 0 * * * ?", timezone_id: UTC} trigger: {file_arrival: {url: "s3://bucket/path"}} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] viaref: - name: viaref trigger: ${resources.jobs.base.trigger} - tasks: [{task_key: t, notebook_task: {notebook_path: ./nb.py}}] diff --git a/acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py b/acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py deleted file mode 100644 index 7c87acfa794..00000000000 --- a/acceptance/bundle/validate/references/job_trigger_whole_reference/nb.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print(1) diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml index 60e02ad20cb..b86df10edca 100644 --- a/acceptance/bundle/validate/required_struct_reference/databricks.yml +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -4,12 +4,8 @@ bundle: resources: clusters: source: - cluster_name: source - spark_version: 15.4.x-scala2.12 is_single_node: true target: - cluster_name: target - spark_version: 15.4.x-scala2.12 # clients is a required field of workload_type. Both of its fields are references. workload_type: clients: From 5c12fafe37d9f042d5b0d8ef596aca06bef1978b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 28 Oct 2026 14:27:13 +0100 Subject: [PATCH 11/14] acc: plain field selection in the scripts of the recorded edge cases --- .../cluster_id_clears_new_cluster_reference/output.txt | 9 +++------ .../cluster_id_clears_new_cluster_reference/script | 2 +- .../preset_clears_forced_schedule_reference/output.txt | 4 ++-- .../preset_clears_forced_schedule_reference/script | 2 +- .../presets/name_prefix_reference_to_name/output.txt | 4 +--- .../presets/name_prefix_reference_to_name/script | 2 +- .../output.txt | 10 +--------- .../script | 4 ++-- 8 files changed, 12 insertions(+), 25 deletions(-) diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt index 974ad463b30..ae756364995 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -2,19 +2,16 @@ >>> [CLI] bundle validate -o json [ { - "task_key": "resource_ref", "existing_cluster_id": "existing-cluster", - "new_cluster": null + "task_key": "resource_ref" }, { - "task_key": "scalar_ref", "existing_cluster_id": "existing-cluster", - "new_cluster": null + "task_key": "scalar_ref" }, { - "task_key": "variable_ref", "existing_cluster_id": "existing-cluster", - "new_cluster": null + "task_key": "variable_ref" } ] diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script index 2016268365d..4095867d170 100644 --- a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script @@ -1,3 +1,3 @@ # bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference. -trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]' +trace $CLI bundle validate -o json | jq .resources.jobs.job.tasks trace $CLI bundle deploy diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt index eb07e4ec538..0250130cf15 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt @@ -8,6 +8,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //tables/main.default.dst/monitor { - "has_schedule": false, - "schedule": null + "assets_dir": "/Workspace/monitors/dst", + "output_schema_name": "main.default" } diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script index 37d355db8c8..a1eaa647920 100644 --- a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -1,4 +1,4 @@ # trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst # with a reference to the schedule of src, which is removed too: the request carries no schedule. trace $CLI bundle deploy -trace print_requests.py //tables/main.default.dst/monitor | jq '.body | {has_schedule: has("schedule"), schedule}' +trace print_requests.py //tables/main.default.dst/monitor | jq .body diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt index e07fab23d09..2a08a8c68d0 100644 --- a/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/output.txt @@ -1,5 +1,3 @@ >>> [CLI] bundle plan -o json -{ - "resources.jobs.viaref": "pre-pre-base" -} +"pre-pre-base" diff --git a/acceptance/bundle/presets/name_prefix_reference_to_name/script b/acceptance/bundle/presets/name_prefix_reference_to_name/script index f43b4a8b5aa..60df63696eb 100644 --- a/acceptance/bundle/presets/name_prefix_reference_to_name/script +++ b/acceptance/bundle/presets/name_prefix_reference_to_name/script @@ -1,2 +1,2 @@ # name_prefix is added to a name that is a reference to the name of another job, which has the prefix too. -trace $CLI bundle plan -o json | jq '.plan | del(.["resources.jobs.base"]) | map_values(.new_state.value.name)' +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.viaref"].new_state.value.name' diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt index 06316029ed1..5f55baff011 100644 --- a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt @@ -1,13 +1,5 @@ >>> [CLI] bundle plan -o json { - "tags": { - "owner": "fixed" - }, - "depends_on": [ - { - "node": "resources.schemas.s", - "label": "${resources.schemas.s.properties}" - } - ] + "owner": "fixed" } diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script index a28b61a766a..c63ddcce013 100644 --- a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script @@ -1,5 +1,5 @@ # The preset tag is not applied to a job whose tags is a reference to a whole map: the job gets the -# tags of the referenced schema only, and depends on the schema. +# tags of the referenced schema only. # The preset value references the job itself. If the preset tag were applied, this plan would fail # with a cycle error. -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"].new_state.value.tags' From d05396d931111bad733960616f5838eeead7c685 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 21 Oct 2026 13:54:24 +0200 Subject: [PATCH 12/14] bundle: read and edit the typed config in five mutators Co-authored-by: Isaac --- bundle/config/loader/process_root_includes.go | 32 ++--- .../loader/process_root_includes_test.go | 27 +++- .../mutator/prepend_workspace_prefix.go | 57 +++----- .../apply_bundle_permissions.go | 106 +++++++------- .../apply_bundle_permissions_test.go | 30 ++++ .../expand_pipeline_glob_paths.go | 133 ++++++++---------- .../expand_pipeline_glob_paths_test.go | 72 ++++++++++ .../process_static_resources.go | 25 +--- .../process_static_resources_test.go | 32 ++++- 9 files changed, 306 insertions(+), 208 deletions(-) diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 655ca27162b..20e80564b0f 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type processRootIncludes struct{} @@ -42,7 +41,8 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Preserve the raw include patterns before they are replaced below with the // expanded list of loaded files, so IsFileIncluded can re-match against them. - b.SetIncludePatterns(b.Config.Include) + patterns := b.Config.Include + b.SetIncludePatterns(patterns) // Map with files we've already seen to avoid loading them twice. seen := map[string]bool{} @@ -131,27 +131,6 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } } - // Swap out the original includes list with the expanded globs. This goes through - // Mutate so the dynamic tree is updated too: the includes below are applied without - // their own mutator scope, so nothing converts the typed field back into the dynamic - // tree afterwards, and the next ToTyped would otherwise restore the raw patterns. - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Include is omitempty in the typed configuration, so an empty list must stay - // absent from the dynamic tree rather than be written as []. - if len(files) == 0 { - return dyn.DropKeys(root, []string{"include"}) - } - - includeValues := make([]dyn.Value, 0, len(files)) - for _, file := range files { - includeValues = append(includeValues, dyn.V(file)) - } - return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations())) - }) - if err != nil { - return diag.FromErr(err) - } - // Track number of bundle YAML (or JSON) files in the configuration. The +1 is there // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 @@ -161,5 +140,12 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // whole accumulated configuration per file, making load quadratic in the number of // included files (~20 minutes for 6000 files). bundle.ApplySeqInScopeContext(ctx, b, out...) + + // Swap out the original include patterns with the expanded globs. This is done after + // the includes are merged: [config.Root.Merge] reloads the typed configuration from + // the dynamic tree, which still holds the patterns, followed by the include sections + // of included files (merging concatenates sequences). Those are kept as they are. + // Include is omitempty, so an empty list is absent from the configuration. + b.Config.Include = append(files, b.Config.Include[len(patterns):]...) return nil } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index e7a850a835b..a3d1c2fec88 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -1,6 +1,7 @@ package loader_test import ( + "path/filepath" "runtime" "testing" @@ -100,8 +101,8 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { } // The expanded include list must be visible in both the typed and the dynamic -// configuration: the per-file includes are applied without their own mutator scope, so -// nothing converts the typed field back into the dynamic tree afterwards. +// configuration: merging the per-file includes reloads the typed configuration from the +// dynamic tree, so the expanded list must not be overwritten by the raw patterns. func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: t.TempDir(), @@ -142,6 +143,28 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { assert.Equal(t, dyn.KindInvalid, b.Config.Value().Get("include").Kind()) } +// Include sections of included files are not applied, but they are merged into the +// include list after the expanded globs. +func TestProcessRootIncludesKeepsIncludesOfIncludedFiles(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), "include:\n - c.yml\n") + testutil.Touch(t, b.BundleRootPath, "b.yml") + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"a.yml", "b.yml", "c.yml"}, b.Config.Include) + assert.Equal(t, []any{"a.yml", "b.yml", "c.yml"}, b.Config.Value().Get("include").AsAny()) +} + func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: t.TempDir(), diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go index 3124244c857..45af605c5f9 100644 --- a/bundle/config/mutator/prepend_workspace_prefix.go +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -2,12 +2,10 @@ package mutator import ( "context" - "fmt" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type prependWorkspacePrefix struct{} @@ -27,46 +25,31 @@ var skipPrefixes = []string{ } func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), - dyn.NewPattern(dyn.Key("workspace"), dyn.Key("resource_path")), + w := &b.Config.Workspace + for _, path := range []*string{ + &w.RootPath, + &w.FilePath, + &w.ArtifactPath, + &w.StatePath, + &w.ResourcePath, + } { + *path = prependWorkspacePrefixTo(*path) } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { - path, ok := pv.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected string, got %s", pv.Kind()) - } - - // Skip prefixing if the path does not start with /, it might be variable reference or smth else. - if !strings.HasPrefix(path, "/") { - return pv, nil - } + return nil +} - for _, prefix := range skipPrefixes { - if strings.HasPrefix(path, prefix) { - return pv, nil - } - } +func prependWorkspacePrefixTo(path string) string { + // Skip prefixing if the path does not start with /, it might be variable reference or smth else. + if !strings.HasPrefix(path, "/") { + return path + } - // Use pv's locations, not the root v's, so diagnostics point at the original config line. - return dyn.NewValue("/Workspace"+path, pv.Locations()), nil - }) - if err != nil { - return dyn.InvalidValue, err - } + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return path } - return v, nil - }) - if err != nil { - return diag.FromErr(err) } - return nil + return "/Workspace" + path } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index f2a6d857090..15493ae9d15 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -3,6 +3,8 @@ package resourcemutator import ( "context" "fmt" + "maps" + "reflect" "slices" "strings" @@ -11,8 +13,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -108,63 +108,65 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di return diag.FromErr(err) } - patterns := make(map[string]dyn.Pattern, 0) - for key := range levelsMap { - patterns[key] = dyn.NewPattern( - dyn.Key("resources"), - dyn.Key(key), - dyn.AnyKey(), - ) - } - - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - for key, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - var permissions []resources.Permission - pv, err := dyn.Get(v, "permissions") - // If the permissions field is not found, we set to an empty array - if err != nil { - pv = dyn.V([]dyn.Value{}) - } - - err = convert.ToTyped(&permissions, pv) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } - - permissions = append(permissions, convertPermissions( - ctx, - b.Config.Permissions, - permissions, - key, - levelsMap[key], - )...) - - if len(permissions) == 0 { - permissions = nil - } - - pv, err = convert.FromTyped(permissions, dyn.NilValue) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to convert permissions: %w", err) - } - - return dyn.Set(v, "permissions", pv) - }) - if err != nil { - return dyn.InvalidValue, err - } + for _, group := range b.Config.Resources.AllResources() { + key := group.Description.PluralName + lm, ok := levelsMap[key] + if !ok { + continue } - return v, nil - }) - if err != nil { - return diag.FromErr(err) + for _, name := range slices.Sorted(maps.Keys(group.Resources)) { + field := reflect.ValueOf(group.Resources[name]).Elem().FieldByName("Permissions") + permissions := readPermissions(field) + permissions = append(permissions, convertPermissions( + ctx, + b.Config.Permissions, + permissions, + key, + lm, + )...) + writePermissions(field, permissions) + } } return nil } +// readPermissions converts the permissions field of a resource to []resources.Permission. +// The element type of the field differs between resources, but all have the same fields. +func readPermissions(field reflect.Value) []resources.Permission { + var permissions []resources.Permission + for i := range field.Len() { + p := field.Index(i) + permissions = append(permissions, resources.Permission{ + Level: iam.PermissionLevel(p.FieldByName("Level").String()), + UserName: p.FieldByName("UserName").String(), + ServicePrincipalName: p.FieldByName("ServicePrincipalName").String(), + GroupName: p.FieldByName("GroupName").String(), + }) + } + return permissions +} + +// writePermissions sets the permissions field of a resource to permissions. +// An empty list is stored as nil, so that the field is absent from the configuration. +func writePermissions(field reflect.Value, permissions []resources.Permission) { + if len(permissions) == 0 { + field.SetZero() + return + } + + out := reflect.MakeSlice(field.Type(), len(permissions), len(permissions)) + for i, p := range permissions { + e := out.Index(i) + e.FieldByName("Level").SetString(string(p.Level)) + e.FieldByName("UserName").SetString(p.UserName) + e.FieldByName("ServicePrincipalName").SetString(p.ServicePrincipalName) + e.FieldByName("GroupName").SetString(p.GroupName) + } + field.Set(out) +} + func validatePermissions(b *bundle.Bundle) error { for _, p := range b.Config.Permissions { if !slices.Contains(allowedLevels, string(p.Level)) { diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..610832c2b92 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -237,3 +238,32 @@ func TestAllResourcesExplicitlyDefinedForPermissionsSupport(t *testing.T) { } } } + +func TestApplyBundlePermissionsEmpty(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "empty": { + JobSettings: jobs.JobSettings{Name: "empty"}, + Permissions: []resources.JobPermission{}, + }, + "unset": { + JobSettings: jobs.JobSettings{Name: "unset"}, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, ApplyBundlePermissions()) + require.NoError(t, diags.Error()) + + // An empty list of permissions is dropped, an unset one stays unset. + for _, name := range []string{"empty", "unset"} { + require.Nil(t, b.Config.Resources.Jobs[name].Permissions) + v, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.jobs."+name)) + require.NoError(t, err) + assert.False(t, v.Get("permissions").IsValid(), name) + } +} diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index 38f835793ea..ed4b02054cc 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -2,14 +2,13 @@ package resourcemutator import ( "context" - "fmt" "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type expandPipelineGlobPaths struct{} @@ -18,95 +17,83 @@ func ExpandPipelineGlobPaths() bundle.Mutator { return &expandPipelineGlobPaths{} } -func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, v dyn.Value) ([]dyn.Value, error) { +// expandLibrary returns the libraries that replace lib: one per match of its glob path. +func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, lib pipelines.PipelineLibrary) ([]pipelines.PipelineLibrary, error) { // Probe for the path field in the library. - for _, p := range []dyn.Path{ - dyn.NewPath(dyn.Key("notebook"), dyn.Key("path")), - dyn.NewPath(dyn.Key("file"), dyn.Key("path")), - } { - pv, err := dyn.GetByPath(v, p) - if dyn.IsNoSuchKeyError(err) { - continue - } - if err != nil { - return nil, err + var path string + var withPath func(path string) pipelines.PipelineLibrary + switch { + case lib.Notebook != nil && lib.Notebook.Path != "": + path = lib.Notebook.Path + withPath = func(path string) pipelines.PipelineLibrary { + nl, notebook := lib, *lib.Notebook + notebook.Path = path + nl.Notebook = ¬ebook + return nl } - - // If the path is empty or not a local path, return the original value. - path := pv.MustString() - if path == "" || !libraries.IsLocalPath(path) { - return []dyn.Value{v}, nil - } - - matches, err := filepath.Glob(filepath.Join(dir, path)) - if err != nil { - return nil, err - } - - // If there are no matches, return the original value. - if len(matches) == 0 { - return []dyn.Value{v}, nil - } - - matches = patchwheel.FilterLatestWheels(ctx, matches) - - // Emit a new value for each match. - var ev []dyn.Value - for _, match := range matches { - m, err := filepath.Rel(dir, match) - if err != nil { - return nil, err - } - nv, err := dyn.SetByPath(v, p, dyn.NewValue(filepath.ToSlash(m), pv.Locations())) - if err != nil { - return nil, err - } - ev = append(ev, nv) + case lib.File != nil && lib.File.Path != "": + path = lib.File.Path + withPath = func(path string) pipelines.PipelineLibrary { + nl, file := lib, *lib.File + file.Path = path + nl.File = &file + return nl } + default: + // Neither of the library paths were found (or the path is empty). This is likely + // an invalid node, but it isn't this mutator's job to enforce that. Return the original value. + return []pipelines.PipelineLibrary{lib}, nil + } - return ev, nil + // If the path is not a local path, return the original value. + if !libraries.IsLocalPath(path) { + return []pipelines.PipelineLibrary{lib}, nil } - // Neither of the library paths were found. This is likely an invalid node, - // but it isn't this mutator's job to enforce that. Return the original value. - return []dyn.Value{v}, nil -} + matches, err := filepath.Glob(filepath.Join(dir, path)) + if err != nil { + return nil, err + } -func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) { - s, ok := v.AsSequence() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected sequence, got %s", v.Kind()) + // If there are no matches, return the original value. + if len(matches) == 0 { + return []pipelines.PipelineLibrary{lib}, nil } - var vs []dyn.Value - for _, sv := range s { - v, err := m.expandLibrary(ctx, dir, sv) + matches = patchwheel.FilterLatestWheels(ctx, matches) + + // Emit a new value for each match. + var expanded []pipelines.PipelineLibrary + for _, match := range matches { + m, err := filepath.Rel(dir, match) if err != nil { - return dyn.InvalidValue, err + return nil, err } - - vs = append(vs, v...) + expanded = append(expanded, withPath(filepath.ToSlash(m))) } - return dyn.NewValue(vs, v.Locations()), nil + return expanded, nil } func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - p := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("libraries"), - ) + // Visit each pipeline's "libraries" field and expand any glob patterns. + for _, pipeline := range b.Config.Resources.Pipelines { + if len(pipeline.Libraries) == 0 { + continue + } - // Visit each pipeline's "libraries" field and expand any glob patterns. - return dyn.MapByPattern(v, p, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return m.expandSequence(ctx, b.BundleRootPath, path, value) - }) - }) + var expanded []pipelines.PipelineLibrary + for _, lib := range pipeline.Libraries { + libs, err := m.expandLibrary(ctx, b.BundleRootPath, lib) + if err != nil { + return diag.FromErr(err) + } + expanded = append(expanded, libs...) + } + pipeline.Libraries = expanded + } - return diag.FromErr(err) + return nil } func (*expandPipelineGlobPaths) Name() string { diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go index ead2084d835..7b640df45e1 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths_test.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" @@ -198,3 +199,74 @@ func collectFiles(libraries []pipelines.PipelineLibrary) []string { return paths } + +func TestExpandGlobPathsInPipelinesKeepsOrder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "a/1.py")) + touchEmptyFile(t, filepath.Join(dir, "a/2.py")) + touchEmptyFile(t, filepath.Join(dir, "n.ipynb")) + + glob := &pipelines.FileLibrary{Path: "a/*.py"} + b := &bundle.Bundle{ + BundleRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Libraries: []pipelines.PipelineLibrary{ + {Notebook: &pipelines.NotebookLibrary{Path: "n.ipynb"}}, + {File: glob}, + {File: &pipelines.FileLibrary{Path: "/Workspace/x.py"}}, + {Notebook: &pipelines.NotebookLibrary{}}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandPipelineGlobPaths()) + require.NoError(t, diags.Error()) + + assert.Equal(t, []pipelines.PipelineLibrary{ + {Notebook: &pipelines.NotebookLibrary{Path: "n.ipynb"}}, + {File: &pipelines.FileLibrary{Path: "a/1.py"}}, + {File: &pipelines.FileLibrary{Path: "a/2.py"}}, + {File: &pipelines.FileLibrary{Path: "/Workspace/x.py"}}, + // The empty notebook is kept; the round trip through the dynamic configuration forces it. + {Notebook: &pipelines.NotebookLibrary{}, ForceSendFields: []string{"Notebook"}}, + }, b.Config.Resources.Pipelines["pipeline"].Libraries) + + // The original library is not modified. + assert.Equal(t, "a/*.py", glob.Path) +} + +func TestExpandGlobPathsInPipelinesWithoutLibraries(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "unset": {}, + "empty": { + CreatePipeline: pipelines.CreatePipeline{ + Libraries: []pipelines.PipelineLibrary{}, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandPipelineGlobPaths()) + require.NoError(t, diags.Error()) + + assert.Nil(t, b.Config.Resources.Pipelines["unset"].Libraries) + assert.Equal(t, []pipelines.PipelineLibrary{}, b.Config.Resources.Pipelines["empty"].Libraries) + + v := b.Config.Value() + assert.False(t, v.Get("resources").Get("pipelines").Get("unset").Get("libraries").IsValid()) + assert.Equal(t, dyn.KindSequence, v.Get("resources").Get("pipelines").Get("empty").Get("libraries").Kind()) +} diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 82aa89ff915..d9c91b260df 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -2,12 +2,10 @@ package resourcemutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" ) @@ -29,10 +27,7 @@ func (p processStaticResources) Name() string { } func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - addedResources, err := getAllResources(b) - if err != nil { - return diag.FromErr(err) - } + addedResources := getAllResources(b) // only YAML resources need to have paths normalized, before normalizing paths // we need to resolve variables because they can change path values: @@ -62,20 +57,12 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia return nil } -func getAllResources(b *bundle.Bundle) (ResourceKeySet, error) { +func getAllResources(b *bundle.Bundle) ResourceKeySet { set := NewResourceKeySet() - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()) - err := set.AddPattern(pattern, root) - if err != nil { - return dyn.InvalidValue, err + for _, group := range b.Config.Resources.AllResources() { + for name := range group.Resources { + set.AddResourceKey(ResourceKey{Type: group.Description.PluralName, Name: name}) } - - return root, nil - }) - if err != nil { - return nil, fmt.Errorf("failed to collect resources: %w", err) } - - return set, nil + return set } diff --git a/bundle/config/mutator/resourcemutator/process_static_resources_test.go b/bundle/config/mutator/resourcemutator/process_static_resources_test.go index 48cee2d8faa..72f8e986a45 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources_test.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources_test.go @@ -1,12 +1,16 @@ package resourcemutator import ( + "reflect" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetAllResources(t *testing.T) { @@ -21,9 +25,8 @@ func TestGetAllResources(t *testing.T) { }, } - set, err := getAllResources(b) + set := getAllResources(b) - assert.NoError(t, err) assert.ElementsMatch( t, set.ToArray(), @@ -33,3 +36,28 @@ func TestGetAllResources(t *testing.T) { }, ) } + +// TestGetAllResourcesMatchesConfigKeys checks that every resource type is collected +// under the same key it has in the configuration (resources..). +func TestGetAllResourcesMatchesConfigKeys(t *testing.T) { + b := &bundle.Bundle{} + + // Add one resource of every type. + rv := reflect.ValueOf(&b.Config.Resources).Elem() + for i := range rv.NumField() { + field := rv.Field(i) + require.Equal(t, reflect.Map, field.Kind(), rv.Type().Field(i).Name) + m := reflect.MakeMap(field.Type()) + m.SetMapIndex(reflect.ValueOf("my_resource"), reflect.New(field.Type().Elem().Elem())) + field.Set(m) + } + + root, err := convert.FromTyped(b.Config, dyn.NilValue) + require.NoError(t, err) + expected := NewResourceKeySet() + err = expected.AddPattern(dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), root) + require.NoError(t, err) + require.Equal(t, rv.NumField(), expected.Size()) + + assert.Equal(t, expected, getAllResources(b)) +} From 04ddca40986ee825f95ac6048ac580cb5efd8d7d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 21 Oct 2026 14:41:05 +0200 Subject: [PATCH 13/14] acceptance: permission warnings now carry their source location Co-authored-by: Isaac --- .../validate/secret_scope_invalid_permission_level/output.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/validate/secret_scope_invalid_permission_level/output.txt b/acceptance/bundle/validate/secret_scope_invalid_permission_level/output.txt index eec035d2671..4c7c87c6c28 100644 --- a/acceptance/bundle/validate/secret_scope_invalid_permission_level/output.txt +++ b/acceptance/bundle/validate/secret_scope_invalid_permission_level/output.txt @@ -2,9 +2,11 @@ >>> [CLI] bundle validate Warning: invalid value "12" for enum field. Valid values are [READ WRITE MANAGE] at resources.secret_scopes.bad_levels.permissions[0].level + in databricks.yml:10:18 Warning: invalid value "fuzz_badlevel" for enum field. Valid values are [READ WRITE MANAGE] at resources.secret_scopes.bad_levels.permissions[1].level + in databricks.yml:12:18 Name: test-bundle Target: default @@ -17,9 +19,11 @@ Found 2 warnings >>> [CLI] bundle validate --strict Warning: invalid value "12" for enum field. Valid values are [READ WRITE MANAGE] at resources.secret_scopes.bad_levels.permissions[0].level + in databricks.yml:10:18 Warning: invalid value "fuzz_badlevel" for enum field. Valid values are [READ WRITE MANAGE] at resources.secret_scopes.bad_levels.permissions[1].level + in databricks.yml:12:18 Name: test-bundle Target: default From 9092e610f1aed69474271c58033198c2a9efff65 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 21 Oct 2026 18:11:18 +0200 Subject: [PATCH 14/14] Keep rejecting an empty resource name when collecting resources Co-authored-by: Isaac --- .../process_static_resources.go | 13 +++++++++--- .../process_static_resources_test.go | 20 +++++++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index d9c91b260df..06ec593f092 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "errors" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" @@ -27,7 +28,10 @@ func (p processStaticResources) Name() string { } func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - addedResources := getAllResources(b) + addedResources, err := getAllResources(b) + if err != nil { + return diag.FromErr(err) + } // only YAML resources need to have paths normalized, before normalizing paths // we need to resolve variables because they can change path values: @@ -57,12 +61,15 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia return nil } -func getAllResources(b *bundle.Bundle) ResourceKeySet { +func getAllResources(b *bundle.Bundle) (ResourceKeySet, error) { set := NewResourceKeySet() for _, group := range b.Config.Resources.AllResources() { for name := range group.Resources { + if name == "" { + return nil, errors.New("failed to collect resources: can't parse resource key") + } set.AddResourceKey(ResourceKey{Type: group.Description.PluralName, Name: name}) } } - return set + return set, nil } diff --git a/bundle/config/mutator/resourcemutator/process_static_resources_test.go b/bundle/config/mutator/resourcemutator/process_static_resources_test.go index 72f8e986a45..f23242d58e0 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources_test.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources_test.go @@ -25,7 +25,8 @@ func TestGetAllResources(t *testing.T) { }, } - set := getAllResources(b) + set, err := getAllResources(b) + require.NoError(t, err) assert.ElementsMatch( t, @@ -59,5 +60,20 @@ func TestGetAllResourcesMatchesConfigKeys(t *testing.T) { require.NoError(t, err) require.Equal(t, rv.NumField(), expected.Size()) - assert.Equal(t, expected, getAllResources(b)) + set, err := getAllResources(b) + require.NoError(t, err) + assert.Equal(t, expected, set) +} + +func TestGetAllResourcesRejectsEmptyName(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"": {}}, + }, + }, + } + + _, err := getAllResources(b) + assert.EqualError(t, err, "failed to collect resources: can't parse resource key") }