From 921704454735ef7ecd4b8ad2f692fee8f9fdd127 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 30 Oct 2026 14:16:02 +0100 Subject: [PATCH 1/2] acc: record how nulls in targets and includes are ignored today New bundle validate tests for nulls that are meant to remove a value: a resource or a field in a target, a map entry, a list, a field of a task matched by task_key (and job_clusters, job parameters, pipeline clusters), nulls in included files, and target overrides split across included files. Today all of them are ignored. Co-authored-by: Isaac --- .../nulls/include_removes/databricks.yml | 34 ++++++++ .../nulls/include_removes/out.test.toml | 2 + .../validate/nulls/include_removes/output.txt | 58 ++++++++++++++ .../nulls/include_removes/override.yml | 23 ++++++ .../validate/nulls/include_removes/script | 10 +++ .../nulls/include_target_overrides/a.yml | 13 ++++ .../nulls/include_target_overrides/b.yml | 10 +++ .../include_target_overrides/databricks.yml | 31 ++++++++ .../nulls/include_target_overrides/nb.py | 1 + .../include_target_overrides/out.test.toml | 2 + .../nulls/include_target_overrides/output.txt | 43 +++++++++++ .../nulls/include_target_overrides/script | 3 + .../nulls/root_without_base/databricks.yml | 13 ++++ .../nulls/root_without_base/out.test.toml | 2 + .../nulls/root_without_base/output.txt | 23 ++++++ .../validate/nulls/root_without_base/script | 2 + .../nulls/target_keyed_slices/databricks.yml | 61 +++++++++++++++ .../validate/nulls/target_keyed_slices/nb.py | 1 + .../nulls/target_keyed_slices/out.test.toml | 2 + .../nulls/target_keyed_slices/output.txt | 73 ++++++++++++++++++ .../validate/nulls/target_keyed_slices/script | 9 +++ .../target_removes_job_fields/databricks.yml | 40 ++++++++++ .../target_removes_job_fields/out.test.toml | 2 + .../target_removes_job_fields/output.txt | 51 ++++++++++++ .../nulls/target_removes_job_fields/script | 2 + .../target_removes_resource/databricks.yml | 22 ++++++ .../target_removes_resource/out.test.toml | 2 + .../nulls/target_removes_resource/output.txt | 29 +++++++ .../nulls/target_removes_resource/script | 3 + .../databricks.yml | 64 +++++++++++++++ .../out.test.toml | 2 + .../output.txt | 77 +++++++++++++++++++ .../target_removes_top_level_fields/script | 11 +++ 33 files changed, 721 insertions(+) create mode 100644 acceptance/bundle/validate/nulls/include_removes/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/include_removes/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/include_removes/output.txt create mode 100644 acceptance/bundle/validate/nulls/include_removes/override.yml create mode 100644 acceptance/bundle/validate/nulls/include_removes/script create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/a.yml create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/b.yml create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/nb.py create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/output.txt create mode 100644 acceptance/bundle/validate/nulls/include_target_overrides/script create mode 100644 acceptance/bundle/validate/nulls/root_without_base/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/root_without_base/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/root_without_base/output.txt create mode 100644 acceptance/bundle/validate/nulls/root_without_base/script create mode 100644 acceptance/bundle/validate/nulls/target_keyed_slices/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/target_keyed_slices/nb.py create mode 100644 acceptance/bundle/validate/nulls/target_keyed_slices/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/target_keyed_slices/output.txt create mode 100644 acceptance/bundle/validate/nulls/target_keyed_slices/script create mode 100644 acceptance/bundle/validate/nulls/target_removes_job_fields/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/target_removes_job_fields/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/target_removes_job_fields/output.txt create mode 100644 acceptance/bundle/validate/nulls/target_removes_job_fields/script create mode 100644 acceptance/bundle/validate/nulls/target_removes_resource/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/target_removes_resource/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/target_removes_resource/output.txt create mode 100644 acceptance/bundle/validate/nulls/target_removes_resource/script create mode 100644 acceptance/bundle/validate/nulls/target_removes_top_level_fields/databricks.yml create mode 100644 acceptance/bundle/validate/nulls/target_removes_top_level_fields/out.test.toml create mode 100644 acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt create mode 100644 acceptance/bundle/validate/nulls/target_removes_top_level_fields/script diff --git a/acceptance/bundle/validate/nulls/include_removes/databricks.yml b/acceptance/bundle/validate/nulls/include_removes/databricks.yml new file mode 100644 index 00000000000..0895b15b833 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_removes/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: test-bundle + cluster_id: base-cluster + +include: + - override.yml + +presets: + name_prefix: base- + tags: + removed: "1" + kept: "1" + +sync: + include: + - databricks.yml + +permissions: + - level: CAN_VIEW + group_name: users + +variables: + with_default: + default: base + +run_as: + user_name: base@example.com + +resources: + jobs: + foo: + name: foo + bar: + name: bar diff --git a/acceptance/bundle/validate/nulls/include_removes/out.test.toml b/acceptance/bundle/validate/nulls/include_removes/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_removes/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/include_removes/output.txt b/acceptance/bundle/validate/nulls/include_removes/output.txt new file mode 100644 index 00000000000..f55a8cad6a6 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_removes/output.txt @@ -0,0 +1,58 @@ + +>>> [CLI] bundle validate -o json +Warning: expected a string value, found null + at bundle.cluster_id + in override.yml:2:15 + +Warning: expected a string value, found null + at presets.name_prefix + in override.yml:5:16 + +Warning: expected a string value, found null + at presets.tags.removed + in override.yml:7:14 + +Warning: expected a string value, found null + at presets.tags.not_in_base + in override.yml:8:18 + +Error: multiple resources or scripts have been defined with the same key: bar + at resources.jobs.bar + in databricks.yml:34:7 + override.yml:23:10 + + +Exit code: 1 +{ + "cluster_id": "base-cluster", + "presets": { + "name_prefix": "base-", + "tags": { + "kept": "1", + "removed": "1" + } + }, + "sync": { + "include": [ + "databricks.yml" + ] + }, + "permissions": [ + { + "group_name": "users", + "level": "CAN_VIEW" + } + ], + "variables": { + "with_default": { + "default": "base" + } + }, + "run_as": { + "user_name": "base@example.com" + }, + "jobs": [ + "bar", + "foo" + ] +} diff --git a/acceptance/bundle/validate/nulls/include_removes/override.yml b/acceptance/bundle/validate/nulls/include_removes/override.yml new file mode 100644 index 00000000000..309c6c0912a --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_removes/override.yml @@ -0,0 +1,23 @@ +bundle: + cluster_id: null + +presets: + name_prefix: null + tags: + removed: null + not_in_base: null + +sync: + include: null + +permissions: null + +variables: + with_default: + default: null + +run_as: null + +resources: + jobs: + bar: null diff --git a/acceptance/bundle/validate/nulls/include_removes/script b/acceptance/bundle/validate/nulls/include_removes/script new file mode 100644 index 00000000000..d1866c9c056 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_removes/script @@ -0,0 +1,10 @@ +# Nulls in an included file are meant to remove the values set in databricks.yml, including a whole resource. +errcode trace $CLI bundle validate -o json | jq '{ + cluster_id: .bundle.cluster_id, + presets: (.presets | {name_prefix, tags}), + sync, + permissions, + variables, + run_as, + jobs: (.resources.jobs | keys) +}' diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/a.yml b/acceptance/bundle/validate/nulls/include_target_overrides/a.yml new file mode 100644 index 00000000000..6acc1034a74 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/a.yml @@ -0,0 +1,13 @@ +# Target overrides from an included file, before b.yml. +targets: + dev: + resources: + jobs: + foo: + description: null + max_concurrent_runs: 10 + tags: + removed: null + tasks: + - task_key: t1 + max_retries: null diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/b.yml b/acceptance/bundle/validate/nulls/include_target_overrides/b.yml new file mode 100644 index 00000000000..6cebb494ed8 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/b.yml @@ -0,0 +1,10 @@ +# Target overrides from an included file, after a.yml. +targets: + dev: + resources: + jobs: + foo: + description: description from b.yml + max_concurrent_runs: null + timeout_seconds: null + bar: null diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/databricks.yml b/acceptance/bundle/validate/nulls/include_target_overrides/databricks.yml new file mode 100644 index 00000000000..63bb24ce799 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: test-bundle + +include: + - a.yml + - b.yml + +resources: + jobs: + foo: + name: foo + description: base description + max_concurrent_runs: 5 + timeout_seconds: 60 + tags: + removed: "1" + kept: "1" + tasks: + - task_key: t1 + max_retries: 3 + notebook_task: + notebook_path: ./nb.py + bar: + name: bar + +targets: + dev: + resources: + jobs: + foo: + timeout_seconds: 120 diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/nb.py b/acceptance/bundle/validate/nulls/include_target_overrides/nb.py new file mode 100644 index 00000000000..1645e04b1de --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/nb.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/out.test.toml b/acceptance/bundle/validate/nulls/include_target_overrides/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/output.txt b/acceptance/bundle/validate/nulls/include_target_overrides/output.txt new file mode 100644 index 00000000000..77ea92b92a9 --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/output.txt @@ -0,0 +1,43 @@ + +>>> [CLI] bundle validate -o json -t dev +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.description + in a.yml:7:24 + +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.tags.removed + in a.yml:10:22 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.tasks[0].max_retries + in a.yml:13:28 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.max_concurrent_runs + in b.yml:8:32 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.timeout_seconds + in b.yml:9:28 + +{ + "foo": { + "description": "description from b.yml", + "max_concurrent_runs": 10, + "timeout_seconds": 120, + "tags": { + "kept": "1", + "removed": "1" + }, + "tasks": [ + { + "task_key": "t1", + "max_retries": 3 + } + ] + }, + "jobs": [ + "bar", + "foo" + ] +} diff --git a/acceptance/bundle/validate/nulls/include_target_overrides/script b/acceptance/bundle/validate/nulls/include_target_overrides/script new file mode 100644 index 00000000000..40ecadf811b --- /dev/null +++ b/acceptance/bundle/validate/nulls/include_target_overrides/script @@ -0,0 +1,3 @@ +# Target overrides spread across included files: a null is meant to apply to the base when the target is selected, +# and a later file is meant to win over an earlier one, whether the later value is null or not. +trace $CLI bundle validate -o json -t dev | jq '.resources.jobs | {foo: (.foo | {description, max_concurrent_runs, timeout_seconds, tags, tasks: (.tasks | map({task_key, max_retries}))}), jobs: keys}' diff --git a/acceptance/bundle/validate/nulls/root_without_base/databricks.yml b/acceptance/bundle/validate/nulls/root_without_base/databricks.yml new file mode 100644 index 00000000000..d4d44ca566e --- /dev/null +++ b/acceptance/bundle/validate/nulls/root_without_base/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: foo + description: null + tags: + kept: "1" + not_in_base: null + email_notifications: + on_failure: null diff --git a/acceptance/bundle/validate/nulls/root_without_base/out.test.toml b/acceptance/bundle/validate/nulls/root_without_base/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/root_without_base/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/root_without_base/output.txt b/acceptance/bundle/validate/nulls/root_without_base/output.txt new file mode 100644 index 00000000000..81520fc5908 --- /dev/null +++ b/acceptance/bundle/validate/nulls/root_without_base/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle validate -o json +Warning: expected a string value, found null + at resources.jobs.foo.description + in databricks.yml:8:20 + +Warning: expected a string value, found null + at resources.jobs.foo.tags.not_in_base + in databricks.yml:11:22 + +{ + "email_notifications": { + "on_failure": null + }, + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "tags": { + "kept": "1" + } +} diff --git a/acceptance/bundle/validate/nulls/root_without_base/script b/acceptance/bundle/validate/nulls/root_without_base/script new file mode 100644 index 00000000000..ab3cd87abd0 --- /dev/null +++ b/acceptance/bundle/validate/nulls/root_without_base/script @@ -0,0 +1,2 @@ +# A null in databricks.yml outside a target has nothing to remove and is meant to add nothing. +trace $CLI bundle validate -o json | jq '.resources.jobs.foo | del(.deployment, .edit_mode, .format)' diff --git a/acceptance/bundle/validate/nulls/target_keyed_slices/databricks.yml b/acceptance/bundle/validate/nulls/target_keyed_slices/databricks.yml new file mode 100644 index 00000000000..161967abe5f --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_keyed_slices/databricks.yml @@ -0,0 +1,61 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: foo + job_clusters: + - job_cluster_key: c1 + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 2 + parameters: + - name: p1 + default: base + tasks: + - task_key: t1 + job_cluster_key: c1 + max_retries: 3 + timeout_seconds: 60 + notebook_task: + notebook_path: ./nb.py + - task_key: t2 + job_cluster_key: c1 + max_retries: 1 + notebook_task: + notebook_path: ./nb.py + pipelines: + bar: + name: bar + clusters: + - label: default + num_workers: 2 + node_type_id: i3.xlarge + +targets: + dev: + resources: + jobs: + foo: + job_clusters: + - job_cluster_key: c1 + new_cluster: + num_workers: null + parameters: + - name: p1 + default: null + tasks: + - task_key: t1 + max_retries: null + - task_key: t3 + job_cluster_key: c1 + max_retries: null + notebook_task: + notebook_path: ./nb.py + pipelines: + bar: + clusters: + - label: default + num_workers: null diff --git a/acceptance/bundle/validate/nulls/target_keyed_slices/nb.py b/acceptance/bundle/validate/nulls/target_keyed_slices/nb.py new file mode 100644 index 00000000000..1645e04b1de --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_keyed_slices/nb.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/validate/nulls/target_keyed_slices/out.test.toml b/acceptance/bundle/validate/nulls/target_keyed_slices/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_keyed_slices/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_keyed_slices/output.txt b/acceptance/bundle/validate/nulls/target_keyed_slices/output.txt new file mode 100644 index 00000000000..7b9a2d8e2d6 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_keyed_slices/output.txt @@ -0,0 +1,73 @@ + +>>> [CLI] bundle validate -o json -t dev +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.job_clusters[0].new_cluster.num_workers + in databricks.yml:45:30 + +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.parameters[0].default + in databricks.yml:48:24 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.tasks[0].max_retries + in databricks.yml:51:28 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.tasks[1].max_retries + in databricks.yml:54:28 + +Warning: expected a int value, found null + at targets.dev.resources.pipelines.bar.clusters[0].num_workers + in databricks.yml:61:28 + +{ + "job_clusters": [ + { + "job_cluster_key": "c1", + "new_cluster": { + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2, + "spark_version": "15.4.x-scala2.12" + } + } + ], + "parameters": [ + { + "default": "base", + "name": "p1" + } + ], + "tasks": [ + { + "job_cluster_key": "c1", + "max_retries": 3, + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb" + }, + "task_key": "t1", + "timeout_seconds": 60 + }, + { + "job_cluster_key": "c1", + "max_retries": 1, + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb" + }, + "task_key": "t2" + }, + { + "job_cluster_key": "c1", + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb" + }, + "task_key": "t3" + } + ], + "pipeline_clusters": [ + { + "label": "default", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2 + } + ] +} diff --git a/acceptance/bundle/validate/nulls/target_keyed_slices/script b/acceptance/bundle/validate/nulls/target_keyed_slices/script new file mode 100644 index 00000000000..bc68b6b5c47 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_keyed_slices/script @@ -0,0 +1,9 @@ +# Lists whose elements are merged by key (tasks by task_key, job_clusters by job_cluster_key, job parameters by name, +# pipeline clusters by label): a null field in a target element is meant to remove that field +# from the base element with the same key. An element whose key is not in the base (t3) should not get the null field. +trace $CLI bundle validate -o json -t dev | jq '.resources | { + job_clusters: .jobs.foo.job_clusters, + parameters: .jobs.foo.parameters, + tasks: .jobs.foo.tasks, + pipeline_clusters: .pipelines.bar.clusters +}' diff --git a/acceptance/bundle/validate/nulls/target_removes_job_fields/databricks.yml b/acceptance/bundle/validate/nulls/target_removes_job_fields/databricks.yml new file mode 100644 index 00000000000..9e37b64d3fa --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_job_fields/databricks.yml @@ -0,0 +1,40 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: foo + description: base description + max_concurrent_runs: 5 + queue: + enabled: false + schedule: + quartz_cron_expression: 0 0 * * * ? + timezone_id: UTC + pause_status: PAUSED + tags: + removed: "1" + kept: "1" + email_notifications: + on_failure: + - base@example.com + on_success: + - base@example.com + +targets: + dev: + resources: + jobs: + foo: + description: null + max_concurrent_runs: null + queue: null + schedule: + pause_status: null + tags: + removed: null + not_in_base: null + email_notifications: + on_failure: null + timeout_seconds: null diff --git a/acceptance/bundle/validate/nulls/target_removes_job_fields/out.test.toml b/acceptance/bundle/validate/nulls/target_removes_job_fields/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_job_fields/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_removes_job_fields/output.txt b/acceptance/bundle/validate/nulls/target_removes_job_fields/output.txt new file mode 100644 index 00000000000..551c3980f68 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_job_fields/output.txt @@ -0,0 +1,51 @@ + +>>> [CLI] bundle validate -o json -t dev +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.description + in databricks.yml:30:24 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.max_concurrent_runs + in databricks.yml:31:32 + +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.schedule.pause_status + in databricks.yml:34:27 + +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.tags.removed + in databricks.yml:36:22 + +Warning: expected a string value, found null + at targets.dev.resources.jobs.foo.tags.not_in_base + in databricks.yml:37:26 + +Warning: expected a int value, found null + at targets.dev.resources.jobs.foo.timeout_seconds + in databricks.yml:40:28 + +{ + "description": "base description", + "email_notifications": { + "on_failure": [ + "base@example.com" + ], + "on_success": [ + "base@example.com" + ] + }, + "max_concurrent_runs": 5, + "name": "foo", + "queue": { + "enabled": false + }, + "schedule": { + "pause_status": "PAUSED", + "quartz_cron_expression": "0 0 * * * ?", + "timezone_id": "UTC" + }, + "tags": { + "kept": "1", + "removed": "1" + } +} diff --git a/acceptance/bundle/validate/nulls/target_removes_job_fields/script b/acceptance/bundle/validate/nulls/target_removes_job_fields/script new file mode 100644 index 00000000000..705ec5f5769 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_job_fields/script @@ -0,0 +1,2 @@ +# Null fields of a resource in a target are meant to remove the fields: scalars, blocks, nested fields, map entries and lists. A null for a field missing from the base should add nothing. +trace $CLI bundle validate -o json -t dev | jq '.resources.jobs.foo | del(.deployment, .edit_mode, .format)' diff --git a/acceptance/bundle/validate/nulls/target_removes_resource/databricks.yml b/acceptance/bundle/validate/nulls/target_removes_resource/databricks.yml new file mode 100644 index 00000000000..582631f411d --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_resource/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: foo + bar: + name: bar + pipelines: + baz: + name: baz + +targets: + dev: + resources: + jobs: + bar: null + not_in_base: null + pipelines: + baz: + prod: {} diff --git a/acceptance/bundle/validate/nulls/target_removes_resource/out.test.toml b/acceptance/bundle/validate/nulls/target_removes_resource/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_resource/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_removes_resource/output.txt b/acceptance/bundle/validate/nulls/target_removes_resource/output.txt new file mode 100644 index 00000000000..e4c5e98b82b --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_resource/output.txt @@ -0,0 +1,29 @@ + +>>> [CLI] bundle validate -o json -t dev +Error: job not_in_base is not defined + at resources.jobs.not_in_base + in databricks.yml:19:22 + + +Exit code: 1 +{ + "jobs": [ + "bar", + "foo", + "not_in_base" + ], + "pipelines": [ + "baz" + ] +} + +>>> [CLI] bundle validate -o json -t prod +{ + "jobs": [ + "bar", + "foo" + ], + "pipelines": [ + "baz" + ] +} diff --git a/acceptance/bundle/validate/nulls/target_removes_resource/script b/acceptance/bundle/validate/nulls/target_removes_resource/script new file mode 100644 index 00000000000..48d5e4638bc --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_resource/script @@ -0,0 +1,3 @@ +# A null resource in a target is meant to remove the resource from that target only; a null resource missing from the base should add nothing. +errcode trace $CLI bundle validate -o json -t dev | jq '.resources | map_values(keys)' +trace $CLI bundle validate -o json -t prod | jq '.resources | map_values(keys)' diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/databricks.yml b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/databricks.yml new file mode 100644 index 00000000000..56d8e9b7f1c --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/databricks.yml @@ -0,0 +1,64 @@ +bundle: + name: test-bundle + mode: development + cluster_id: base-cluster + git: + branch: base-branch + +workspace: + root_path: ~/base + +presets: + name_prefix: base-${workspace.current_user.short_name}- + tags: + removed: "1" + kept: "1" + +sync: + include: + - databricks.yml + +permissions: + - level: CAN_MANAGE + user_name: ${workspace.current_user.userName} + +variables: + with_default: + default: base + with_lookup: + lookup: + cluster: some-test-cluster + +run_as: + user_name: base@example.com + +artifacts: + removed: + type: whl + build: build-removed + kept: + type: whl + build: build-kept + +targets: + dev: + mode: null + cluster_id: null + git: null + run_as: null + workspace: + root_path: null + presets: + name_prefix: null + tags: + removed: null + sync: + include: null + permissions: null + variables: + with_default: + default: null + with_lookup: + lookup: null + artifacts: + removed: null diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/out.test.toml b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt new file mode 100644 index 00000000000..a41109f84fb --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt @@ -0,0 +1,77 @@ + +>>> [CLI] bundle validate -o json -t dev +Warning: expected a string value, found null + at targets.dev.mode + in databricks.yml:45:11 + +Warning: expected a string value, found null + at targets.dev.cluster_id + in databricks.yml:46:17 + +Warning: expected a string value, found null + at targets.dev.workspace.root_path + in databricks.yml:50:18 + +Warning: expected a string value, found null + at targets.dev.presets.name_prefix + in databricks.yml:52:20 + +Warning: expected a string value, found null + at targets.dev.presets.tags.removed + in databricks.yml:54:18 + +{ + "bundle": { + "mode": "development", + "cluster_id": "base-cluster", + "git": { + "branch": "base-branch" + } + }, + "root_path": "/Workspace/Users/[USERNAME]/base", + "presets": { + "name_prefix": "base-[USERNAME]-", + "tags": { + "dev": "[USERNAME]", + "kept": "1", + "removed": "1" + } + }, + "sync": { + "include": [ + "databricks.yml" + ], + "paths": [ + "." + ] + }, + "permissions": [ + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ], + "variables": { + "with_default": { + "default": "base", + "value": "base" + }, + "with_lookup": { + "lookup": { + "cluster": "some-test-cluster" + }, + "value": "4321" + } + }, + "run_as": { + "user_name": "base@example.com" + }, + "artifacts": { + "kept": { + "build": "build-kept" + }, + "removed": { + "build": "build-removed" + } + } +} diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script new file mode 100644 index 00000000000..1c1db46e9e2 --- /dev/null +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script @@ -0,0 +1,11 @@ +# Null top-level fields in a target are meant to remove them, including the fields a target replaces rather than merges (mode, cluster_id, run_as, variable default and lookup). +trace $CLI bundle validate -o json -t dev | jq '{ + bundle: (.bundle | {mode, cluster_id, git: (.git | {branch})}), + root_path: .workspace.root_path, + presets: (.presets | {name_prefix, tags}), + sync, + permissions, + variables, + run_as, + artifacts: (.artifacts | map_values({build})) +}' From 2fe8e299747b3e472f1bd60780b2dc6c266b981d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 30 Oct 2026 15:13:56 +0100 Subject: [PATCH 2/2] acc: give the variables a value in nulls/target_removes_top_level_fields Co-authored-by: Isaac --- .../nulls/target_removes_top_level_fields/output.txt | 6 +++--- .../validate/nulls/target_removes_top_level_fields/script | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt index a41109f84fb..0397ae3730c 100644 --- a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle validate -o json -t dev +>>> [CLI] bundle validate -o json -t dev --var with_default=from-cli --var with_lookup=from-cli Warning: expected a string value, found null at targets.dev.mode in databricks.yml:45:11 @@ -54,13 +54,13 @@ Warning: expected a string value, found null "variables": { "with_default": { "default": "base", - "value": "base" + "value": "from-cli" }, "with_lookup": { "lookup": { "cluster": "some-test-cluster" }, - "value": "4321" + "value": "from-cli" } }, "run_as": { diff --git a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script index 1c1db46e9e2..5419081c5e2 100644 --- a/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script +++ b/acceptance/bundle/validate/nulls/target_removes_top_level_fields/script @@ -1,5 +1,6 @@ # Null top-level fields in a target are meant to remove them, including the fields a target replaces rather than merges (mode, cluster_id, run_as, variable default and lookup). -trace $CLI bundle validate -o json -t dev | jq '{ +# The variables get a value from --var, so the ones without a default or lookup do not fail validation. +trace $CLI bundle validate -o json -t dev --var with_default=from-cli --var with_lookup=from-cli | jq '{ bundle: (.bundle | {mode, cluster_id, git: (.git | {branch})}), root_path: .workspace.root_path, presets: (.presets | {name_prefix, tags}),