Make a null in an include or a target remove the value it overrides

.nextchanges/bundles/7261.md

@@ -0,0 +1 @@
+* A null in an included file or in a target now removes the value it overrides, e.g. `targets.dev.resources.jobs.foo: null` leaves job `foo` out of target `dev`; a null that has nothing to remove is ignored without a warning. ([#7261](https://github.com/databricks/cli/pull/7261))

acceptance/bundle/deploy/mlops-stacks/output.txt

@@ -25,10 +25,6 @@ This directory contains an ML project based on the default
 [Databricks MLOps Stacks](https://github.com/databricks/mlops-stacks),
 
 >>> [CLI] bundle summary --var catalog_name=main
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -56,10 +52,6 @@ Resources:
       URL:  (not deployed)
 
 >>> [CLI] bundle validate --var catalog_name=main
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -70,13 +62,9 @@ Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev
 
-Found 2 warnings
+Found 1 warning
 
 >>> [CLI] bundle deploy --var catalog_name=main
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -94,10 +82,6 @@ Files: 26 uploaded, 0 deleted
 Resources: 8 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> [CLI] bundle summary --var catalog_name=main -o json
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -110,18 +94,10 @@ Warning: unknown field: description
 }
 
 === Assert the batch inference job actually exists
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
 
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -131,10 +107,6 @@ Warning: unknown field: description
 }
 
 >>> [CLI] bundle destroy --var catalog_name=main --auto-approve
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7
@@ -150,10 +122,6 @@ All files and directories at the following location will be deleted: /Workspace/
 Destroy: 4 deleted
 
 >>> [CLI] schemas delete main.project_name_[UNIQUE_NAME]
-Warning: expected a string value, found null
-  at targets.dev.workspace.host
-  in databricks.yml:37:12
-
 Warning: unknown field: description
   at resources.experiments.experiment
   in resources/ml-artifacts-resource.yml:28:7

acceptance/bundle/run_as/empty_override/output.txt

@@ -1,2 +1,6 @@
-        "run_as": {
-  "run_as": {
+
+>>> [CLI] bundle validate -o json
+{
+  "run_as": null,
+  "job_run_as": null
+}

acceptance/bundle/run_as/empty_override/script

@@ -1 +1,2 @@
-$CLI bundle validate -o json | grep run_as
+# An empty run_as in a target override removes run_as.
+trace $CLI bundle validate -o json | jq '{run_as, job_run_as: .resources.jobs.foo.run_as}'

acceptance/bundle/validate/empty_tasks/output.txt

@@ -12,8 +12,7 @@
     "name": "job with empty tasks",
     "queue": {
       "enabled": true
-    },
-    "tasks": null
+    }
   },
   "valid": {
     "deployment": {

acceptance/bundle/validate/nulls/include_removes/output.txt

@@ -1,58 +1,28 @@
 
 >>> [CLI] bundle validate -o json
-Warning: expected a string value, found null
-  at bundle.cluster_id
-  in override.yml:2:15
-
-Warning: expected a string value, found null
-  at presets.name_prefix
-  in override.yml:5:16
-
-Warning: expected a string value, found null
-  at presets.tags.removed
-  in override.yml:7:14
-
-Warning: expected a string value, found null
-  at presets.tags.not_in_base
-  in override.yml:8:18
-
-Error: multiple resources or scripts have been defined with the same key: bar
-  at resources.jobs.bar
-  in databricks.yml:34:7
-     override.yml:23:10
+Error: no value assigned to required variable with_default. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the BUNDLE_VAR_with_default environment variable, or .databricks/bundle/<target>/variable-overrides.json
 
 
 Exit code: 1
 {
-  "cluster_id": "base-cluster",
+  "cluster_id": null,
   "presets": {
-    "name_prefix": "base-",
+    "name_prefix": null,
     "tags": {
-      "kept": "1",
-      "removed": "1"
+      "kept": "1"
     }
   },
   "sync": {
-    "include": [
-      "databricks.yml"
+    "paths": [
+      "."
     ]
   },
-  "permissions": [
-    {
-      "group_name": "users",
-      "level": "CAN_VIEW"
-    }
-  ],
+  "permissions": null,
   "variables": {
-    "with_default": {
-      "default": "base"
-    }
-  },
-  "run_as": {
-    "user_name": "base@example.com"
+    "with_default": {}
   },
+  "run_as": null,
   "jobs": [
-    "bar",
     "foo"
   ]
 }

acceptance/bundle/validate/nulls/include_target_overrides/output.txt

@@ -1,43 +1,21 @@
 
 >>> [CLI] bundle validate -o json -t dev
-Warning: expected a string value, found null
-  at targets.dev.resources.jobs.foo.description
-  in a.yml:7:24
-
-Warning: expected a string value, found null
-  at targets.dev.resources.jobs.foo.tags.removed
-  in a.yml:10:22
-
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.tasks[0].max_retries
-  in a.yml:13:28
-
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.max_concurrent_runs
-  in b.yml:8:32
-
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.timeout_seconds
-  in b.yml:9:28
-
 {
   "foo": {
     "description": "description from b.yml",
-    "max_concurrent_runs": 10,
-    "timeout_seconds": 120,
+    "max_concurrent_runs": 1,
+    "timeout_seconds": null,
     "tags": {
-      "kept": "1",
-      "removed": "1"
+      "kept": "1"
     },
     "tasks": [
       {
         "task_key": "t1",
-        "max_retries": 3
+        "max_retries": null
       }
     ]
   },
   "jobs": [
-    "bar",
     "foo"
   ]
 }

acceptance/bundle/validate/nulls/resources_container/output.txt

@@ -1,9 +1,9 @@
 
 >>> [CLI] bundle validate
-Error: expected a map at "resources", found nil
-
 Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
-Found 1 error
-
-Exit code: 1
+Validation OK!

acceptance/bundle/validate/nulls/resources_container/script

@@ -1,2 +1,2 @@
-# resources: null is rejected.
+# A null resources section in databricks.yml has nothing to remove and is accepted.
 errcode trace $CLI bundle validate

acceptance/bundle/validate/nulls/root_without_base/output.txt

@@ -1,17 +1,7 @@
 
 >>> [CLI] bundle validate -o json
-Warning: expected a string value, found null
-  at resources.jobs.foo.description
-  in databricks.yml:8:20
-
-Warning: expected a string value, found null
-  at resources.jobs.foo.tags.not_in_base
-  in databricks.yml:11:22
-
 {
-  "email_notifications": {
-    "on_failure": null
-  },
+  "email_notifications": {},
   "max_concurrent_runs": 1,
   "name": "foo",
   "queue": {

acceptance/bundle/validate/nulls/schedule_default_materialization/output.txt

@@ -1,13 +1,5 @@
 
 >>> [CLI] bundle validate -o json
-Warning: required field "quartz_cron_expression" is not set
-  at resources.jobs.j.schedule
-  in databricks.yml:8:17
-
-Warning: required field "timezone_id" is not set
-  at resources.jobs.j.schedule
-  in databricks.yml:8:17
-
 {
   "schedule": null
 }

acceptance/bundle/validate/nulls/schedule_default_materialization/script

@@ -1,2 +1,2 @@
-# Defaults are not applied under an explicitly null block (schedule stays null).
+# A null schedule in databricks.yml has nothing to remove: the job has no schedule.
 trace $CLI bundle validate -o json | jq '.resources.jobs.j | {schedule}'

acceptance/bundle/validate/nulls/target_keyed_slices/output.txt

@@ -1,46 +1,29 @@
 
 >>> [CLI] bundle validate -o json -t dev
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.job_clusters[0].new_cluster.num_workers
-  in databricks.yml:45:30
-
-Warning: expected a string value, found null
-  at targets.dev.resources.jobs.foo.parameters[0].default
-  in databricks.yml:48:24
-
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.tasks[0].max_retries
-  in databricks.yml:51:28
-
-Warning: expected a int value, found null
-  at targets.dev.resources.jobs.foo.tasks[1].max_retries
-  in databricks.yml:54:28
-
-Warning: expected a int value, found null
-  at targets.dev.resources.pipelines.bar.clusters[0].num_workers
-  in databricks.yml:61:28
+Warning: required field "default" is not set
+  at resources.jobs.foo.parameters[0]
+  in databricks.yml:15:11
+     databricks.yml:47:15
 
 {
   "job_clusters": [
     {
       "job_cluster_key": "c1",
       "new_cluster": {
         "node_type_id": "[NODE_TYPE_ID]",
-        "num_workers": 2,
+        "num_workers": 0,
         "spark_version": "15.4.x-scala2.12"
       }
     }
   ],
   "parameters": [
     {
-      "default": "base",
       "name": "p1"
     }
   ],
   "tasks": [
     {
       "job_cluster_key": "c1",
-      "max_retries": 3,
       "notebook_task": {
         "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files/nb"
       },
@@ -66,8 +49,7 @@ Warning: expected a int value, found null
   "pipeline_clusters": [
     {
       "label": "default",
-      "node_type_id": "[NODE_TYPE_ID]",
-      "num_workers": 2
+      "node_type_id": "[NODE_TYPE_ID]"
     }
   ]
 }