acc: record how nulls in targets and includes are ignored today

acceptance/bundle/validate/nulls/include_removes/databricks.yml

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle
+  cluster_id: base-cluster
+
+include:
+  - override.yml
+
+presets:
+  name_prefix: base-
+  tags:
+    removed: "1"
+    kept: "1"
+
+sync:
+  include:
+    - databricks.yml
+
+permissions:
+  - level: CAN_VIEW
+    group_name: users
+
+variables:
+  with_default:
+    default: base
+
+run_as:
+  user_name: base@example.com
+
+resources:
+  jobs:
+    foo:
+      name: foo
+    bar:
+      name: bar

acceptance/bundle/validate/nulls/include_removes/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/nulls/include_removes/output.txt

@@ -0,0 +1,58 @@
+
+>>> [CLI] bundle validate -o json
+Warning: expected a string value, found null
+  at bundle.cluster_id
+  in override.yml:2:15
+
+Warning: expected a string value, found null
+  at presets.name_prefix
+  in override.yml:5:16
+
+Warning: expected a string value, found null
+  at presets.tags.removed
+  in override.yml:7:14
+
+Warning: expected a string value, found null
+  at presets.tags.not_in_base
+  in override.yml:8:18
+
+Error: multiple resources or scripts have been defined with the same key: bar
+  at resources.jobs.bar
+  in databricks.yml:34:7
+     override.yml:23:10
+
+
+Exit code: 1
+{
+  "cluster_id": "base-cluster",
+  "presets": {
+    "name_prefix": "base-",
+    "tags": {
+      "kept": "1",
+      "removed": "1"
+    }
+  },
+  "sync": {
+    "include": [
+      "databricks.yml"
+    ]
+  },
+  "permissions": [
+    {
+      "group_name": "users",
+      "level": "CAN_VIEW"
+    }
+  ],
+  "variables": {
+    "with_default": {
+      "default": "base"
+    }
+  },
+  "run_as": {
+    "user_name": "base@example.com"
+  },
+  "jobs": [
+    "bar",
+    "foo"
+  ]
+}

acceptance/bundle/validate/nulls/include_removes/override.yml

@@ -0,0 +1,23 @@
+bundle:
+  cluster_id: null
+
+presets:
+  name_prefix: null
+  tags:
+    removed: null
+    not_in_base: null
+
+sync:
+  include: null
+
+permissions: null
+
+variables:
+  with_default:
+    default: null
+
+run_as: null
+
+resources:
+  jobs:
+    bar: null

acceptance/bundle/validate/nulls/include_removes/script

@@ -0,0 +1,10 @@
+# Nulls in an included file are meant to remove the values set in databricks.yml, including a whole resource.
+errcode trace $CLI bundle validate -o json | jq '{
+  cluster_id: .bundle.cluster_id,
+  presets: (.presets | {name_prefix, tags}),
+  sync,
+  permissions,
+  variables,
+  run_as,
+  jobs: (.resources.jobs | keys)
+}'

acceptance/bundle/validate/nulls/include_target_overrides/a.yml

@@ -0,0 +1,13 @@
+# Target overrides from an included file, before b.yml.
+targets:
+  dev:
+    resources:
+      jobs:
+        foo:
+          description: null
+          max_concurrent_runs: 10
+          tags:
+            removed: null
+          tasks:
+            - task_key: t1
+              max_retries: null

acceptance/bundle/validate/nulls/include_target_overrides/b.yml

@@ -0,0 +1,10 @@
+# Target overrides from an included file, after a.yml.
+targets:
+  dev:
+    resources:
+      jobs:
+        foo:
+          description: description from b.yml
+          max_concurrent_runs: null
+          timeout_seconds: null
+        bar: null

acceptance/bundle/validate/nulls/include_target_overrides/databricks.yml

@@ -0,0 +1,31 @@
+bundle:
+  name: test-bundle
+
+include:
+  - a.yml
+  - b.yml
+
+resources:
+  jobs:
+    foo:
+      name: foo
+      description: base description
+      max_concurrent_runs: 5
+      timeout_seconds: 60
+      tags:
+        removed: "1"
+        kept: "1"
+      tasks:
+        - task_key: t1
+          max_retries: 3
+          notebook_task:
+            notebook_path: ./nb.py
+    bar:
+      name: bar
+
+targets:
+  dev:
+    resources:
+      jobs:
+        foo:
+          timeout_seconds: 120

acceptance/bundle/validate/nulls/include_target_overrides/nb.py

@@ -0,0 +1 @@
+# Databricks notebook source

acceptance/bundle/validate/nulls/include_target_overrides/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/nulls/include_target_overrides/output.txt

@@ -0,0 +1,43 @@
+
+>>> [CLI] bundle validate -o json -t dev
+Warning: expected a string value, found null
+  at targets.dev.resources.jobs.foo.description
+  in a.yml:7:24
+
+Warning: expected a string value, found null
+  at targets.dev.resources.jobs.foo.tags.removed
+  in a.yml:10:22
+
+Warning: expected a int value, found null
+  at targets.dev.resources.jobs.foo.tasks[0].max_retries
+  in a.yml:13:28
+
+Warning: expected a int value, found null
+  at targets.dev.resources.jobs.foo.max_concurrent_runs
+  in b.yml:8:32
+
+Warning: expected a int value, found null
+  at targets.dev.resources.jobs.foo.timeout_seconds
+  in b.yml:9:28
+
+{
+  "foo": {
+    "description": "description from b.yml",
+    "max_concurrent_runs": 10,
+    "timeout_seconds": 120,
+    "tags": {
+      "kept": "1",
+      "removed": "1"
+    },
+    "tasks": [
+      {
+        "task_key": "t1",
+        "max_retries": 3
+      }
+    ]
+  },
+  "jobs": [
+    "bar",
+    "foo"
+  ]
+}

acceptance/bundle/validate/nulls/include_target_overrides/script

@@ -0,0 +1,3 @@
+# Target overrides spread across included files: a null is meant to apply to the base when the target is selected,
+# and a later file is meant to win over an earlier one, whether the later value is null or not.
+trace $CLI bundle validate -o json -t dev | jq '.resources.jobs | {foo: (.foo | {description, max_concurrent_runs, timeout_seconds, tags, tasks: (.tasks | map({task_key, max_retries}))}), jobs: keys}'

acceptance/bundle/validate/nulls/root_without_base/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    foo:
+      name: foo
+      description: null
+      tags:
+        kept: "1"
+        not_in_base: null
+      email_notifications:
+        on_failure: null

acceptance/bundle/validate/nulls/root_without_base/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/nulls/root_without_base/output.txt

@@ -0,0 +1,23 @@
+
+>>> [CLI] bundle validate -o json
+Warning: expected a string value, found null
+  at resources.jobs.foo.description
+  in databricks.yml:8:20
+
+Warning: expected a string value, found null
+  at resources.jobs.foo.tags.not_in_base
+  in databricks.yml:11:22
+
+{
+  "email_notifications": {
+    "on_failure": null
+  },
+  "max_concurrent_runs": 1,
+  "name": "foo",
+  "queue": {
+    "enabled": true
+  },
+  "tags": {
+    "kept": "1"
+  }
+}

acceptance/bundle/validate/nulls/root_without_base/script

@@ -0,0 +1,2 @@
+# A null in databricks.yml outside a target has nothing to remove and is meant to add nothing.
+trace $CLI bundle validate -o json | jq '.resources.jobs.foo | del(.deployment, .edit_mode, .format)'

acceptance/bundle/validate/nulls/target_keyed_slices/databricks.yml

@@ -0,0 +1,61 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    foo:
+      name: foo
+      job_clusters:
+        - job_cluster_key: c1
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 2
+      parameters:
+        - name: p1
+          default: base
+      tasks:
+        - task_key: t1
+          job_cluster_key: c1
+          max_retries: 3
+          timeout_seconds: 60
+          notebook_task:
+            notebook_path: ./nb.py
+        - task_key: t2
+          job_cluster_key: c1
+          max_retries: 1
+          notebook_task:
+            notebook_path: ./nb.py
+  pipelines:
+    bar:
+      name: bar
+      clusters:
+        - label: default
+          num_workers: 2
+          node_type_id: i3.xlarge
+
+targets:
+  dev:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            - job_cluster_key: c1
+              new_cluster:
+                num_workers: null
+          parameters:
+            - name: p1
+              default: null
+          tasks:
+            - task_key: t1
+              max_retries: null
+            - task_key: t3
+              job_cluster_key: c1
+              max_retries: null
+              notebook_task:
+                notebook_path: ./nb.py
+      pipelines:
+        bar:
+          clusters:
+            - label: default
+              num_workers: null

acceptance/bundle/validate/nulls/target_keyed_slices/nb.py

@@ -0,0 +1 @@
+# Databricks notebook source

acceptance/bundle/validate/nulls/target_keyed_slices/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]