Add a new boolean field 'sensitive' for variables

.nextchanges/bundles/sensitive-variables.md

@@ -0,0 +1 @@
+* Add `sensitive: true` for bundle variables. A variable used as the value of a secret is sensitive by default. `bundle validate -o json` redacts only sensitive variables instead of every string variable, and a sensitive variable can only be referenced from a sensitive resource field (such as a secret's `value`) or from another sensitive variable. ([#7288](https://github.com/databricks/cli/pull/7288))

.nextchanges/notable-changes/sensitive-variables.md

@@ -0,0 +1 @@
+* A variable used as the value of a secret is now sensitive: referencing it from any other field, such as a job parameter or a secret's `comment`, is an error because the value would be used in plain text. Mark other secret-holding variables with `sensitive: true` to get the same protection. ([#7288](https://github.com/databricks/cli/pull/7288))

acceptance/bundle/python/secrets-support/databricks.yml

@@ -6,6 +6,8 @@ sync: {paths: []} # don't need to copy files
 variables:
   secret_value:
     description: The value of the secret
+  python_secret_value:
+    description: The value of the secret added from Python
 
 python:
   resources:

acceptance/bundle/python/secrets-support/output.txt

@@ -1,5 +1,5 @@
 
->>> uv run [UV_ARGS] -q [CLI] bundle validate --var secret_value=my-secret-value --output json
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --var secret_value=my-secret-value --var python_secret_value=my-python-secret-value --output json
 {
   "experimental": {
     "python": {
@@ -28,5 +28,17 @@
         "value": "[redacted]"
       }
     }
+  },
+  "variables": {
+    "python_secret_value": {
+      "description": "The value of the secret added from Python",
+      "sensitive": true,
+      "value": "[redacted]"
+    },
+    "secret_value": {
+      "description": "The value of the secret",
+      "sensitive": true,
+      "value": "[redacted]"
+    }
   }
 }

acceptance/bundle/python/secrets-support/resources.py

@@ -10,7 +10,7 @@ def load_resources() -> Resources:
             "catalog_name": "main",
             "schema_name": "default",
             "name": "my_secret_2",
-            "value": "${var.secret_value}",
+            "value": "${var.python_secret_value}",
             "expire_time": "2030-01-01T00:00:00Z",
         },
     )

acceptance/bundle/python/secrets-support/script

@@ -1,6 +1,7 @@
 
 # secrets require value to be a variable reference, so supply it at validate time.
-trace uv run $UV_ARGS -q $CLI bundle validate --var secret_value=my-secret-value --output json | \
-  jq "pick(.experimental.python, .resources)"
+# The variable used only by the secret added from Python must be redacted too.
+trace uv run $UV_ARGS -q $CLI bundle validate --var secret_value=my-secret-value --var python_secret_value=my-python-secret-value --output json | \
+  contains.py '!my-secret-value' '!my-python-secret-value' | jq "pick(.experimental.python, .resources, .variables)"
 
 rm -fr .databricks __pycache__

acceptance/bundle/refschema/output/root.txt

@@ -148,6 +148,7 @@ environments.*.variables.*.lookup.pipeline	string	INPUT
 environments.*.variables.*.lookup.query	string	INPUT
 environments.*.variables.*.lookup.service_principal	string	INPUT
 environments.*.variables.*.lookup.warehouse	string	INPUT
+environments.*.variables.*.sensitive	bool	INPUT
 environments.*.variables.*.type	variable.VariableType	INPUT
 environments.*.variables.*.value	any	INPUT
 environments.*.workspace	*config.Workspace	INPUT
@@ -364,6 +365,7 @@ targets.*.variables.*.lookup.pipeline	string	INPUT
 targets.*.variables.*.lookup.query	string	INPUT
 targets.*.variables.*.lookup.service_principal	string	INPUT
 targets.*.variables.*.lookup.warehouse	string	INPUT
+targets.*.variables.*.sensitive	bool	INPUT
 targets.*.variables.*.type	variable.VariableType	INPUT
 targets.*.variables.*.value	any	INPUT
 targets.*.workspace	*config.Workspace	INPUT
@@ -447,6 +449,7 @@ variables.*.lookup.pipeline	string	INPUT
 variables.*.lookup.query	string	INPUT
 variables.*.lookup.service_principal	string	INPUT
 variables.*.lookup.warehouse	string	INPUT
+variables.*.sensitive	bool	INPUT
 variables.*.type	variable.VariableType	INPUT
 variables.*.value	any	INPUT
 workspace	config.Workspace	INPUT

acceptance/bundle/resources/secrets/basic/output.txt

@@ -16,6 +16,7 @@
 {
   "secret_value": {
     "description": "Test secret value",
+    "sensitive": true,
     "value": "[redacted]"
   }
 }

acceptance/bundle/variables/sensitive/invalid-resource-field/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: sensitive-variables
+
+variables:
+  api_credential:
+    description: Sensitive because it is used as the value of a secret
+
+resources:
+  secrets:
+    example_secret:
+      name: example_secret
+      catalog_name: main
+      schema_name: default
+      value: ${var.api_credential}
+      comment: "Credential: ${var.api_credential}"
+
+  jobs:
+    example_job:
+      name: example_job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/Shared/notebook
+            base_parameters:
+              token: ${var.api_credential}

acceptance/bundle/variables/sensitive/invalid-variable/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: sensitive-variables
+
+variables:
+  api_credential:
+    sensitive: true
+  api_header:
+    description: Not marked as sensitive, so it must not reference a sensitive variable
+    default: "Bearer ${var.api_credential}"

acceptance/bundle/variables/sensitive/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/variables/sensitive/output.txt

@@ -0,0 +1,97 @@
+
+=== A variable used as the value of a secret is sensitive and redacted in validate -o json
+
+>>> [CLI] bundle validate -o json
+{
+  "variables": {
+    "api_credential": {
+      "description": "Credential for the external API, passed via BUNDLE_VAR_api_credential. Sensitive because it is used as the value of a secret.",
+      "sensitive": true,
+      "value": "[redacted]"
+    },
+    "api_header": {
+      "default": "[redacted]",
+      "description": "A sensitive variable may reference another sensitive variable",
+      "sensitive": true,
+      "value": "[redacted]"
+    },
+    "target_catalog": {
+      "default": "example_catalog",
+      "value": "example_catalog"
+    },
+    "target_schema": {
+      "default": "example_schema",
+      "value": "example_schema"
+    }
+  },
+  "resources": {
+    "secrets": {
+      "example_secret": {
+        "catalog_name": "example_catalog",
+        "name": "example_secret",
+        "schema_name": "example_schema",
+        "value": "[redacted]"
+      }
+    }
+  }
+}
+
+=== Sensitive variable used in non-sensitive resource fields
+
+>>> [CLI] bundle validate
+Error: Sensitive variable "api_credential" is used in a non-sensitive field
+  at resources.jobs.example_job.tasks[0].notebook_task.base_parameters.token
+  in databricks.yml:25:22
+
+The variable "api_credential" is sensitive, so its value must not appear in plain text.
+A variable is sensitive when it is marked with "sensitive: true" or used as the value of a sensitive resource field (such as resources.secrets.*.value). It can only be used in such a field or in the definition of another sensitive variable.
+
+Error: Sensitive variable "api_credential" is used in a non-sensitive field
+  at resources.secrets.example_secret.comment
+  in databricks.yml:15:16
+
+The variable "api_credential" is sensitive, so its value must not appear in plain text.
+A variable is sensitive when it is marked with "sensitive: true" or used as the value of a sensitive resource field (such as resources.secrets.*.value). It can only be used in such a field or in the definition of another sensitive variable.
+
+Name: sensitive-variables
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default
+
+Found 2 errors
+
+=== Sensitive variable used in the default of a non-sensitive variable
+
+>>> [CLI] bundle validate
+Error: Sensitive variable "api_credential" is used in a non-sensitive field
+  at variables.api_header.default
+  in databricks.yml:9:14
+
+The variable "api_credential" is sensitive, so its value must not appear in plain text.
+A variable is sensitive when it is marked with "sensitive: true" or used as the value of a sensitive resource field (such as resources.secrets.*.value). It can only be used in such a field or in the definition of another sensitive variable.
+
+Name: sensitive-variables
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default
+
+Found 1 error
+
+=== Sensitive variable passed as the value of a non-sensitive variable
+
+>>> [CLI] bundle validate --var target_schema=${var.api_credential}
+Error: Sensitive variable "api_credential" is used in a non-sensitive field
+  at variables.target_schema.value
+
+The variable "api_credential" is sensitive, so its value must not appear in plain text.
+A variable is sensitive when it is marked with "sensitive: true" or used as the value of a sensitive resource field (such as resources.secrets.*.value). It can only be used in such a field or in the definition of another sensitive variable.
+
+Name: sensitive-variables
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default
+
+Found 1 error

acceptance/bundle/variables/sensitive/redact/databricks.yml

@@ -0,0 +1,22 @@
+bundle:
+  name: sensitive-variables
+
+variables:
+  target_catalog:
+    default: example_catalog
+  target_schema:
+    default: example_schema
+  api_credential:
+    description: Credential for the external API, passed via BUNDLE_VAR_api_credential. Sensitive because it is used as the value of a secret.
+  api_header:
+    description: A sensitive variable may reference another sensitive variable
+    sensitive: true
+    default: "Bearer ${var.api_credential}"
+
+resources:
+  secrets:
+    example_secret:
+      name: example_secret
+      catalog_name: ${var.target_catalog}
+      schema_name: ${var.target_schema}
+      value: ${var.api_credential}

acceptance/bundle/variables/sensitive/script

@@ -0,0 +1,11 @@
+title "A variable used as the value of a secret is sensitive and redacted in validate -o json\n"
+withdir redact trace $CLI bundle validate -o json | contains.py '!dummy-credential' | jq '{variables, resources}'
+
+title "Sensitive variable used in non-sensitive resource fields\n"
+withdir invalid-resource-field musterr trace $CLI bundle validate
+
+title "Sensitive variable used in the default of a non-sensitive variable\n"
+withdir invalid-variable musterr trace $CLI bundle validate
+
+title "Sensitive variable passed as the value of a non-sensitive variable\n"
+withdir redact musterr trace $CLI bundle validate --var 'target_schema=${var.api_credential}'

acceptance/bundle/variables/sensitive/test.toml

@@ -0,0 +1,3 @@
+Env.BUNDLE_VAR_api_credential = "dummy-credential"
+
+Ignore = [".databricks"]

bundle/config/mutator/python/python_mutator.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle/config/mutator/resourcemutator"
+	"github.com/databricks/cli/bundle/config/validate"
 
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/logdiag"
@@ -342,6 +343,14 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		return mutateDiags
 	}
 
+	// Resources added or updated by Python were not loaded when the initialize phase
+	// checked sensitive variable references. Check them on the full config: normalization
+	// below only merges resources back, so a marker inferred inside it would be lost.
+	bundle.ApplyContext(ctx, b, validate.SensitiveVariableReferences())
+	if logdiag.HasError(ctx) {
+		return mutateDiags
+	}
+
 	resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources)
 	if logdiag.HasError(ctx) {
 		return mutateDiags