Ignore local resource state when deployment history is enabled

.nextchanges/bundles/dms-ignore-local-state.md

@@ -0,0 +1 @@
+* Fix stale local state preventing bundles from switching deployment history on or off after destroy. ([#6686](https://github.com/databricks/cli/pull/6686))

acceptance/bin/print_state.py

@@ -85,7 +85,8 @@ def get_remote_state_path(target):
     load would need whatever --var and flags the test deployed with, which a helper cannot know.
     A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those
     bundles are the ones with nothing to parameterize."""
-    target_dir = os.path.dirname(get_state_file(target, False))
+    # DMS has no local state file; its sync snapshots still live beside the direct-engine path.
+    target_dir = os.path.dirname(get_state_files(target, False)[-1])
     snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json")
     if snapshots:
         # One snapshot per remote path, so a test that moved its root leaves several: the newest
@@ -173,13 +174,12 @@ def get_last_version_id(target):
     return (deployment or {}).get("last_version_id")
 
 
-def print_recorded_state(filename, target):
+def print_recorded_state(data, target):
     """Print the state file with its resources filled in from the deployment metadata service.
 
     While recording, the file itself carries only the header - the service holds the resources - so
     printing it raw would show an empty state and differ from the same test's non-recording run.
     """
-    data = json.loads(open(filename).read())
     # Recording stamps each resource payload with the deployment and version; drop it here so the
     # printed state matches a non-recording run without every caller piping through nostamp.
     data["state"] = scrub(get_recorded_state(target))
@@ -217,16 +217,24 @@ def main():
     )
     args = parser.parse_args()
 
+    recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true"
+    terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform"
+    if recording and not terraform and not args.backup:
+        state_path = get_remote_state_path(args.target)
+        data = run_json(
+            [CLI, "workspace", "export", f"{state_path}/resources.json", "--format", "RAW"], allow_failure=True
+        )
+        if data is not None:
+            if args.no_dms:
+                print(json.dumps(data, indent=1))
+            else:
+                print_recorded_state(data, args.target)
+        return
+
     for filename in get_state_files(args.target, args.backup):
         if not os.path.exists(filename):
             continue
-        # Recording only applies to the direct engine, so a terraform run prints the file as-is.
-        recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true"
-        terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform"
-        if recording and not terraform and not args.no_dms:
-            print_recorded_state(filename, args.target)
-        else:
-            print_file(filename)
+        print_file(filename)
 
 
 if __name__ == "__main__":

acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt

@@ -48,3 +48,27 @@ Available state files:
 - resources.json: remote direct state serial=[SERIAL] lineage="[LINEAGE_B]"
 - [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=[SERIAL] lineage="[LINEAGE_A]"
 
+
+=== Deployment history can start over despite a stale local marker
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files...
+Created jobs.test_job
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.test_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default
+
+Destroy: 1 deleted
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files...
+Created jobs.test_job
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script

@@ -34,3 +34,16 @@ title "Machine A: any command that pulls remote state now fails\n"
 trace musterr $CLI bundle plan
 trace musterr $CLI bundle deploy
 trace musterr $CLI bundle destroy --auto-approve
+
+title "Deployment history can start over despite a stale local marker"
+rm -rf .databricks
+$CLI bundle destroy --auto-approve > /dev/null 2>&1
+export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true
+trace $CLI bundle deploy
+stale_marker=$(print_state.py --no-dms)
+trace $CLI bundle destroy --auto-approve
+mkdir -p .databricks/bundle/default
+printf '%s' "$stale_marker" > .databricks/bundle/default/resources.json
+trace $CLI bundle deploy
+trace $CLI bundle plan
+test "$(cat .databricks/bundle/default/resources.json)" = "$stale_marker"

acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml

@@ -1,3 +1,6 @@
+# The script exercises both modes explicitly because local state has different semantics.
+EnvMatrix.DMS = [""]
+
 # Lineage is a direct-engine concept (a UUID minted per deployment history),
 # so this scenario only applies to the direct engine.
 EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/dms/existing-state/corrupt.json

@@ -0,0 +1 @@
+not valid JSON

acceptance/bundle/dms/existing-state/output.txt

@@ -3,7 +3,7 @@
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
 Created jobs.one
-Files: 4 uploaded, 0 deleted
+Files: 6 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> [CLI] bundle plan -o json
@@ -47,9 +47,14 @@ To record this bundle's history, start it over as a new deployment:
   3. add experimental.deployment_history back and deploy again
 
 
-=== A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record
+=== A leftover local WAL does not override the remote deployment's non-recording state
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: unexpected WAL file found at [TEST_TMP_DIR]/.databricks/bundle/default/resources.json.wal: this deployment records deployment history, which does not write one
+Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources
+
+To record this bundle's history, start it over as a new deployment:
+  1. remove experimental.deployment_history from your bundle configuration
+  2. run "databricks bundle destroy" to delete the existing resources
+  3. add experimental.deployment_history back and deploy again
 
 
 === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone
@@ -61,10 +66,11 @@ All files and directories at the following location will be deleted: /Workspace/
 
 Destroy: 1 deleted
 
+=== Enable recording after destroy even if old local state and WAL survived cleanup
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
 Created jobs.one
-Files: 4 uploaded, 0 deleted
+Files: 6 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> print_requests.py --dms //api/2.0/bundle --oneline
@@ -73,12 +79,66 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
 {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
 
->>> find.py resources[.]json[.]wal --expect 0
+>>> cmp stale.wal .databricks/bundle/default/resources.json.wal
+
+=== Read-only commands ignore corrupt local state and WAL and use DMS
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> cmp corrupt.json .databricks/bundle/default/resources.json
+
+=== DMS deploys do not recreate local resources.json, but keep the remote marker
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> find.py resources[.]json([.]wal)?$ --expect 0
+
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms
+{
+ "state_version": 2,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "features": {
+  "deployment_history": {}
+ },
+ "state": {}
+}
+
+=== Disabling history while deployed is refused without creating a local marker
+>>> [CLI] bundle deploy
+Error: unsetting experimental.deployment_history is not supported
+
+This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle
+
+
+>>> find.py resources[.]json([.]wal)?$ --expect 0
+
+=== Switch back to direct after destroy even if an older CLI cached the DMS marker
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.one
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted
+
+>>> [CLI] bundle plan
+create jobs.one
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> find.py resources[.]json([.]wal)?$ --expect 0
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
+Created jobs.one
+Files: 6 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> jq {state_version, features} .databricks/bundle/default/resources.json
 {
   "state_version": 2,
-  "features": {
-    "deployment_history": {}
-  }
+  "features": null
 }

acceptance/bundle/dms/existing-state/script

@@ -5,6 +5,7 @@ trace $CLI bundle deploy
 dms_plan
 trace print_requests.py --dms //api/2.0/bundle --oneline
 trace jq '{state_version, features}' .databricks/bundle/default/resources.json
+direct_state=$(cat .databricks/bundle/default/resources.json)
 
 title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time"
 musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
@@ -14,19 +15,50 @@ title "The check is against the remote state, not the local cache: wiping .datab
 rm -rf .databricks
 musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 
-title "A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record"
+title "A leftover local WAL does not override the remote deployment's non-recording state"
 mkdir -p .databricks/bundle/default
-printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal
+cp stale.wal .databricks/bundle/default/resources.json.wal
 musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 
 title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone"
 # Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it
 # would otherwise try to replay it and fail on the lineage mismatch.
 rm -f .databricks/bundle/default/resources.json.wal
 trace $CLI bundle destroy --auto-approve
+
+title "Enable recording after destroy even if old local state and WAL survived cleanup"
+mkdir -p .databricks/bundle/default
+printf '%s' "$direct_state" > .databricks/bundle/default/resources.json
+cp stale.wal .databricks/bundle/default/resources.json.wal
 trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 trace print_requests.py --dms //api/2.0/bundle --oneline
-trace find.py 'resources[.]json[.]wal' --expect 0
+test "$(cat .databricks/bundle/default/resources.json)" = "$direct_state"
+trace cmp stale.wal .databricks/bundle/default/resources.json.wal
+
+title "Read-only commands ignore corrupt local state and WAL and use DMS"
+cp corrupt.json .databricks/bundle/default/resources.json
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan | contains.py '1 unchanged'
+trace cmp corrupt.json .databricks/bundle/default/resources.json
+
+title "DMS deploys do not recreate local resources.json, but keep the remote marker"
+rm .databricks/bundle/default/resources.json .databricks/bundle/default/resources.json.wal
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+trace find.py 'resources[.]json([.]wal)?$' --expect 0
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms
+
+title "Disabling history while deployed is refused without creating a local marker"
+musterr trace $CLI bundle deploy
+trace find.py 'resources[.]json([.]wal)?$' --expect 0
+
+title "Switch back to direct after destroy even if an older CLI cached the DMS marker"
+legacy_marker=$(DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms)
+printf '%s' "$legacy_marker" > .databricks/bundle/default/resources.json
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve
+test "$(cat .databricks/bundle/default/resources.json)" = "$legacy_marker"
+trace $CLI bundle plan
+trace find.py 'resources[.]json([.]wal)?$' --expect 0
+printf '%s' "$legacy_marker" > .databricks/bundle/default/resources.json
+trace $CLI bundle deploy
 trace jq '{state_version, features}' .databricks/bundle/default/resources.json
 
 # The recorded requests are asserted above via print_requests.py; drop whatever the run added after

acceptance/bundle/dms/existing-state/stale.wal

@@ -0,0 +1 @@
+{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}

acceptance/bundle/dms/generate-resource-cache/dashboard.lvdash.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/dms/generate-resource-cache/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: dms-generate-resource-cache
+
+resources:
+  dashboards:
+    dashboard:
+      display_name: current dashboard
+      warehouse_id: my-warehouse-1234
+      file_path: ./dashboard.lvdash.json
+  genie_spaces:
+    space:
+      title: current genie space
+      warehouse_id: my-warehouse-1234
+      file_path: ./space.geniespace.json

acceptance/bundle/dms/generate-resource-cache/out/config/current_dashboard.dashboard.yml

@@ -0,0 +1,6 @@
+resources:
+  dashboards:
+    current_dashboard:
+      display_name: "current dashboard"
+      warehouse_id: my-warehouse-1234
+      file_path: ../dashboard/current_dashboard.lvdash.json

acceptance/bundle/dms/generate-resource-cache/out/config/current_genie.genie_space.yml

@@ -0,0 +1,7 @@
+resources:
+  genie_spaces:
+    current_genie:
+      title: "current genie space"
+      warehouse_id: my-warehouse-1234
+      file_path: ../genie/current_genie.geniespace.json
+      parent_path: /Workspace/Users/[USERNAME]/.bundle/dms-generate-resource-cache/default/resources

acceptance/bundle/dms/generate-resource-cache/out/dashboard/current_dashboard.lvdash.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/dms/generate-resource-cache/out/genie/current_genie.geniespace.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/dms/generate-resource-cache/output.txt

@@ -0,0 +1,53 @@
+
+=== Deploy the current resources with DMS
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-generate-resource-cache/default/files...
+Created dashboards.dashboard
+Created genie_spaces.space
+Files: 6 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Dashboard generation must not fetch the stale local ID or overwrite its asset
+>>> [CLI] bundle generate dashboard --resource dashboard --force
+Error: generating a dashboard with --resource is not supported when deployment history is enabled; use --existing-id instead
+
+
+>>> print_requests.py --get //api/2.0/lakeview/dashboards
+
+>>> cmp LOG.dashboard-before dashboard.lvdash.json
+
+=== Genie generation must not fetch the stale local ID or overwrite its asset
+>>> [CLI] bundle generate genie-space --resource space --force
+Error: generating a Genie space with --resource is not supported when deployment history is enabled; use --existing-id instead
+
+
+>>> print_requests.py --get //api/2.0/genie/spaces
+
+>>> cmp LOG.genie-before space.geniespace.json
+
+>>> cmp stale-resources.json .databricks/bundle/default/resources.json
+
+=== Generation by explicit ID remains available
+>>> [CLI] bundle generate dashboard --existing-id [DASHBOARD_ID] --key current_dashboard --resource-dir out/config --dashboard-dir out/dashboard
+Writing dashboard to out/dashboard/current_dashboard.lvdash.json
+Writing configuration to out/config/current_dashboard.dashboard.yml
+Warning: Generated configuration is not included in the bundle
+
+The file out/config/current_dashboard.dashboard.yml is not matched by any pattern in the 'include' section of databricks.yml,
+so it will not be deployed. Add a matching entry to the 'include' section, for example:
+
+include:
+  - out/config/*.yml
+
+
+>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --key current_genie --resource-dir out/config --genie-space-dir out/genie
+Writing genie space to out/genie/current_genie.geniespace.json
+Writing configuration to out/config/current_genie.genie_space.yml
+Warning: Generated configuration is not included in the bundle
+
+The file out/config/current_genie.genie_space.yml is not matched by any pattern in the 'include' section of databricks.yml,
+so it will not be deployed. Add a matching entry to the 'include' section, for example:
+
+include:
+  - out/config/*.yml
+