direct: store serialized_dashboard in state as a content hash

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json

@@ -12,7 +12,7 @@
     "etag": [ETAG],
     "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}",
+    "serialized_dashboard": "sha256_hashed_in_state:[HASH]",
     "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
    }
   }

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -12,7 +12,7 @@
     "etag": "[NUMID]",
     "parent_path": "/Workspace/Users/[USERNAME]",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+    "serialized_dashboard": "sha256_hashed_in_state:[HASH]",
     "warehouse_id": "123456"
    }
   }

acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json

@@ -50,9 +50,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
-          "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
-          "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]",
+          "remote": "sha256_hashed_in_state:[HASH][1]"
         },
         "update_time": {
           "action": "skip",

acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl

@@ -0,0 +1 @@
+{"pages":[{"name":"main","displayName":"$DASH_TITLE"}]}

acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: dashboard-state-sha-$UNIQUE_NAME
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: $DASHBOARD_DISPLAY_NAME
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      embed_credentials: true
+      file_path: "dashboard.lvdash.json"
+      parent_path: /Users/$CURRENT_USER_NAME

acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json

@@ -0,0 +1,3 @@
+[
+  "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n"
+]

acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json

@@ -0,0 +1,19 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "display_name": "dashboard-state-sha [UUID]",
+          "embed_credentials": true,
+          "parent_path": "/Workspace/Users/[USERNAME]",
+          "published": true,
+          "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n",
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json

@@ -0,0 +1,65 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "skip",
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "dashboard-state-sha [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json",
+        "published": true,
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "create_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        },
+        "dashboard_id": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[DASHBOARD_ID]"
+        },
+        "etag": {
+          "action": "skip",
+          "reason": "custom",
+          "old": [ETAG],
+          "remote": [ETAG]
+        },
+        "lifecycle_state": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "ACTIVE"
+        },
+        "path": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json"
+        },
+        "serialized_dashboard": {
+          "action": "skip",
+          "reason": "etag_based",
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]",
+          "remote": "sha256_hashed_in_state:[HASH][1]"
+        },
+        "update_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json

@@ -0,0 +1,74 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "update",
+      "new_state": {
+        "value": {
+          "display_name": "dashboard-state-sha [UUID]",
+          "embed_credentials": true,
+          "parent_path": "/Workspace/Users/[USERNAME]",
+          "published": true,
+          "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n",
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      },
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "dashboard-state-sha [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json",
+        "published": true,
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "create_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        },
+        "dashboard_id": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[DASHBOARD_ID]"
+        },
+        "etag": {
+          "action": "skip",
+          "reason": "custom",
+          "old": [ETAG],
+          "remote": [ETAG]
+        },
+        "lifecycle_state": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "ACTIVE"
+        },
+        "path": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json"
+        },
+        "serialized_dashboard": {
+          "action": "update",
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][1]",
+          "remote": "sha256_hashed_in_state:[HASH][2]"
+        },
+        "update_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt

@@ -0,0 +1 @@
+json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:[HASH]";

acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt

@@ -0,0 +1 @@
+json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:[HASH]";

acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json

@@ -0,0 +1,3 @@
+[
+  "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n"
+]

acceptance/bundle/resources/dashboard-state-sha/output.txt

@@ -0,0 +1,26 @@
+
+=== Create: plan keeps full content in new_state but reports the diff as a hash
+>>> [CLI] bundle plan -o json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Re-plan with no local change is a no-op (server normalization is ignored, etag_based)
+>>> [CLI] bundle plan -o json
+
+=== Edit serialized_dashboard: the hash changes, so the local diff detects an update
+>>> [CLI] bundle plan -o json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboard-state-sha/script

@@ -0,0 +1,60 @@
+DASHBOARD_DISPLAY_NAME="dashboard-state-sha $(uuid)"
+if [ -z "$CLOUD_ENV" ]; then
+    export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234"
+    echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS
+fi
+export DASHBOARD_DISPLAY_NAME
+envsubst < databricks.yml.tmpl > databricks.yml
+
+render_dashboard() {
+    # Strip CR so the rendered content is LF on every platform. dashboard.tmpl is not pinned to
+    # eol=lf, so on Windows it renders with a CRLF trailing newline; since serialized_dashboard is
+    # hashed into state, a CRLF here would hash differently than the LF content in a saved plan and
+    # make the no-op re-plan (READPLAN=1) spuriously report an update.
+    DASH_TITLE="$1" envsubst < dashboard.tmpl | tr -d '\r' > dashboard.lvdash.json
+}
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+rm -f out.requests.txt
+
+title "Create: plan keeps full content in new_state but reports the diff as a hash"
+render_dashboard "Sales Overview"
+trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+
+# Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The plan
+# and the persisted state keep only the hash, but the create API call must still send the
+# FULL serialized_dashboard. out.create.serialized.json is identical for READPLAN="" and
+# READPLAN=1, which proves the saved plan applies the real content rather than the hash.
+# Not traced: the deploy command line differs by READPLAN (--plan vs none).
+$CLI bundle deploy $(readplanarg out.plan_create.direct.json)
+
+DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
+echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+
+jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json
+rm -f out.requests.txt
+
+# Persisted state holds ONLY the content hash, never the dashboard JSON.
+print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt
+
+title "Re-plan with no local change is a no-op (server normalization is ignored, etag_based)"
+trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json
+
+title "Edit serialized_dashboard: the hash changes, so the local diff detects an update"
+render_dashboard "Sales Overview v2"
+trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
+
+# Apply the update (in-memory or from the saved plan). The update API call must carry the
+# FULL new content; out.update.serialized.json is identical across READPLAN, proving the
+# saved plan applies the real content for updates too.
+$CLI bundle deploy $(readplanarg out.plan_update.direct.json)
+
+jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.update.serialized.json
+rm -f out.requests.txt
+
+# State holds the NEW content hash after the update.
+print_state.py | gron.py | grep serialized_dashboard > out.state_after_update.$DATABRICKS_BUNDLE_ENGINE.txt

acceptance/bundle/resources/dashboard-state-sha/test.toml

@@ -0,0 +1,24 @@
+# Direct-engine hashed-state behaviour for dashboards. Kept outside dashboards/ so it can run
+# direct-only: terraform does not hash state, and the saved-plan path (deploy --plan, i.e.
+# READPLAN=1) is direct-only. RecordRequests is inherited from resources/test.toml.
+# Local-only: this exercises direct-engine state format (serialized_dashboard stored as a
+# content hash), which the in-memory testserver covers fully and deterministically. On real
+# cloud the read-after-deploy plan/state reads hit dashboard eventual consistency, and unlike
+# the tests under dashboards/ this dir doesn't inherit their retry/INJECT_STALE machinery.
+Local = true
+Cloud = false
+
+# dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit).
+Ignore = ["dashboard.lvdash.json"]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.READPLAN = ["", "1"]
+
+# Etag can be both negative and positive.
+[[Repls]]
+Old = "\"[-0-9]{8,}\""
+New = "[ETAG]"
+
+[[Repls]]
+Old = "\"[0-9]{8,}\""
+New = "[ETAG]"

acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json

@@ -59,9 +59,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\n  \"pages\": [\n    {\n      \"displayName\": \"New Page\",\n      \"layout\": [\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"widget\": {\n            \"name\": \"82eb9107\",\n            \"textbox_spec\": \"# I'm a title\"\n          }\n        },\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"widget\": {\n            \"name\": \"ffa6de4f\",\n            \"textbox_spec\": \"Text\"\n          }\n        }\n      ],\n      \"name\": \"fdd21a3c\"\n    }\n  ]\n}\n",
-          "new": "{\n  \"pages\": [\n    {\n      \"displayName\": \"New Page\",\n      \"layout\": [\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"widget\": {\n            \"name\": \"82eb9107\",\n            \"textbox_spec\": \"# I'm a title\"\n          }\n        },\n        {\n          \"position\": {\n            \"height\": 2,\n            \"width\": 6,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"widget\": {\n            \"name\": \"ffa6de4f\",\n            \"textbox_spec\": \"Text\"\n          }\n        }\n      ],\n      \"name\": \"fdd21a3c\"\n    }\n  ]\n}\n",
-          "remote": "{}\n"
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]",
+          "remote": "sha256_hashed_in_state:[HASH][1]"
         },
         "update_time": {
           "action": "skip",

acceptance/bundle/resources/dashboards/simple/out.plan.direct.json

@@ -50,9 +50,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{ }\n",
-          "new": "{ }\n",
-          "remote": "{}\n"
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]",
+          "remote": "sha256_hashed_in_state:[HASH][1]"
         },
         "update_time": {
           "action": "skip",

acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json

@@ -66,9 +66,9 @@
         "serialized_dashboard": {
           "action": "skip",
           "reason": "etag_based",
-          "old": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}",
-          "new": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}",
-          "remote": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}"
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]",
+          "remote": "sha256_hashed_in_state:[HASH][1]"
         },
         "update_time": {
           "action": "skip",

acceptance/bundle/test.toml

@@ -22,3 +22,13 @@ New = 'os/[OS]'
 [[Repls]]
 Old = ' cicd/github'
 New = ''
+
+# Mask the serialized_dashboard content hash before the generic numeric/id rules can
+# nibble digit-runs out of the hex (Order below their default). Distinct numbers the
+# hashes, so old==new stays one token and a differing remote gets another
+# (e.g. [HASH][0] vs [HASH][1]).
+[[Repls]]
+Old = "sha256_hashed_in_state:[0-9a-f]{64}"
+New = "sha256_hashed_in_state:[HASH]"
+Order = -1
+Distinct = true