diff --git a/.nextchanges/bundles/dms-ignore-local-state.md b/.nextchanges/bundles/dms-ignore-local-state.md new file mode 100644 index 00000000000..e982f010b35 --- /dev/null +++ b/.nextchanges/bundles/dms-ignore-local-state.md @@ -0,0 +1 @@ +* Fix stale local state preventing bundles from switching deployment history on or off after destroy. ([#6686](https://github.com/databricks/cli/pull/6686)) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 7eb9ea89c79..bd8124cf8e9 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -85,7 +85,8 @@ def get_remote_state_path(target): load would need whatever --var and flags the test deployed with, which a helper cannot know. A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those bundles are the ones with nothing to parameterize.""" - target_dir = os.path.dirname(get_state_file(target, False)) + # DMS has no local state file; its sync snapshots still live beside the direct-engine path. + target_dir = os.path.dirname(get_state_files(target, False)[-1]) snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") if snapshots: # One snapshot per remote path, so a test that moved its root leaves several: the newest @@ -173,13 +174,12 @@ def get_last_version_id(target): return (deployment or {}).get("last_version_id") -def print_recorded_state(filename, target): +def print_recorded_state(data, target): """Print the state file with its resources filled in from the deployment metadata service. While recording, the file itself carries only the header - the service holds the resources - so printing it raw would show an empty state and differ from the same test's non-recording run. """ - data = json.loads(open(filename).read()) # Recording stamps each resource payload with the deployment and version; drop it here so the # printed state matches a non-recording run without every caller piping through nostamp. data["state"] = scrub(get_recorded_state(target)) @@ -217,16 +217,24 @@ def main(): ) args = parser.parse_args() + recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true" + terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform" + if recording and not terraform and not args.backup: + state_path = get_remote_state_path(args.target) + data = run_json( + [CLI, "workspace", "export", f"{state_path}/resources.json", "--format", "RAW"], allow_failure=True + ) + if data is not None: + if args.no_dms: + print(json.dumps(data, indent=1)) + else: + print_recorded_state(data, args.target) + return + for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - # Recording only applies to the direct engine, so a terraform run prints the file as-is. - recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true" - terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform" - if recording and not terraform and not args.no_dms: - print_recorded_state(filename, args.target) - else: - print_file(filename) + print_file(filename) if __name__ == "__main__": diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt index 74c75a901b8..47afc7cc9d3 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt @@ -48,3 +48,27 @@ Available state files: - resources.json: remote direct state serial=[SERIAL] lineage="[LINEAGE_B]" - [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=[SERIAL] lineage="[LINEAGE_A]" + +=== Deployment history can start over despite a stale local marker +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files... +Created jobs.test_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default + +Destroy: 1 deleted + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files... +Created jobs.test_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script index fb89d480852..967b73e252f 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script @@ -34,3 +34,16 @@ title "Machine A: any command that pulls remote state now fails\n" trace musterr $CLI bundle plan trace musterr $CLI bundle deploy trace musterr $CLI bundle destroy --auto-approve + +title "Deployment history can start over despite a stale local marker" +rm -rf .databricks +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true +trace $CLI bundle deploy +stale_marker=$(print_state.py --no-dms) +trace $CLI bundle destroy --auto-approve +mkdir -p .databricks/bundle/default +printf '%s' "$stale_marker" > .databricks/bundle/default/resources.json +trace $CLI bundle deploy +trace $CLI bundle plan +test "$(cat .databricks/bundle/default/resources.json)" = "$stale_marker" diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml index a075bb4eefc..4f346733a02 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + # Lineage is a direct-engine concept (a UUID minted per deployment history), # so this scenario only applies to the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/existing-state/corrupt.json b/acceptance/bundle/dms/existing-state/corrupt.json new file mode 100644 index 00000000000..d1464176500 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/corrupt.json @@ -0,0 +1 @@ +not valid JSON diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a8d2f79a989..9b64a40f097 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one -Files: 4 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json @@ -47,9 +47,14 @@ To record this bundle's history, start it over as a new deployment: 3. add experimental.deployment_history back and deploy again -=== A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record +=== A leftover local WAL does not override the remote deployment's non-recording state >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: unexpected WAL file found at [TEST_TMP_DIR]/.databricks/bundle/default/resources.json.wal: this deployment records deployment history, which does not write one +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.deployment_history back and deploy again === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone @@ -61,10 +66,11 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted +=== Enable recording after destroy even if old local state and WAL survived cleanup >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one -Files: 4 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --oneline @@ -73,12 +79,66 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} ->>> find.py resources[.]json[.]wal --expect 0 +>>> cmp stale.wal .databricks/bundle/default/resources.json.wal + +=== Read-only commands ignore corrupt local state and WAL and use DMS +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> cmp corrupt.json .databricks/bundle/default/resources.json + +=== DMS deploys do not recreate local resources.json, but keep the remote marker +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> find.py resources[.]json([.]wal)?$ --expect 0 + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "features": { + "deployment_history": {} + }, + "state": {} +} + +=== Disabling history while deployed is refused without creating a local marker +>>> [CLI] bundle deploy +Error: unsetting experimental.deployment_history is not supported + +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.deployment_history: true to deploy or destroy this bundle + + +>>> find.py resources[.]json([.]wal)?$ --expect 0 + +=== Switch back to direct after destroy even if an older CLI cached the DMS marker +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> [CLI] bundle plan +create jobs.one + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> find.py resources[.]json([.]wal)?$ --expect 0 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> jq {state_version, features} .databricks/bundle/default/resources.json { "state_version": 2, - "features": { - "deployment_history": {} - } + "features": null } diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index aca3e87b5bc..59a0c5f3d70 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -5,6 +5,7 @@ trace $CLI bundle deploy dms_plan trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json +direct_state=$(cat .databricks/bundle/default/resources.json) title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time" musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy @@ -14,9 +15,9 @@ title "The check is against the remote state, not the local cache: wiping .datab rm -rf .databricks musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -title "A leftover local WAL is refused: a recorded deployment writes none, so one can only have come from a deploy that did not record" +title "A leftover local WAL does not override the remote deployment's non-recording state" mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +cp stale.wal .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" @@ -24,9 +25,40 @@ title "Destroy clears the deployment. With nothing recorded remotely the bundle # would otherwise try to replay it and fail on the lineage mismatch. rm -f .databricks/bundle/default/resources.json.wal trace $CLI bundle destroy --auto-approve + +title "Enable recording after destroy even if old local state and WAL survived cleanup" +mkdir -p .databricks/bundle/default +printf '%s' "$direct_state" > .databricks/bundle/default/resources.json +cp stale.wal .databricks/bundle/default/resources.json.wal trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline -trace find.py 'resources[.]json[.]wal' --expect 0 +test "$(cat .databricks/bundle/default/resources.json)" = "$direct_state" +trace cmp stale.wal .databricks/bundle/default/resources.json.wal + +title "Read-only commands ignore corrupt local state and WAL and use DMS" +cp corrupt.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan | contains.py '1 unchanged' +trace cmp corrupt.json .databricks/bundle/default/resources.json + +title "DMS deploys do not recreate local resources.json, but keep the remote marker" +rm .databricks/bundle/default/resources.json .databricks/bundle/default/resources.json.wal +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace find.py 'resources[.]json([.]wal)?$' --expect 0 +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms + +title "Disabling history while deployed is refused without creating a local marker" +musterr trace $CLI bundle deploy +trace find.py 'resources[.]json([.]wal)?$' --expect 0 + +title "Switch back to direct after destroy even if an older CLI cached the DMS marker" +legacy_marker=$(DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py --no-dms) +printf '%s' "$legacy_marker" > .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve +test "$(cat .databricks/bundle/default/resources.json)" = "$legacy_marker" +trace $CLI bundle plan +trace find.py 'resources[.]json([.]wal)?$' --expect 0 +printf '%s' "$legacy_marker" > .databricks/bundle/default/resources.json +trace $CLI bundle deploy trace jq '{state_version, features}' .databricks/bundle/default/resources.json # The recorded requests are asserted above via print_requests.py; drop whatever the run added after diff --git a/acceptance/bundle/dms/existing-state/stale.wal b/acceptance/bundle/dms/existing-state/stale.wal new file mode 100644 index 00000000000..df55d0bc321 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/stale.wal @@ -0,0 +1 @@ +{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}} diff --git a/acceptance/bundle/dms/generate-resource-cache/dashboard.lvdash.json b/acceptance/bundle/dms/generate-resource-cache/dashboard.lvdash.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/dashboard.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/dms/generate-resource-cache/databricks.yml b/acceptance/bundle/dms/generate-resource-cache/databricks.yml new file mode 100644 index 00000000000..f5bc80e3fc1 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: dms-generate-resource-cache + +resources: + dashboards: + dashboard: + display_name: current dashboard + warehouse_id: my-warehouse-1234 + file_path: ./dashboard.lvdash.json + genie_spaces: + space: + title: current genie space + warehouse_id: my-warehouse-1234 + file_path: ./space.geniespace.json diff --git a/acceptance/bundle/dms/generate-resource-cache/out.test.toml b/acceptance/bundle/dms/generate-resource-cache/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/generate-resource-cache/out/config/current_dashboard.dashboard.yml b/acceptance/bundle/dms/generate-resource-cache/out/config/current_dashboard.dashboard.yml new file mode 100644 index 00000000000..8f798dcefe2 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/out/config/current_dashboard.dashboard.yml @@ -0,0 +1,6 @@ +resources: + dashboards: + current_dashboard: + display_name: "current dashboard" + warehouse_id: my-warehouse-1234 + file_path: ../dashboard/current_dashboard.lvdash.json diff --git a/acceptance/bundle/dms/generate-resource-cache/out/config/current_genie.genie_space.yml b/acceptance/bundle/dms/generate-resource-cache/out/config/current_genie.genie_space.yml new file mode 100644 index 00000000000..017dd5675f8 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/out/config/current_genie.genie_space.yml @@ -0,0 +1,7 @@ +resources: + genie_spaces: + current_genie: + title: "current genie space" + warehouse_id: my-warehouse-1234 + file_path: ../genie/current_genie.geniespace.json + parent_path: /Workspace/Users/[USERNAME]/.bundle/dms-generate-resource-cache/default/resources diff --git a/acceptance/bundle/dms/generate-resource-cache/out/dashboard/current_dashboard.lvdash.json b/acceptance/bundle/dms/generate-resource-cache/out/dashboard/current_dashboard.lvdash.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/out/dashboard/current_dashboard.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/dms/generate-resource-cache/out/genie/current_genie.geniespace.json b/acceptance/bundle/dms/generate-resource-cache/out/genie/current_genie.geniespace.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/out/genie/current_genie.geniespace.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/dms/generate-resource-cache/output.txt b/acceptance/bundle/dms/generate-resource-cache/output.txt new file mode 100644 index 00000000000..6cfc7470038 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/output.txt @@ -0,0 +1,53 @@ + +=== Deploy the current resources with DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-generate-resource-cache/default/files... +Created dashboards.dashboard +Created genie_spaces.space +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Dashboard generation must not fetch the stale local ID or overwrite its asset +>>> [CLI] bundle generate dashboard --resource dashboard --force +Error: generating a dashboard with --resource is not supported when deployment history is enabled; use --existing-id instead + + +>>> print_requests.py --get //api/2.0/lakeview/dashboards + +>>> cmp LOG.dashboard-before dashboard.lvdash.json + +=== Genie generation must not fetch the stale local ID or overwrite its asset +>>> [CLI] bundle generate genie-space --resource space --force +Error: generating a Genie space with --resource is not supported when deployment history is enabled; use --existing-id instead + + +>>> print_requests.py --get //api/2.0/genie/spaces + +>>> cmp LOG.genie-before space.geniespace.json + +>>> cmp stale-resources.json .databricks/bundle/default/resources.json + +=== Generation by explicit ID remains available +>>> [CLI] bundle generate dashboard --existing-id [DASHBOARD_ID] --key current_dashboard --resource-dir out/config --dashboard-dir out/dashboard +Writing dashboard to out/dashboard/current_dashboard.lvdash.json +Writing configuration to out/config/current_dashboard.dashboard.yml +Warning: Generated configuration is not included in the bundle + +The file out/config/current_dashboard.dashboard.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/config/*.yml + + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --key current_genie --resource-dir out/config --genie-space-dir out/genie +Writing genie space to out/genie/current_genie.geniespace.json +Writing configuration to out/config/current_genie.genie_space.yml +Warning: Generated configuration is not included in the bundle + +The file out/config/current_genie.genie_space.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/config/*.yml + diff --git a/acceptance/bundle/dms/generate-resource-cache/script b/acceptance/bundle/dms/generate-resource-cache/script new file mode 100644 index 00000000000..a915f1b6304 --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/script @@ -0,0 +1,28 @@ +title "Deploy the current resources with DMS" +trace $CLI bundle deploy +cp dashboard.lvdash.json LOG.dashboard-before +cp space.geniespace.json LOG.genie-before +cp stale-resources.json .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + +title "Dashboard generation must not fetch the stale local ID or overwrite its asset" +musterr trace $CLI bundle generate dashboard --resource dashboard --force +touch "$OUT_REQUESTS" +trace print_requests.py --get //api/2.0/lakeview/dashboards +trace cmp LOG.dashboard-before dashboard.lvdash.json + +title "Genie generation must not fetch the stale local ID or overwrite its asset" +musterr trace $CLI bundle generate genie-space --resource space --force +touch "$OUT_REQUESTS" +trace print_requests.py --get //api/2.0/genie/spaces +trace cmp LOG.genie-before space.geniespace.json +trace cmp stale-resources.json .databricks/bundle/default/resources.json + +title "Generation by explicit ID remains available" +dashboard_id=$($CLI bundle summary -o json | jq -r '.resources.dashboards.dashboard.id') +space_id=$($CLI bundle summary -o json | jq -r '.resources.genie_spaces.space.id') +add_repl "$dashboard_id" DASHBOARD_ID +add_repl "$space_id" GENIE_SPACE_ID +trace $CLI bundle generate dashboard --existing-id "$dashboard_id" --key current_dashboard --resource-dir out/config --dashboard-dir out/dashboard +trace $CLI bundle generate genie-space --existing-id "$space_id" --key current_genie --resource-dir out/config --genie-space-dir out/genie +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/generate-resource-cache/space.geniespace.json b/acceptance/bundle/dms/generate-resource-cache/space.geniespace.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/space.geniespace.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/dms/generate-resource-cache/stale-resources.json b/acceptance/bundle/dms/generate-resource-cache/stale-resources.json new file mode 100644 index 00000000000..2c69d30f97f --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/stale-resources.json @@ -0,0 +1,15 @@ +{ + "state_version": 2, + "lineage": "stale-local-lineage", + "serial": 99, + "state": { + "resources.dashboards.dashboard": { + "__id__": "stale-dashboard-id", + "state": {} + }, + "resources.genie_spaces.space": { + "__id__": "stale-genie-id", + "state": {} + } + } +} diff --git a/acceptance/bundle/dms/generate-resource-cache/test.toml b/acceptance/bundle/dms/generate-resource-cache/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/dms/generate-resource-cache/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 3ccb46a3e11..aade20b60f6 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -71,7 +71,7 @@ Updated jobs.foo Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> jq {serial} .databricks/bundle/default/resources.json +>>> print_state.py --no-dms { "serial": null } diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index bb6d0b4a598..2d45aa35758 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -30,7 +30,7 @@ $CLI bundle plan -o json > tmp.plan2.json trace jq '{serial}' tmp.plan2.json trace update_file.py databricks.yml "name: foo" "name: foo_renamed" trace $CLI bundle deploy -trace jq '{serial}' .databricks/bundle/default/resources.json +trace print_state.py --no-dms | jq '{serial}' title "Replaying the older plan is rejected: same deployment, outdated version" trace musterr $CLI bundle deploy --plan tmp.plan2.json diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index ae800809893..e1af1a235ad 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard-inplace/test.toml b/acceptance/bundle/generate/dashboard-inplace/test.toml index 10eef9f96c4..d8610976ce5 100644 --- a/acceptance/bundle/generate/dashboard-inplace/test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/test.toml @@ -1,6 +1,6 @@ -# bundle generate reading recorded state from DMS is a fast followup; until then this -# deploy-then-generate flow can't run under recording (local state is a tombstone). -EnvMatrix.DMS = [""] +# DMS is a no-op for Terraform; direct DMS generation is covered by its rejection test. +EnvMatrix.DMS = ["", "true"] +EnvMatrixExclude.dms_needs_direct = ["DMS=true", "DATABRICKS_BUNDLE_ENGINE=direct"] [[Repls]] Old = "[0-9a-f]{32}" diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/bad_json_local/output.txt b/acceptance/bundle/state/bad_json_local/output.txt index cb86fc388d4..3c15a83181c 100644 --- a/acceptance/bundle/state/bad_json_local/output.txt +++ b/acceptance/bundle/state/bad_json_local/output.txt @@ -6,3 +6,9 @@ Error: parsing [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfs >>> musterr [CLI] bundle plan Error: parsing [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: unexpected end of JSON input + +=== Deployment history ignores corrupt local resource state +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/bad_json_local/script b/acceptance/bundle/state/bad_json_local/script index fac1d1a17d3..7654f501399 100644 --- a/acceptance/bundle/state/bad_json_local/script +++ b/acceptance/bundle/state/bad_json_local/script @@ -6,3 +6,6 @@ trace musterr $CLI bundle plan 2>&1 | contains.py 'Error' 'parsing' 'terraform.t rm .databricks/bundle/default/terraform/terraform.tfstate echo "{" > .databricks/bundle/default/resources.json trace musterr $CLI bundle plan 2>&1 | contains.py 'Error' 'parsing' 'resources.json' 'unexpected end' + +title "Deployment history ignores corrupt local resource state" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan diff --git a/acceptance/bundle/state/bad_json_local/test.toml b/acceptance/bundle/state/bad_json_local/test.toml new file mode 100644 index 00000000000..05b476415aa --- /dev/null +++ b/acceptance/bundle/state/bad_json_local/test.toml @@ -0,0 +1,2 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/engine_mismatch/test.toml b/acceptance/bundle/state/engine_mismatch/test.toml index 601384fdf96..aff7d28ca09 100644 --- a/acceptance/bundle/state/engine_mismatch/test.toml +++ b/acceptance/bundle/state/engine_mismatch/test.toml @@ -1 +1,4 @@ Ignore = [".databricks"] + +# Local-only debug state inspection does not support deployment history. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index b6385259dd2..209ab334ab2 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -20,3 +20,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> gron.py .databricks/bundle/default/resources.json json.state_version = 2; + +=== Deployment history checks the remote deployment rather than unsupported local features +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.deployment_history back and deploy again + + +>>> cmp resources.with_feature.json .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index ef9570e318c..a8cf73464d3 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -9,3 +9,8 @@ cp resources.empty_features.json .databricks/bundle/default/resources.json trace $CLI bundle plan | contains.py "Plan:" trace $CLI bundle deploy trace gron.py .databricks/bundle/default/resources.json | grep state_version + +title "Deployment history checks the remote deployment rather than unsupported local features" +cp resources.with_feature.json .databricks/bundle/default/resources.json +musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan 2>&1 | contains.py 'already exists and is not recorded' +trace cmp resources.with_feature.json .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5640bffa4dc..5ba3d2b8fea 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/force_pull_commands/output.txt b/acceptance/bundle/state/force_pull_commands/output.txt index 0ef6ccf69fb..af2b07fc5a9 100644 --- a/acceptance/bundle/state/force_pull_commands/output.txt +++ b/acceptance/bundle/state/force_pull_commands/output.txt @@ -39,3 +39,46 @@ Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME" } } + +=== Deployment history always reads remote state, with or without --force-pull +>>> [CLI] bundle summary +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME" + } +} + +>>> [CLI] bundle summary --force-pull +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME" + } +} + +>>> [CLI] bundle open foo +Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME" + } +} + +>>> [CLI] bundle open foo --force-pull +Opening browser at [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/force-pull-commands/default/state/STATE_FILENAME" + } +} diff --git a/acceptance/bundle/state/force_pull_commands/script b/acceptance/bundle/state/force_pull_commands/script index fc287979037..6c0216aa680 100644 --- a/acceptance/bundle/state/force_pull_commands/script +++ b/acceptance/bundle/state/force_pull_commands/script @@ -27,3 +27,20 @@ title "bundle open --force-pull: remote state read\n" trace $CLI bundle open foo --force-pull > /dev/null touch "$OUT_REQUESTS" print_requests.py --get //workspace/export + +title "Deployment history always reads remote state, with or without --force-pull" +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -rf .databricks +export DATABRICKS_BUNDLE_ENGINE=direct +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true +$CLI bundle deploy > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +trace $CLI bundle summary > /dev/null +print_requests.py --get //workspace/export +trace $CLI bundle summary --force-pull > /dev/null +print_requests.py --get //workspace/export +trace $CLI bundle open foo > /dev/null +print_requests.py --get //workspace/export +trace $CLI bundle open foo --force-pull > /dev/null +print_requests.py --get //workspace/export diff --git a/acceptance/bundle/state/force_pull_commands/test.toml b/acceptance/bundle/state/force_pull_commands/test.toml index fdfa3ab33c0..1fea515ee9b 100644 --- a/acceptance/bundle/state/force_pull_commands/test.toml +++ b/acceptance/bundle/state/force_pull_commands/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + Cloud = false RecordRequests = true diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..2b55bab4bca 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,7 @@ state version 999 is newer than supported version 3; upgrade the CLI -Exit code: 1 +=== Deployment history ignores the local state version +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/future_version/script b/acceptance/bundle/state/future_version/script index 85fdf969f72..480832cd869 100644 --- a/acceptance/bundle/state/future_version/script +++ b/acceptance/bundle/state/future_version/script @@ -1,4 +1,7 @@ mkdir -p .databricks/bundle/default cp resources.future.json .databricks/bundle/default/resources.json -trace $CLI bundle plan 2>&1 | grep -o 'state version.*' +musterr trace $CLI bundle plan 2>&1 | grep -o 'state version.*' + +title "Deployment history ignores the local state version" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan diff --git a/acceptance/bundle/state/future_version/test.toml b/acceptance/bundle/state/future_version/test.toml index 5640bffa4dc..5ba3d2b8fea 100644 --- a/acceptance/bundle/state/future_version/test.toml +++ b/acceptance/bundle/state/future_version/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/lineage_different/output.txt b/acceptance/bundle/state/lineage_different/output.txt index bea441cf1fa..8fddfac47b4 100644 --- a/acceptance/bundle/state/lineage_different/output.txt +++ b/acceptance/bundle/state/lineage_different/output.txt @@ -41,3 +41,9 @@ Workspace: Validation OK! >>> [CLI] bundle + +=== Deployment history ignores the local resource lineage +>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/lineage_different/script b/acceptance/bundle/state/lineage_different/script index df898290049..6f99bac473a 100644 --- a/acceptance/bundle/state/lineage_different/script +++ b/acceptance/bundle/state/lineage_different/script @@ -9,3 +9,7 @@ trace musterr $CLI bundle destroy --auto-approve # These commands don't care: trace $CLI bundle validate trace $CLI bundle > /dev/null + +title "Deployment history ignores the local resource lineage" +rm .databricks/bundle/default/terraform/terraform.tfstate +trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan diff --git a/acceptance/bundle/state/lineage_different/test.toml b/acceptance/bundle/state/lineage_different/test.toml index 0f5f0266936..3774b6e54a2 100644 --- a/acceptance/bundle/state/lineage_different/test.toml +++ b/acceptance/bundle/state/lineage_different/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + RecordRequests = false IncludeRequestHeaders = ["User-Agent"] Ignore = [".databricks"] diff --git a/acceptance/bundle/state/newer_cli_version/out.test.toml b/acceptance/bundle/state/newer_cli_version/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/state/newer_cli_version/out.test.toml +++ b/acceptance/bundle/state/newer_cli_version/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/newer_cli_version/output.txt b/acceptance/bundle/state/newer_cli_version/output.txt index cfe256b2c2c..50a4adeac3a 100644 --- a/acceptance/bundle/state/newer_cli_version/output.txt +++ b/acceptance/bundle/state/newer_cli_version/output.txt @@ -13,3 +13,10 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged create jobs.my_job Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deployment history ignores the local writer version + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/newer_cli_version/script b/acceptance/bundle/state/newer_cli_version/script index 172dc126cb3..6298e59676e 100644 --- a/acceptance/bundle/state/newer_cli_version/script +++ b/acceptance/bundle/state/newer_cli_version/script @@ -9,3 +9,7 @@ title "State written by this CLI: no warning\n" version=$($CLI version | sed 's/^Databricks CLI v//') jq --arg v "$version" '.cli_version = $v' resources.newer.json > .databricks/bundle/default/resources.json trace $CLI bundle plan + +title "Deployment history ignores the local writer version\n" +cp resources.newer.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan | contains.py '!Warn:' diff --git a/acceptance/bundle/state/newer_cli_version/test.toml b/acceptance/bundle/state/newer_cli_version/test.toml index c1e21fbbd8b..8b8469cf210 100644 --- a/acceptance/bundle/state/newer_cli_version/test.toml +++ b/acceptance/bundle/state/newer_cli_version/test.toml @@ -1,3 +1,6 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] # The warning is emitted when reading the direct engine's resources.json. diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/same_serial/output.txt b/acceptance/bundle/state/same_serial/output.txt index 93a58414de8..ecd10c266a2 100644 --- a/acceptance/bundle/state/same_serial/output.txt +++ b/acceptance/bundle/state/same_serial/output.txt @@ -7,3 +7,9 @@ Available state files: - [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=20 lineage="" - [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=20 lineage="" + +=== Deployment history ignores the local resource serial +>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/state/same_serial/script b/acceptance/bundle/state/same_serial/script index 66c4e3285c8..807279ded26 100644 --- a/acceptance/bundle/state/same_serial/script +++ b/acceptance/bundle/state/same_serial/script @@ -4,3 +4,7 @@ echo '{"version": 4, "serial": 20}' > .databricks/bundle/default/terraform/terra echo '{"serial": 20}' > .databricks/bundle/default/resources.json #trace musterr $CLI bundle plan trace musterr $CLI bundle deploy + +title "Deployment history ignores the local resource serial" +rm .databricks/bundle/default/terraform/terraform.tfstate +trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle plan diff --git a/acceptance/bundle/state/same_serial/test.toml b/acceptance/bundle/state/same_serial/test.toml index 601384fdf96..65dac016474 100644 --- a/acceptance/bundle/state/same_serial/test.toml +++ b/acceptance/bundle/state/same_serial/test.toml @@ -1 +1,4 @@ +# The script exercises both modes explicitly because local state has different semantics. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f4fb00c6b9e..7b34f21c710 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -142,11 +142,17 @@ type DeploymentState struct { // VersionID is the DMS counterpart of Data.Serial - same meaning, but owned by the service // rather than the file, so it lives outside Data and is never written to resources.json. VersionID int + + // StateForUpload holds the finalized DMS marker until it is uploaded. DMS never caches it on disk. + StateForUpload []byte } // OpenDmsArgs identifies the recorded deployment Open reads from. The zero value means the // bundle does not record deployment history, or no deployment exists for it yet. type OpenDmsArgs struct { + // State is the remote resources.json marker; the local file belongs to non-DMS deployments. + State []byte + // DeploymentID is the deployment's server-minted id. DeploymentID string @@ -484,19 +490,13 @@ type ( // but disables GetResourceEntry (since writes go strictly into WAL and not in memory). WithWrite bool - // If true, the deployment records history with the metadata service: Open builds a DMS - // client from the workspace client, reads resources from the service, and refuses a state - // that tracks resources without the recording marker. It forces WithRecovery off, since the - // service is the source of truth and a leftover WAL is discarded rather than replayed. + // If true, Open reads the remote marker and resources from DMS, ignoring local state and WAL. WithDeploymentHistory bool ) // Open reads the deployment state from disk, recovering the WAL when withRecovery is set. -// When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from -// the service instead, with dmsDeploymentID the id the service holds (empty before the first -// recorded deploy); lineage and serial still come from the file, since that is what the write path -// increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a -// version exists. +// With deployment history, it reads the remote marker from dmsDeployment.State and resources and +// version from the service instead. InitializeOperationBuffer installs the write path once a version exists. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment OpenDmsArgs) error { db.mu.Lock() defer db.mu.Unlock() @@ -533,16 +533,19 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe // Cleared here rather than in reset, which CompleteVersion needs it to survive: the same state // is reopened (see bind.go), and a stale value would misroute a non-recording open. db.recordsHistory = false + db.StateForUpload = nil - // The state file is the source of truth for whether this deployment records history: read it - // first. - data, err := os.ReadFile(db.Path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - db.Data = NewDatabase("", 0) - } else { + // DMS uses only the remote marker, even if a previous deployment left local state behind. + data := dmsDeployment.State + if !withDeploymentHistory { + var err error + data, err = os.ReadFile(db.Path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } + } + if data == nil { + db.Data = NewDatabase("", 0) } else { if err := json.Unmarshal(data, &db.Data); err != nil { return err @@ -567,25 +570,21 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe recording := bool(withDeploymentHistory) walPath := db.Path + walSuffix - _, err = os.Stat(walPath) - switch { - case errors.Is(err, fs.ErrNotExist): - // no WAL, nothing to do - case err != nil: - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) - default: // WAL exists + if !recording { + _, err := os.Stat(walPath) switch { - case recording: - // A recorded deployment writes no WAL, so finding one means this state was written by a - // deployment that did not record history. Refuse rather than discard it: the file is the - // only record of that deploy's writes, and recording is refused below anyway. - return fmt.Errorf("unexpected WAL file found at %s: this deployment records deployment history, which does not write one", walPath) - case bool(withRecovery): - if err := db.replayWAL(ctx); err != nil { - return fmt.Errorf("reading state from %s: %w", path, err) + case errors.Is(err, fs.ErrNotExist): + // no WAL, nothing to do + case err != nil: + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) + default: // WAL exists + if withRecovery { + if err := db.replayWAL(ctx); err != nil { + return fmt.Errorf("reading state from %s: %w", path, err) + } + } else { + return fmt.Errorf("unexpected WAL file found at %s", walPath) } - default: - return fmt.Errorf("unexpected WAL file found at %s", walPath) } } @@ -834,11 +833,8 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported db.walFile = nil err = db.replayWAL(ctx) } else if db.openedForWrite && db.isDeploymentMetadataService() { - // replayWAL is what normally persists the file. Without one, write the tombstone here so - // the header (lineage, state version, CLI version) still lands on disk - even when no - // operations were recorded, since the deployment and any version this run created exist - // either way. - err = db.unlockedSave() + // Preserve the remote marker across reset without leaving a local DMS state file. + db.StateForUpload, err = json.MarshalIndent(db.dataForFile(), "", " ") } // Wait until all operations are recorded in the service. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 24af630b069..e9f0a1fd757 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -2,11 +2,14 @@ package dstate import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,6 +48,74 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestDeploymentHistoryIgnoresLocalState(t *testing.T) { + const remoteState = `{"state_version":2,"lineage":"remote-lineage","features":{"deployment_history":{}},"state":{}}` + for _, tt := range []struct { + name string + local string + wal string + }{ + {name: "no local files"}, + {name: "corrupt local files", local: "not json", wal: "not a WAL"}, + { + name: "previous direct deployment", + local: `{"state_version":2,"lineage":"old-lineage","serial":99,"state":{"resources.jobs.old":{"__id__":"old-job"}}}`, + wal: `{"state_version":2,"lineage":"old-lineage","serial":100}`, + }, + } { + for _, remote := range []string{"", remoteState} { + for _, write := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/remote=%t/write=%t", tt.name, remote != "", write), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resources.json") + for name, content := range map[string]string{path: tt.local, path + walSuffix: tt.wal} { + if content != "" { + require.NoError(t, os.WriteFile(name, []byte(content), 0o600)) + } + } + w := databricks.Must(databricks.NewWorkspaceClient(&databricks.Config{Host: "https://workspace.test", Token: "token"})) + ctx := cmdctx.SetWorkspaceClient(t.Context(), w) + var state []byte + if remote != "" { + state = []byte(remote) + } + var db DeploymentState + require.NoError(t, db.Open(ctx, path, WithRecovery(true), WithWrite(write), WithDeploymentHistory(true), OpenDmsArgs{State: state})) + assert.Empty(t, db.Data.State) + assert.Empty(t, db.GetResourceID("resources.jobs.old")) + if remote != "" { + assert.Equal(t, "remote-lineage", db.Data.Lineage) + } + const resourceKey = "resources.jobs.current" + if write { + require.NoError(t, db.SaveState(ctx, resourceKey, "123", map[string]string{"name": "current"}, nil)) + } + exported, err := db.Finalize(ctx) + require.NoError(t, err) + if write { + assert.Equal(t, "123", exported[resourceKey].ID) + var uploaded Database + require.NoError(t, json.Unmarshal(db.StateForUpload, &uploaded)) + assert.Contains(t, uploaded.Features, FeatureDeploymentHistory) + assert.NotEmpty(t, uploaded.Lineage) + assert.Empty(t, uploaded.State) + } else { + assert.Nil(t, db.StateForUpload) + } + for name, content := range map[string]string{path: tt.local, path + walSuffix: tt.wal} { + if content == "" { + assert.NoFileExists(t, name) + } else { + actual, err := os.ReadFile(name) + require.NoError(t, err) + assert.Equal(t, content, string(actual)) + } + } + }) + } + } + } +} + func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 8fb5967196c..b70fee05e96 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -135,6 +135,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle target that records deployment history")) + return + } + groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index b382e82dd1c..2173a6aacaa 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" @@ -25,9 +26,12 @@ import ( type AlwaysPull bool +const resourcesWALSuffix = ".wal" + type StateDesc struct { - Serial int `json:"serial"` - Lineage string `json:"lineage"` + Serial int `json:"serial"` + Lineage string `json:"lineage"` + Features map[string]struct{} `json:"features,omitempty"` // additional fields describing state: SourcePath string @@ -39,6 +43,12 @@ type StateDesc struct { AllStates []*StateDesc } +// hasDeploymentHistory reports whether the state belongs to a recorded deployment. +func (s *StateDesc) hasDeploymentHistory() bool { + _, ok := s.Features[dstate.FeatureDeploymentHistory] + return ok +} + func (s *StateDesc) String() string { source := "remote" if s.IsLocal { @@ -127,7 +137,7 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always _, localPathDirect := b.StateFilenameDirect(ctx) _, localPathTerraform := b.StateFilenameTerraform(ctx) - states := readStates(ctx, b, alwaysPull) + states, localDMS := readStates(ctx, b, alwaysPull) if logdiag.HasError(ctx) { return ctx, nil @@ -154,6 +164,12 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always logStatesError(ctx, err.Error(), states) return ctx, winner } + if localDMS { + if err := removeLocalDMSState(localPathDirect); err != nil { + logdiag.LogError(ctx, err) + return ctx, winner + } + } if requiredEngine.Type != engine.EngineNotSet && requiredEngine.Type != winner.Engine { // Direct (whether selected explicitly or by default) against a terraform @@ -185,6 +201,10 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always // local state is fresh, nothing to do return ctx, winner } + if winner.Engine.IsDirect() && b.ConfiguresDeploymentHistory(ctx) { + // DMS reads the remote marker in memory; a local copy can outlive the deployment. + return ctx, winner + } if !winner.IsLocal { log.Info(ctx, "Remote state is newer than local state. Using remote resources state.") @@ -213,24 +233,33 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always return ctx, winner } -func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) []*StateDesc { +func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) ([]*StateDesc, bool) { var states []*StateDesc remotePathDirect, localPathDirect := b.StateFilenameDirect(ctx) remotePathTerraform, localPathTerraform := b.StateFilenameTerraform(ctx) if logdiag.HasError(ctx) { - return nil + return nil, false } - directLocalState := localRead(ctx, localPathDirect, engine.EngineDirect) + var directLocalState *StateDesc + recording := b.ConfiguresDeploymentHistory(ctx) + if !recording { + directLocalState = localRead(ctx, localPathDirect, engine.EngineDirect) + } + localDMS := directLocalState != nil && directLocalState.hasDeploymentHistory() + if localDMS { + // A cached DMS marker is never authoritative, including after recording is disabled. + directLocalState = nil + } terraformLocalState := localRead(ctx, localPathTerraform, engine.EngineTerraform) - if (directLocalState == nil && terraformLocalState == nil) || alwaysPull { + if recording || localDMS || (directLocalState == nil && terraformLocalState == nil) || bool(alwaysPull) { f, err := deploy.StateFiler(ctx, b) if err != nil { logdiag.LogError(ctx, err) - return nil + return nil, false } var wg sync.WaitGroup @@ -245,6 +274,13 @@ func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) [] }) wg.Wait() + if logdiag.HasError(ctx) { + return nil, false + } + if !recording && directRemoteState != nil && directRemoteState.hasDeploymentHistory() { + logdiag.LogError(ctx, dstate.ErrUnsettingRecording) + return nil, false + } // find highest serial across all state files // sorting is stable, so initial setting represents preference (later is preferred): @@ -257,7 +293,23 @@ func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) [] return a.Serial - b.Serial }) - return states + return states, localDMS +} + +// removeLocalDMSState discards a legacy marker only after remote state has been checked. +func removeLocalDMSState(localPath string) error { + // Removing the marker before replaying an unrelated WAL could resurrect old resource IDs. + _, err := os.Stat(localPath + resourcesWALSuffix) + if err == nil { + return fmt.Errorf("cannot discard local deployment-history state %s while its WAL exists; reconcile or move both local files before retrying", filepath.ToSlash(localPath)) + } + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("checking WAL for %s: %w", filepath.ToSlash(localPath), err) + } + if err := os.Remove(localPath); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("removing obsolete deployment-history state %s: %w", filepath.ToSlash(localPath), err) + } + return nil } func validateStates(states []*StateDesc) error { diff --git a/bundle/statemgmt/state_pull_test.go b/bundle/statemgmt/state_pull_test.go new file mode 100644 index 00000000000..d59efbd9cb0 --- /dev/null +++ b/bundle/statemgmt/state_pull_test.go @@ -0,0 +1,123 @@ +package statemgmt_test + +import ( + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + legacyDMSState = `{"state_version":2,"lineage":"old-dms","features":{"deployment_history":{}},"state":{}}` + ordinaryState = `{"state_version":2,"lineage":"ordinary","serial":9,"state":{}}` + stateWALSuffix = ".wal" +) + +type statePullTransport struct { + state string + status int + reads int +} + +func (s *statePullTransport) RoundTrip(r *http.Request) (*http.Response, error) { + status := http.StatusNotFound + body := `{"error_code":"RESOURCE_DOES_NOT_EXIST","message":"not found"}` + if strings.HasSuffix(r.URL.Query().Get("path"), "/resources.json") { + if s.status != 0 { + status = s.status + body = `{"error_code":"PERMISSION_DENIED","message":"remote state denied"}` + } else if s.state != "" { + status = http.StatusOK + body = `{"object_type":"FILE","path":"/Workspace/test/state/resources.json"}` + if r.URL.Path == "/api/2.0/workspace/export" { + body = s.state + s.reads++ + } + } + } + return &http.Response{ + StatusCode: status, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(body)), + Request: r, + }, nil +} + +func TestPullResourcesStateAfterDeploymentHistory(t *testing.T) { + for _, tt := range []struct { + name string + local string + remote string + status int + wal bool + wantErr string + wantAbsent bool + }{ + {name: "destroyed deployment discards legacy cache", local: legacyDMSState, wantAbsent: true}, + {name: "ordinary remote state replaces legacy cache", local: legacyDMSState, remote: ordinaryState}, + {name: "live DMS deployment remains protected", local: legacyDMSState, remote: legacyDMSState, wantErr: "unsetting experimental.deployment_history"}, + {name: "higher serial local state cannot override live DMS", local: ordinaryState, remote: legacyDMSState, wantErr: "unsetting experimental.deployment_history"}, + {name: "remote access error preserves cache", local: legacyDMSState, status: http.StatusForbidden, wantErr: "remote state denied"}, + {name: "invalid remote state preserves cache", local: legacyDMSState, remote: "{", wantErr: "parsing state"}, + {name: "invalid local state preserves cache", local: `{"serial":"invalid","features":{"deployment_history":{}}}`, wantErr: "parsing"}, + {name: "legacy marker and WAL require reconciliation", local: legacyDMSState, wal: true, wantErr: "while its WAL exists"}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + b := &bundle.Bundle{BundleRootPath: t.TempDir()} + b.Config.Bundle.Target = "default" + b.Config.Workspace.StatePath = "/Workspace/test/state" + transport := &statePullTransport{state: tt.remote, status: tt.status} + w := databricks.Must(databricks.NewWorkspaceClient(&databricks.Config{ + Host: "https://workspace.test", Token: "token", HTTPTransport: transport, + })) + b.SetWorkpaceClient(w) + _, localPath := b.StateFilenameDirect(ctx) + require.NoError(t, os.MkdirAll(filepath.Dir(localPath), 0o700)) + require.NoError(t, os.WriteFile(localPath, []byte(tt.local), 0o600)) + const wal = "{\"state_version\":2,\"lineage\":\"stale\",\"serial\":1}\n{\"k\":\"resources.jobs.old\",\"v\":{\"__id__\":\"stale-id\"}}\n" + if tt.wal { + require.NoError(t, os.WriteFile(localPath+stateWALSuffix, []byte(wal), 0o600)) + } + + // A high-serial ordinary cache is valid for read-only commands; deploy always pulls. + alwaysPull := statemgmt.AlwaysPull(tt.local == ordinaryState) + _, winner := statemgmt.PullResourcesState(ctx, b, alwaysPull, engine.EngineSetting{Type: engine.EngineDirect}) + if tt.wantErr != "" { + assert.True(t, logdiag.HasError(ctx)) + assert.Contains(t, logdiag.GetFirstErrorSummary(ctx), tt.wantErr) + local, err := os.ReadFile(localPath) + require.NoError(t, err) + assert.Equal(t, tt.local, string(local)) + } else { + require.False(t, logdiag.HasError(ctx), logdiag.GetFirstErrorSummary(ctx)) + require.NotNil(t, winner) + if tt.wantAbsent { + assert.NoFileExists(t, localPath) + } else { + assert.Positive(t, transport.reads) + assert.False(t, winner.IsLocal) + local, err := os.ReadFile(localPath) + require.NoError(t, err) + assert.Equal(t, tt.remote, string(local)) + } + } + if tt.wal { + contents, err := os.ReadFile(localPath + stateWALSuffix) + require.NoError(t, err) + assert.Equal(t, wal, string(contents)) + } + }) + } +} diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index c7e43d277e0..96fd7e9197b 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -1,8 +1,10 @@ package statemgmt import ( + "bytes" "context" "errors" + "io" "io/fs" "os" @@ -14,7 +16,7 @@ import ( "github.com/databricks/cli/libs/logdiag" ) -// PushResourcesState uploads the local state file to the remote location. +// PushResourcesState uploads finalized resource state to the remote location. func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { f, err := deploy.StateFiler(ctx, b) if err != nil { @@ -30,21 +32,30 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.Eng remotePath, localPath = b.StateFilenameTerraform(ctx) } - local, err := os.Open(localPath) - if errors.Is(err, fs.ErrNotExist) { - // The state file can be absent if terraform apply is skipped because - // there are no changes to apply in the plan. - log.Debugf(ctx, "Local state file does not exist: %s", localPath) - return - } - if err != nil { - logdiag.LogError(ctx, err) - return + var state io.Reader + if engine.IsDirect() && b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + data := b.DeploymentBundle.StateDB.StateForUpload + if data == nil { + return + } + state = bytes.NewReader(data) + } else { + local, err := os.Open(localPath) + if errors.Is(err, fs.ErrNotExist) { + // The state file can be absent if terraform apply is skipped because + // there are no changes to apply in the plan. + log.Debugf(ctx, "Local state file does not exist: %s", localPath) + return + } + if err != nil { + logdiag.LogError(ctx, err) + return + } + defer local.Close() + state = local } - defer local.Close() - // Upload state file from local cache directory to filer. - err = f.Write(ctx, remotePath, local, filer.CreateParentDirectories, filer.OverwriteIfExists) + err = f.Write(ctx, remotePath, state, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { logdiag.LogError(ctx, err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 0c62196739f..34f658c16fa 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -403,6 +403,10 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("generating a dashboard with --resource is not supported when deployment history is enabled; use --existing-id instead")) + return + } _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index e171df7388a..9cc324b9745 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -321,6 +321,10 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("generating a Genie space with --resource is not supported when deployment history is enabled; use --existing-id instead")) + return + } _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2dcd549eba2..8c4261ef6fb 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -297,7 +297,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{State: stateDesc.Content, DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted }