From a0727ee7621ed1d7a5436969609c81c214d63820 Mon Sep 17 00:00:00 2001
From: "pieter.noordhuis" <pieter.noordhuis@databricks.com>
Date: Wed, 4 Nov 2026 08:59:23 +0000
Subject: [PATCH 1/2] Skip shadow deployment history when the deployment read
 fails

In deployment history shadow mode, a failed read of the recorded
deployment (workspace lookup, GetDeployment, or parsing last_version_id)
only warned and left the deployment ID empty. Deploy and destroy then
took the first-deploy path and called CreateDeployment again, recording
the run against a second deployment for the same bundle.

When the read fails for an unknown reason (403, 5xx, unparseable
last_version_id), skip history for this run through RecordsHistory, so
deploy, destroy and telemetry all see it while the shadow marker and WAL
mode stay unchanged. When the workspace lookup found the deployment, keep
stamping its id on jobs and pipelines so the skipped run does not update
them just to clear it. A node whose record is missing (404) is a
half-created deployment and keeps taking the first-deploy path, which
re-registers it. A destroy that skipped history warns that the record is
kept. A failed delete still fails the apply in a skipped run, as it does
when history is recorded.

DMS-primary state is unchanged: a read failure there already fails the
command.

Co-authored-by: Isaac <no-reply@databricks.com>
---
 .../dms/read-failure/databricks.yml.tmpl      |   9 ++
 .../bundle/dms/read-failure/out.test.toml     |   2 +
 acceptance/bundle/dms/read-failure/output.txt | 133 ++++++++++++++++++
 acceptance/bundle/dms/read-failure/script     |  57 ++++++++
 acceptance/bundle/dms/read-failure/test.toml  |   2 +
 bundle/direct/apply.go                        |   5 +-
 bundle/direct/dstate/state.go                 |  26 ++++
 bundle/direct/dstate/state_test.go            |  22 +++
 bundle/phases/destroy.go                      |   2 +
 cmd/bundle/utils/process.go                   |  31 +++-
 10 files changed, 280 insertions(+), 9 deletions(-)
 create mode 100644 acceptance/bundle/dms/read-failure/databricks.yml.tmpl
 create mode 100644 acceptance/bundle/dms/read-failure/out.test.toml
 create mode 100644 acceptance/bundle/dms/read-failure/output.txt
 create mode 100644 acceptance/bundle/dms/read-failure/script
 create mode 100644 acceptance/bundle/dms/read-failure/test.toml

diff --git a/acceptance/bundle/dms/read-failure/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/databricks.yml.tmpl
new file mode 100644
index 00000000000..81bac3c0344
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/databricks.yml.tmpl
@@ -0,0 +1,9 @@
+bundle:
+  name: dms-read-failure-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo
diff --git a/acceptance/bundle/dms/read-failure/out.test.toml b/acceptance/bundle/dms/read-failure/out.test.toml
new file mode 100644
index 00000000000..88e6fbd847c
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/out.test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]
diff --git a/acceptance/bundle/dms/read-failure/output.txt b/acceptance/bundle/dms/read-failure/output.txt
new file mode 100644
index 00000000000..ceca4a0c597
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/output.txt
@@ -0,0 +1,133 @@
+
+=== Deploy records version 1
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Created jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== A failed deployment read still stamps the deployment id found by the lookup, so the plan shows no change
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle plan
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== A failed deployment read skips history for the deploy: without the deployment's version, recording it would create a second deployment
+>>> update_file.py databricks.yml name: foo name: renamed
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle deploy
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+
+>>> [CLI] jobs get [FOO_ID]
+{
+  "deployment_id": "[DEPLOYMENT_ID]",
+  "kind": "BUNDLE",
+  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json"
+}
+
+=== The next deploy resumes history on the same deployment
+>>> update_file.py databricks.yml name: renamed name: renamed-again
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"renamed-again\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+>>> [CLI] jobs get [FOO_ID]
+{
+  "deployment_id": "[DEPLOYMENT_ID]",
+  "kind": "BUNDLE",
+  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json",
+  "version_id": "3"
+}
+
+=== A deployment node without a record is a half-created deployment: the deploy registers it again and records history
+>>> update_file.py databricks.yml name: renamed-again name: half-created
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 404 0 1 NOT_FOUND
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-read-failure-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default"}}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions", "q": {"version_id": "4"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/4/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NEW_DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"4\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"half-created\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+=== A delete that fails in a run that skips history still fails the deploy and keeps the job in state
+>>> update_file.py databricks.yml name: half-created name: half-created
+    bar:
+      name: bar
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Created jobs.bar
+Files: 1 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> update_file.py databricks.yml name: half-created
+    bar:
+      name: bar name: half-created
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE
+
+>>> musterr [CLI] bundle deploy
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Error: cannot delete resources.jobs.bar: deleting id=[BAR_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Fault injected by test.
+
+Files: 1 uploaded, 0 deleted
+
+>>> [CLI] jobs get [BAR_ID]
+"bar"
+
+>>> [CLI] bundle plan
+delete jobs.bar
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+=== A failed deployment read skips history for destroy as well and keeps the record
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle destroy --auto-approve
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+The following resources will be deleted:
+  delete resources.jobs.bar
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default
+
+Warn: Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it
+Destroy: 2 deleted
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]"}
diff --git a/acceptance/bundle/dms/read-failure/script b/acceptance/bundle/dms/read-failure/script
new file mode 100644
index 00000000000..8355902bdf5
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/script
@@ -0,0 +1,57 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "Deploy records version 1"
+trace $CLI bundle deploy
+replace_ids.py
+job_id=$(read_id.py foo)
+deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
+add_repl "$deployment_id" DEPLOYMENT_ID
+rm -f "$OUT_REQUESTS"
+
+title "A failed deployment read still stamps the deployment id found by the lookup, so the plan shows no change"
+trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
+trace $CLI bundle plan
+rm -f "$OUT_REQUESTS"
+
+title "A failed deployment read skips history for the deploy: without the deployment's version, recording it would create a second deployment"
+trace update_file.py databricks.yml 'name: foo' 'name: renamed'
+trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
+trace $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+trace $CLI jobs get "$job_id" | jq .settings.deployment
+
+title "The next deploy resumes history on the same deployment"
+trace update_file.py databricks.yml 'name: renamed' 'name: renamed-again'
+trace $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+trace $CLI jobs get "$job_id" | jq .settings.deployment
+
+title "A deployment node without a record is a half-created deployment: the deploy registers it again and records history"
+trace update_file.py databricks.yml 'name: renamed-again' 'name: half-created'
+trace fault.py "GET /api/2.0/bundle/deployments/*" 404 0 1 NOT_FOUND
+trace $CLI bundle deploy
+# The fake registers a new node, so the deployment gets a new id.
+new_deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
+add_repl "$new_deployment_id" NEW_DEPLOYMENT_ID
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+
+title "A delete that fails in a run that skips history still fails the deploy and keeps the job in state"
+trace update_file.py databricks.yml 'name: half-created' 'name: half-created
+    bar:
+      name: bar'
+trace $CLI bundle deploy
+bar_id=$(read_id.py bar)
+trace update_file.py databricks.yml 'name: half-created
+    bar:
+      name: bar' 'name: half-created'
+trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
+trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE
+trace musterr $CLI bundle deploy
+trace $CLI jobs get "$bar_id" | jq .settings.name
+trace $CLI bundle plan
+rm -f "$OUT_REQUESTS"
+
+title "A failed deployment read skips history for destroy as well and keeps the record"
+trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
+trace $CLI bundle destroy --auto-approve
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
diff --git a/acceptance/bundle/dms/read-failure/test.toml b/acceptance/bundle/dms/read-failure/test.toml
new file mode 100644
index 00000000000..4cdf348a0a1
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/test.toml
@@ -0,0 +1,2 @@
+# Local only: fault.py registers its rule on the fake server.
+Cloud = false
diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go
index 9cdd5a4f505..4e2e45e9315 100644
--- a/bundle/direct/apply.go
+++ b/bundle/direct/apply.go
@@ -265,8 +265,9 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState,
 			log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err)
 		} else if d.deleteConfirmedGone(ctx, oldID) {
 			log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err)
-		} else if db.RecordsHistory() {
-			// When using the deployment metadata service, record the error.
+		} else if db.RecordsHistory() || db.HistorySkipped() {
+			// When using the deployment metadata service, record the error. A run that skips history
+			// after a failed DMS read still fails here, so a DMS outage does not change delete behavior.
 			err = fmt.Errorf("deleting id=%s: %w", oldID, err)
 			db.RecordFailure(d.ResourceKey, oldID, err)
 			return err
diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go
index 384246913f4..a107a7c2662 100644
--- a/bundle/direct/dstate/state.go
+++ b/bundle/direct/dstate/state.go
@@ -119,6 +119,11 @@ type DeploymentState struct {
 	// resources.json as Serial to ensure outdated resource state is not read.
 	// Open always takes the authoritative value from the service.
 	VersionID int
+
+	// historySkipped turns off history recording for this run without touching the mode, so the
+	// shadow marker is still written. Kept across reset, like mode, so telemetry reads it after
+	// Finalize.
+	historySkipped bool
 }
 
 // OpenDmsArgs identifies the recorded deployment Open reads from. The zero value means the
@@ -475,10 +480,29 @@ func (db *DeploymentState) IsDeploymentMetadataService() bool {
 	return db.isDeploymentMetadataService()
 }
 
+// SkipHistory stops this run from recording deployment history. Shadow mode calls it when the
+// recorded deployment cannot be read: without its id and version, recording would create a second
+// deployment for the bundle.
+func (db *DeploymentState) SkipHistory() {
+	db.mu.Lock()
+	defer db.mu.Unlock()
+	db.historySkipped = true
+}
+
+// HistorySkipped reports whether SkipHistory was called since the state was opened.
+func (db *DeploymentState) HistorySkipped() bool {
+	db.mu.Lock()
+	defer db.mu.Unlock()
+	return db.historySkipped
+}
+
 // RecordsHistory reports whether the loaded file state or selected write mode records DMS history.
 func (db *DeploymentState) RecordsHistory() bool {
 	db.mu.Lock()
 	defer db.mu.Unlock()
+	if db.historySkipped {
+		return false
+	}
 	switch db.mode {
 	case ReadingFile:
 		_, history := db.Data.Features[FeatureDeploymentHistoryShadow]
@@ -570,6 +594,7 @@ func (db *DeploymentState) reset() {
 func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode Mode, withRecovery WithRecovery, dmsDeployment OpenDmsArgs) error {
 	db.Path = path
 	db.mode = mode
+	db.historySkipped = false
 	db.DeploymentID = dmsDeployment.DeploymentID
 	db.VersionID = dmsDeployment.LastVersionID
 
@@ -701,6 +726,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) {
 
 	db.Path = path
 	db.mode = ReadingFile
+	db.historySkipped = false
 	db.Data = data
 	db.stateIDs = make(map[string]string)
 	for key, entry := range data.State {
diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go
index c5bfea2783b..24bcfdb1177 100644
--- a/bundle/direct/dstate/state_test.go
+++ b/bundle/direct/dstate/state_test.go
@@ -89,6 +89,28 @@ func TestFinalizeWithoutResourceWrites(t *testing.T) {
 	}
 }
 
+func TestSkipHistoryKeepsShadowMarker(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	var db DeploymentState
+	require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(true), OpenDmsArgs{}))
+	db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryShadow: {}}
+	require.True(t, db.RecordsHistory())
+
+	db.SkipHistory()
+	assert.False(t, db.RecordsHistory())
+	require.NoError(t, db.UpgradeToWrite())
+	mustFinalize(t, &db)
+	// Telemetry reads it after Finalize.
+	assert.False(t, db.RecordsHistory())
+
+	// The skip covers one run: the marker is persisted, and the next Open records history again.
+	require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{}))
+	assert.False(t, db.HistorySkipped())
+	assert.True(t, db.RecordsHistory())
+	assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryShadow)
+	mustFinalize(t, &db)
+}
+
 func TestDataForFileSerial(t *testing.T) {
 	tests := []struct {
 		name       string
diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go
index cfac15e5930..dde5e434b5c 100644
--- a/bundle/phases/destroy.go
+++ b/bundle/phases/destroy.go
@@ -374,6 +374,8 @@ func Destroy(ctx context.Context, b *bundle.Bundle) {
 				}
 				log.Warnf(ctx, "Could not record deployment history for this destroy: %s", err)
 			}
+		} else if b.DeploymentBundle.StateDB.HistorySkipped() {
+			log.Warnf(ctx, "Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it")
 		}
 		destroyCore(ctx, b, plan)
 	} else {
diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go
index 68798a4bb01..e8a130c4066 100644
--- a/cmd/bundle/utils/process.go
+++ b/cmd/bundle/utils/process.go
@@ -405,7 +405,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
 			var err error
 			dmsDeployment, err = prepareDeploymentHistoryShadow(ctx, b)
 			if err != nil {
-				log.Warnf(ctx, "Could not read deployment history: %s", err)
+				log.Warnf(ctx, "Could not read deployment history, not recording it for this run: %s", err)
 			}
 		}
 
@@ -666,7 +666,9 @@ func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceCl
 		})
 	})
 	if err != nil {
-		return "", nil, err
+		// The id is still returned: the node exists, so the deployment is known even when its record
+		// cannot be read.
+		return deploymentID, nil, err
 	}
 	return deploymentID, deployment, nil
 }
@@ -714,17 +716,32 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st
 	return nil
 }
 
-// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS.
+// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. On a read
+// failure it skips history for this run, still stamping the deployment id when the lookup found it.
 func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) {
+	db := &b.DeploymentBundle.StateDB
 	id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
-	if err != nil {
-		return nil, err
+	// A node without a record is a half-created deployment, which CreateDeployment finalizes, so it
+	// takes the first-deploy path.
+	if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) {
+		id, err = "", nil
+	}
+	var lastVersion int
+	if err == nil {
+		lastVersion, err = parseLastVersionID(deployment)
 	}
-	lastVersion, err := parseLastVersionID(deployment)
 	if err != nil {
+		db.SkipHistory()
+		// Without the stamp the plan would update every job and pipeline just to clear it. The
+		// version is left off: this run records none, and the last one is unknown.
+		if id != "" {
+			bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) {
+				b.Config.Bundle.Deployment.DeploymentID = id
+			})
+			bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(id))
+		}
 		return nil, err
 	}
-	db := &b.DeploymentBundle.StateDB
 	db.DeploymentID, db.VersionID = id, lastVersion
 	bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) {
 		b.Config.Bundle.Deployment.DeploymentID = id

From 5e155785f0205abc61e1ffc61d130364e99d5b24 Mon Sep 17 00:00:00 2001
From: "pieter.noordhuis" <pieter.noordhuis@databricks.com>
Date: Thu, 5 Nov 2026 09:23:08 +0000
Subject: [PATCH 2/2] Split the DMS read-failure acceptance test by scenario

acceptance/bundle/dms/read-failure ran six steps in one script, each
depending on the state the previous one left, so it was hard to tell
which behavior a given output line pinned. Split it into four tests that
each start from their own recorded deployment (setup output goes to
LOG.setup and its requests are drained):

- deploy: plan and deploy with a failed read send only the GET, keep the
  job's deployment_id stamp, and the next deploy resumes on the same
  deployment.
- not-found: a 404 (half-created deployment) re-registers the deployment
  and records history.
- failed-delete: a failed job delete in a run with a failed read still
  fails the deploy and keeps the job in state.
- destroy: a destroy with a failed read records nothing and keeps the
  deployment record.

The shared setup and fault helpers live in read-failure/script.prepare,
and read-failure/test.toml keeps the tests local-only.

Co-authored-by: Isaac <no-reply@databricks.com>
---
 .../{ => deploy}/databricks.yml.tmpl          |   0
 .../read-failure/{ => deploy}/out.test.toml   |   0
 .../bundle/dms/read-failure/deploy/output.txt |  52 +++++++
 .../bundle/dms/read-failure/deploy/script     |  19 +++
 .../read-failure/destroy/databricks.yml.tmpl  |   9 ++
 .../dms/read-failure/destroy/out.test.toml    |   2 +
 .../dms/read-failure/destroy/output.txt       |  20 +++
 .../bundle/dms/read-failure/destroy/script    |  10 ++
 .../failed-delete/databricks.yml.tmpl         |  10 ++
 .../read-failure/failed-delete/out.test.toml  |   2 +
 .../dms/read-failure/failed-delete/output.txt |  30 ++++
 .../dms/read-failure/failed-delete/script     |  11 ++
 .../not-found/databricks.yml.tmpl             |   9 ++
 .../dms/read-failure/not-found/out.test.toml  |   2 +
 .../dms/read-failure/not-found/output.txt     |  18 +++
 .../bundle/dms/read-failure/not-found/script  |   9 ++
 acceptance/bundle/dms/read-failure/output.txt | 133 ------------------
 acceptance/bundle/dms/read-failure/script     |  57 --------
 .../bundle/dms/read-failure/script.prepare    |  25 ++++
 19 files changed, 228 insertions(+), 190 deletions(-)
 rename acceptance/bundle/dms/read-failure/{ => deploy}/databricks.yml.tmpl (100%)
 rename acceptance/bundle/dms/read-failure/{ => deploy}/out.test.toml (100%)
 create mode 100644 acceptance/bundle/dms/read-failure/deploy/output.txt
 create mode 100644 acceptance/bundle/dms/read-failure/deploy/script
 create mode 100644 acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl
 create mode 100644 acceptance/bundle/dms/read-failure/destroy/out.test.toml
 create mode 100644 acceptance/bundle/dms/read-failure/destroy/output.txt
 create mode 100644 acceptance/bundle/dms/read-failure/destroy/script
 create mode 100644 acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl
 create mode 100644 acceptance/bundle/dms/read-failure/failed-delete/out.test.toml
 create mode 100644 acceptance/bundle/dms/read-failure/failed-delete/output.txt
 create mode 100644 acceptance/bundle/dms/read-failure/failed-delete/script
 create mode 100644 acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl
 create mode 100644 acceptance/bundle/dms/read-failure/not-found/out.test.toml
 create mode 100644 acceptance/bundle/dms/read-failure/not-found/output.txt
 create mode 100644 acceptance/bundle/dms/read-failure/not-found/script
 delete mode 100644 acceptance/bundle/dms/read-failure/output.txt
 delete mode 100644 acceptance/bundle/dms/read-failure/script
 create mode 100644 acceptance/bundle/dms/read-failure/script.prepare

diff --git a/acceptance/bundle/dms/read-failure/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/deploy/databricks.yml.tmpl
similarity index 100%
rename from acceptance/bundle/dms/read-failure/databricks.yml.tmpl
rename to acceptance/bundle/dms/read-failure/deploy/databricks.yml.tmpl
diff --git a/acceptance/bundle/dms/read-failure/out.test.toml b/acceptance/bundle/dms/read-failure/deploy/out.test.toml
similarity index 100%
rename from acceptance/bundle/dms/read-failure/out.test.toml
rename to acceptance/bundle/dms/read-failure/deploy/out.test.toml
diff --git a/acceptance/bundle/dms/read-failure/deploy/output.txt b/acceptance/bundle/dms/read-failure/deploy/output.txt
new file mode 100644
index 00000000000..024d28b7cba
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/deploy/output.txt
@@ -0,0 +1,52 @@
+
+=== Plan with a failed read: the job keeps the deployment id the lookup found, so nothing changes
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle plan
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Deploy with a failed read: no history is recorded, so no second deployment is created, and the job keeps its deployment id
+>>> update_file.py databricks.yml name: foo name: renamed
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle deploy
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+
+>>> [CLI] jobs get [FOO_ID]
+{
+  "deployment_id": "[DEPLOYMENT_ID]",
+  "kind": "BUNDLE",
+  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json"
+}
+
+=== The next deploy resumes history on the same deployment and stamps the new version
+>>> update_file.py databricks.yml name: renamed name: resumed
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"resumed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+>>> [CLI] jobs get [FOO_ID]
+{
+  "deployment_id": "[DEPLOYMENT_ID]",
+  "kind": "BUNDLE",
+  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json",
+  "version_id": "3"
+}
diff --git a/acceptance/bundle/dms/read-failure/deploy/script b/acceptance/bundle/dms/read-failure/deploy/script
new file mode 100644
index 00000000000..c4e2f8537d6
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/deploy/script
@@ -0,0 +1,19 @@
+deploy_recorded foo
+
+title "Plan with a failed read: the job keeps the deployment id the lookup found, so nothing changes"
+fail_deployment_read
+trace $CLI bundle plan
+rm -f "$OUT_REQUESTS"
+
+title "Deploy with a failed read: no history is recorded, so no second deployment is created, and the job keeps its deployment id"
+trace update_file.py databricks.yml 'name: foo' 'name: renamed'
+fail_deployment_read
+trace $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+trace $CLI jobs get "$foo_id" | jq .settings.deployment
+
+title "The next deploy resumes history on the same deployment and stamps the new version"
+trace update_file.py databricks.yml 'name: renamed' 'name: resumed'
+trace $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+trace $CLI jobs get "$foo_id" | jq .settings.deployment
diff --git a/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl
new file mode 100644
index 00000000000..81bac3c0344
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl
@@ -0,0 +1,9 @@
+bundle:
+  name: dms-read-failure-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo
diff --git a/acceptance/bundle/dms/read-failure/destroy/out.test.toml b/acceptance/bundle/dms/read-failure/destroy/out.test.toml
new file mode 100644
index 00000000000..88e6fbd847c
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/destroy/out.test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]
diff --git a/acceptance/bundle/dms/read-failure/destroy/output.txt b/acceptance/bundle/dms/read-failure/destroy/output.txt
new file mode 100644
index 00000000000..59396bf8eab
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/destroy/output.txt
@@ -0,0 +1,20 @@
+
+=== Destroy with a failed read: no history is recorded, so no second deployment is created and then deleted
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> [CLI] bundle destroy --auto-approve
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default
+
+Warn: Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it
+Destroy: 1 deleted
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+
+=== The deployment record is kept
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments get-deployment deployments/[DEPLOYMENT_ID] -o json
+{"last_version_id":"1"}
diff --git a/acceptance/bundle/dms/read-failure/destroy/script b/acceptance/bundle/dms/read-failure/destroy/script
new file mode 100644
index 00000000000..1ed991cf3a7
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/destroy/script
@@ -0,0 +1,10 @@
+deploy_recorded
+
+title "Destroy with a failed read: no history is recorded, so no second deployment is created and then deleted"
+fail_deployment_read
+trace $CLI bundle destroy --auto-approve
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+
+title "The deployment record is kept"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments get-deployment "deployments/${deployment_id}" -o json | jq -c '{last_version_id}'
+rm -f "$OUT_REQUESTS"
diff --git a/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl
new file mode 100644
index 00000000000..50ba4e6a742
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl
@@ -0,0 +1,10 @@
+bundle:
+  name: dms-read-failure-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo
+    bar: {name: bar}
diff --git a/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml b/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml
new file mode 100644
index 00000000000..88e6fbd847c
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]
diff --git a/acceptance/bundle/dms/read-failure/failed-delete/output.txt b/acceptance/bundle/dms/read-failure/failed-delete/output.txt
new file mode 100644
index 00000000000..5bce91d4e1f
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/failed-delete/output.txt
@@ -0,0 +1,30 @@
+
+=== A delete that fails in a run with a failed read still fails the deploy and keeps the job in state
+>>> update_file.py databricks.yml bar: {name: bar} 
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
+
+>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE
+
+>>> musterr [CLI] bundle deploy
+Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Error: cannot delete resources.jobs.bar: deleting id=[BAR_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Fault injected by test.
+
+Files: 2 uploaded, 0 deleted
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+
+>>> [CLI] jobs get [BAR_ID]
+"bar"
+
+>>> [CLI] bundle plan
+delete jobs.bar
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
diff --git a/acceptance/bundle/dms/read-failure/failed-delete/script b/acceptance/bundle/dms/read-failure/failed-delete/script
new file mode 100644
index 00000000000..8c4638a8e7a
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/failed-delete/script
@@ -0,0 +1,11 @@
+deploy_recorded bar
+
+title "A delete that fails in a run with a failed read still fails the deploy and keeps the job in state"
+trace update_file.py databricks.yml 'bar: {name: bar}' ''
+fail_deployment_read
+trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE
+trace musterr $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
+trace $CLI jobs get "$bar_id" | jq .settings.name
+trace $CLI bundle plan
+rm -f "$OUT_REQUESTS"
diff --git a/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl
new file mode 100644
index 00000000000..81bac3c0344
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl
@@ -0,0 +1,9 @@
+bundle:
+  name: dms-read-failure-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo
diff --git a/acceptance/bundle/dms/read-failure/not-found/out.test.toml b/acceptance/bundle/dms/read-failure/not-found/out.test.toml
new file mode 100644
index 00000000000..88e6fbd847c
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/not-found/out.test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]
diff --git a/acceptance/bundle/dms/read-failure/not-found/output.txt b/acceptance/bundle/dms/read-failure/not-found/output.txt
new file mode 100644
index 00000000000..91cfcb9ec6c
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/not-found/output.txt
@@ -0,0 +1,18 @@
+
+=== A deployment node without a record (404) is a half-created deployment: the deploy registers it again and records history
+>>> update_file.py databricks.yml name: foo name: renamed
+
+>>> fault.py GET /api/2.0/bundle/deployments/* 404 0 1 NOT_FOUND
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle --get --oneline
+{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
+{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-read-failure-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default"}}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NEW_DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"renamed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
diff --git a/acceptance/bundle/dms/read-failure/not-found/script b/acceptance/bundle/dms/read-failure/not-found/script
new file mode 100644
index 00000000000..9fdf3eb0fa4
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/not-found/script
@@ -0,0 +1,9 @@
+deploy_recorded foo
+
+title "A deployment node without a record (404) is a half-created deployment: the deploy registers it again and records history"
+trace update_file.py databricks.yml 'name: foo' 'name: renamed'
+trace fault.py "GET /api/2.0/bundle/deployments/*" 404 0 1 NOT_FOUND
+trace $CLI bundle deploy
+# The fake registers a new node, so the deployment gets a new id.
+add_repl "$(deployment_node_id)" NEW_DEPLOYMENT_ID
+trace print_requests.py --dms //api/2.0/bundle --get --oneline
diff --git a/acceptance/bundle/dms/read-failure/output.txt b/acceptance/bundle/dms/read-failure/output.txt
deleted file mode 100644
index ceca4a0c597..00000000000
--- a/acceptance/bundle/dms/read-failure/output.txt
+++ /dev/null
@@ -1,133 +0,0 @@
-
-=== Deploy records version 1
->>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Created jobs.foo
-Files: 2 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
-
-=== A failed deployment read still stamps the deployment id found by the lookup, so the plan shows no change
->>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
-
->>> [CLI] bundle plan
-Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
-
-=== A failed deployment read skips history for the deploy: without the deployment's version, recording it would create a second deployment
->>> update_file.py databricks.yml name: foo name: renamed
-
->>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
-
->>> [CLI] bundle deploy
-Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Updated jobs.foo
-Files: 1 uploaded, 0 deleted
-Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --get --oneline
-{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
-
->>> [CLI] jobs get [FOO_ID]
-{
-  "deployment_id": "[DEPLOYMENT_ID]",
-  "kind": "BUNDLE",
-  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json"
-}
-
-=== The next deploy resumes history on the same deployment
->>> update_file.py databricks.yml name: renamed name: renamed-again
-
->>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Updated jobs.foo
-Files: 1 uploaded, 0 deleted
-Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --get --oneline
-{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"renamed-again\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
-
->>> [CLI] jobs get [FOO_ID]
-{
-  "deployment_id": "[DEPLOYMENT_ID]",
-  "kind": "BUNDLE",
-  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json",
-  "version_id": "3"
-}
-
-=== A deployment node without a record is a half-created deployment: the deploy registers it again and records history
->>> update_file.py databricks.yml name: renamed-again name: half-created
-
->>> fault.py GET /api/2.0/bundle/deployments/* 404 0 1 NOT_FOUND
-
->>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Updated jobs.foo
-Files: 1 uploaded, 0 deleted
-Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --get --oneline
-{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"}
-{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-read-failure-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default"}}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions", "q": {"version_id": "4"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/4/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NEW_DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"4\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"half-created\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
-
-=== A delete that fails in a run that skips history still fails the deploy and keeps the job in state
->>> update_file.py databricks.yml name: half-created name: half-created
-    bar:
-      name: bar
-
->>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Created jobs.bar
-Files: 1 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 1 unchanged
-
->>> update_file.py databricks.yml name: half-created
-    bar:
-      name: bar name: half-created
-
->>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
-
->>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE
-
->>> musterr [CLI] bundle deploy
-Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files...
-Error: cannot delete resources.jobs.bar: deleting id=[BAR_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)
-
-Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete
-HTTP Status: 400 Bad Request
-API error_code: INVALID_PARAMETER_VALUE
-API message: Fault injected by test.
-
-Files: 1 uploaded, 0 deleted
-
->>> [CLI] jobs get [BAR_ID]
-"bar"
-
->>> [CLI] bundle plan
-delete jobs.bar
-
-Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
-
-=== A failed deployment read skips history for destroy as well and keeps the record
->>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED
-
->>> [CLI] bundle destroy --auto-approve
-Warn: Could not read deployment history, not recording it for this run: Fault injected by test.
-The following resources will be deleted:
-  delete resources.jobs.bar
-  delete resources.jobs.foo
-
-All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default
-
-Warn: Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it
-Destroy: 2 deleted
-
->>> print_requests.py --dms //api/2.0/bundle --get --oneline
-{"method": "GET", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]"}
diff --git a/acceptance/bundle/dms/read-failure/script b/acceptance/bundle/dms/read-failure/script
deleted file mode 100644
index 8355902bdf5..00000000000
--- a/acceptance/bundle/dms/read-failure/script
+++ /dev/null
@@ -1,57 +0,0 @@
-envsubst < databricks.yml.tmpl > databricks.yml
-
-title "Deploy records version 1"
-trace $CLI bundle deploy
-replace_ids.py
-job_id=$(read_id.py foo)
-deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
-add_repl "$deployment_id" DEPLOYMENT_ID
-rm -f "$OUT_REQUESTS"
-
-title "A failed deployment read still stamps the deployment id found by the lookup, so the plan shows no change"
-trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
-trace $CLI bundle plan
-rm -f "$OUT_REQUESTS"
-
-title "A failed deployment read skips history for the deploy: without the deployment's version, recording it would create a second deployment"
-trace update_file.py databricks.yml 'name: foo' 'name: renamed'
-trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
-trace $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --get --oneline
-trace $CLI jobs get "$job_id" | jq .settings.deployment
-
-title "The next deploy resumes history on the same deployment"
-trace update_file.py databricks.yml 'name: renamed' 'name: renamed-again'
-trace $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --get --oneline
-trace $CLI jobs get "$job_id" | jq .settings.deployment
-
-title "A deployment node without a record is a half-created deployment: the deploy registers it again and records history"
-trace update_file.py databricks.yml 'name: renamed-again' 'name: half-created'
-trace fault.py "GET /api/2.0/bundle/deployments/*" 404 0 1 NOT_FOUND
-trace $CLI bundle deploy
-# The fake registers a new node, so the deployment gets a new id.
-new_deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
-add_repl "$new_deployment_id" NEW_DEPLOYMENT_ID
-trace print_requests.py --dms //api/2.0/bundle --get --oneline
-
-title "A delete that fails in a run that skips history still fails the deploy and keeps the job in state"
-trace update_file.py databricks.yml 'name: half-created' 'name: half-created
-    bar:
-      name: bar'
-trace $CLI bundle deploy
-bar_id=$(read_id.py bar)
-trace update_file.py databricks.yml 'name: half-created
-    bar:
-      name: bar' 'name: half-created'
-trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
-trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE
-trace musterr $CLI bundle deploy
-trace $CLI jobs get "$bar_id" | jq .settings.name
-trace $CLI bundle plan
-rm -f "$OUT_REQUESTS"
-
-title "A failed deployment read skips history for destroy as well and keeps the record"
-trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
-trace $CLI bundle destroy --auto-approve
-trace print_requests.py --dms //api/2.0/bundle --get --oneline
diff --git a/acceptance/bundle/dms/read-failure/script.prepare b/acceptance/bundle/dms/read-failure/script.prepare
new file mode 100644
index 00000000000..66d177ba696
--- /dev/null
+++ b/acceptance/bundle/dms/read-failure/script.prepare
@@ -0,0 +1,25 @@
+# Each test starts from a recorded deployment (version 1). The setup output is in
+# LOG.setup and its requests are drained, so output.txt shows only the step under test.
+# Sets deployment_id, and <name>_id for each job name passed.
+deploy_recorded() {
+    envsubst < databricks.yml.tmpl > databricks.yml
+    $CLI bundle deploy &> LOG.setup
+    deployment_id=$(deployment_node_id)
+    add_repl "$deployment_id" DEPLOYMENT_ID
+    local name
+    for name in "$@"; do
+        printf -v "${name}_id" '%s' "$(read_id.py "$name")"
+    done
+    rm -f "$OUT_REQUESTS"
+}
+
+# The deployment id is the id of the deployment's workspace node. Extracted with
+# python, not jq: the id exceeds 2^53 and jq 1.6 rounds it.
+deployment_node_id() {
+    MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])'
+}
+
+# A 403 is a read failure of unknown cause, and it is not retried.
+fail_deployment_read() {
+    trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED
+}