diff --git a/acceptance/bundle/dms/read-failure/deploy/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..81bac3c0344 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/deploy/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-read-failure-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/read-failure/deploy/out.test.toml b/acceptance/bundle/dms/read-failure/deploy/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/read-failure/deploy/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/read-failure/deploy/output.txt b/acceptance/bundle/dms/read-failure/deploy/output.txt new file mode 100644 index 00000000000..024d28b7cba --- /dev/null +++ b/acceptance/bundle/dms/read-failure/deploy/output.txt @@ -0,0 +1,52 @@ + +=== Plan with a failed read: the job keeps the deployment id the lookup found, so nothing changes +>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED + +>>> [CLI] bundle plan +Warn: Could not read deployment history, not recording it for this run: Fault injected by test. +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Deploy with a failed read: no history is recorded, so no second deployment is created, and the job keeps its deployment id +>>> update_file.py databricks.yml name: foo name: renamed + +>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED + +>>> [CLI] bundle deploy +Warn: Could not read deployment history, not recording it for this run: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} + +>>> [CLI] jobs get [FOO_ID] +{ + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json" +} + +=== The next deploy resumes history on the same deployment and stamps the new version +>>> update_file.py databricks.yml name: renamed name: resumed + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"resumed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> [CLI] jobs get [FOO_ID] +{ + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "3" +} diff --git a/acceptance/bundle/dms/read-failure/deploy/script b/acceptance/bundle/dms/read-failure/deploy/script new file mode 100644 index 00000000000..c4e2f8537d6 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/deploy/script @@ -0,0 +1,19 @@ +deploy_recorded foo + +title "Plan with a failed read: the job keeps the deployment id the lookup found, so nothing changes" +fail_deployment_read +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" + +title "Deploy with a failed read: no history is recorded, so no second deployment is created, and the job keeps its deployment id" +trace update_file.py databricks.yml 'name: foo' 'name: renamed' +fail_deployment_read +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline +trace $CLI jobs get "$foo_id" | jq .settings.deployment + +title "The next deploy resumes history on the same deployment and stamps the new version" +trace update_file.py databricks.yml 'name: renamed' 'name: resumed' +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline +trace $CLI jobs get "$foo_id" | jq .settings.deployment diff --git a/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl new file mode 100644 index 00000000000..81bac3c0344 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/destroy/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-read-failure-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/read-failure/destroy/out.test.toml b/acceptance/bundle/dms/read-failure/destroy/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/read-failure/destroy/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/read-failure/destroy/output.txt b/acceptance/bundle/dms/read-failure/destroy/output.txt new file mode 100644 index 00000000000..59396bf8eab --- /dev/null +++ b/acceptance/bundle/dms/read-failure/destroy/output.txt @@ -0,0 +1,20 @@ + +=== Destroy with a failed read: no history is recorded, so no second deployment is created and then deleted +>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED + +>>> [CLI] bundle destroy --auto-approve +Warn: Could not read deployment history, not recording it for this run: Fault injected by test. +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default + +Warn: Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} + +=== The deployment record is kept +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments get-deployment deployments/[DEPLOYMENT_ID] -o json +{"last_version_id":"1"} diff --git a/acceptance/bundle/dms/read-failure/destroy/script b/acceptance/bundle/dms/read-failure/destroy/script new file mode 100644 index 00000000000..1ed991cf3a7 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/destroy/script @@ -0,0 +1,10 @@ +deploy_recorded + +title "Destroy with a failed read: no history is recorded, so no second deployment is created and then deleted" +fail_deployment_read +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --get --oneline + +title "The deployment record is kept" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments get-deployment "deployments/${deployment_id}" -o json | jq -c '{last_version_id}' +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl new file mode 100644 index 00000000000..50ba4e6a742 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/failed-delete/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-read-failure-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo + bar: {name: bar} diff --git a/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml b/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/read-failure/failed-delete/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/read-failure/failed-delete/output.txt b/acceptance/bundle/dms/read-failure/failed-delete/output.txt new file mode 100644 index 00000000000..5bce91d4e1f --- /dev/null +++ b/acceptance/bundle/dms/read-failure/failed-delete/output.txt @@ -0,0 +1,30 @@ + +=== A delete that fails in a run with a failed read still fails the deploy and keeps the job in state +>>> update_file.py databricks.yml bar: {name: bar} + +>>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED + +>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy +Warn: Could not read deployment history, not recording it for this run: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files... +Error: cannot delete resources.jobs.bar: deleting id=[BAR_ID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} + +>>> [CLI] jobs get [BAR_ID] +"bar" + +>>> [CLI] bundle plan +delete jobs.bar + +Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged diff --git a/acceptance/bundle/dms/read-failure/failed-delete/script b/acceptance/bundle/dms/read-failure/failed-delete/script new file mode 100644 index 00000000000..8c4638a8e7a --- /dev/null +++ b/acceptance/bundle/dms/read-failure/failed-delete/script @@ -0,0 +1,11 @@ +deploy_recorded bar + +title "A delete that fails in a run with a failed read still fails the deploy and keeps the job in state" +trace update_file.py databricks.yml 'bar: {name: bar}' '' +fail_deployment_read +trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline +trace $CLI jobs get "$bar_id" | jq .settings.name +trace $CLI bundle plan +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl b/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl new file mode 100644 index 00000000000..81bac3c0344 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/not-found/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-read-failure-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/read-failure/not-found/out.test.toml b/acceptance/bundle/dms/read-failure/not-found/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/read-failure/not-found/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/read-failure/not-found/output.txt b/acceptance/bundle/dms/read-failure/not-found/output.txt new file mode 100644 index 00000000000..91cfcb9ec6c --- /dev/null +++ b/acceptance/bundle/dms/read-failure/not-found/output.txt @@ -0,0 +1,18 @@ + +=== A deployment node without a record (404) is a half-created deployment: the deploy registers it again and records history +>>> update_file.py databricks.yml name: foo name: renamed + +>>> fault.py GET /api/2.0/bundle/deployments/* 404 0 1 NOT_FOUND + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-read-failure-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[FOO_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NEW_DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-read-failure-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"renamed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NEW_DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/read-failure/not-found/script b/acceptance/bundle/dms/read-failure/not-found/script new file mode 100644 index 00000000000..9fdf3eb0fa4 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/not-found/script @@ -0,0 +1,9 @@ +deploy_recorded foo + +title "A deployment node without a record (404) is a half-created deployment: the deploy registers it again and records history" +trace update_file.py databricks.yml 'name: foo' 'name: renamed' +trace fault.py "GET /api/2.0/bundle/deployments/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +# The fake registers a new node, so the deployment gets a new id. +add_repl "$(deployment_node_id)" NEW_DEPLOYMENT_ID +trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/read-failure/script.prepare b/acceptance/bundle/dms/read-failure/script.prepare new file mode 100644 index 00000000000..66d177ba696 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/script.prepare @@ -0,0 +1,25 @@ +# Each test starts from a recorded deployment (version 1). The setup output is in +# LOG.setup and its requests are drained, so output.txt shows only the step under test. +# Sets deployment_id, and _id for each job name passed. +deploy_recorded() { + envsubst < databricks.yml.tmpl > databricks.yml + $CLI bundle deploy &> LOG.setup + deployment_id=$(deployment_node_id) + add_repl "$deployment_id" DEPLOYMENT_ID + local name + for name in "$@"; do + printf -v "${name}_id" '%s' "$(read_id.py "$name")" + done + rm -f "$OUT_REQUESTS" +} + +# The deployment id is the id of the deployment's workspace node. Extracted with +# python, not jq: the id exceeds 2^53 and jq 1.6 rounds it. +deployment_node_id() { + MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-read-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])' +} + +# A 403 is a read failure of unknown cause, and it is not retried. +fail_deployment_read() { + trace fault.py "GET /api/2.0/bundle/deployments/*" 403 0 1 PERMISSION_DENIED +} diff --git a/acceptance/bundle/dms/read-failure/test.toml b/acceptance/bundle/dms/read-failure/test.toml new file mode 100644 index 00000000000..4cdf348a0a1 --- /dev/null +++ b/acceptance/bundle/dms/read-failure/test.toml @@ -0,0 +1,2 @@ +# Local only: fault.py registers its rule on the fake server. +Cloud = false diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 9cdd5a4f505..4e2e45e9315 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -265,8 +265,9 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else if db.RecordsHistory() { - // When using the deployment metadata service, record the error. + } else if db.RecordsHistory() || db.HistorySkipped() { + // When using the deployment metadata service, record the error. A run that skips history + // after a failed DMS read still fails here, so a DMS outage does not change delete behavior. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) return err diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 384246913f4..a107a7c2662 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -119,6 +119,11 @@ type DeploymentState struct { // resources.json as Serial to ensure outdated resource state is not read. // Open always takes the authoritative value from the service. VersionID int + + // historySkipped turns off history recording for this run without touching the mode, so the + // shadow marker is still written. Kept across reset, like mode, so telemetry reads it after + // Finalize. + historySkipped bool } // OpenDmsArgs identifies the recorded deployment Open reads from. The zero value means the @@ -475,10 +480,29 @@ func (db *DeploymentState) IsDeploymentMetadataService() bool { return db.isDeploymentMetadataService() } +// SkipHistory stops this run from recording deployment history. Shadow mode calls it when the +// recorded deployment cannot be read: without its id and version, recording would create a second +// deployment for the bundle. +func (db *DeploymentState) SkipHistory() { + db.mu.Lock() + defer db.mu.Unlock() + db.historySkipped = true +} + +// HistorySkipped reports whether SkipHistory was called since the state was opened. +func (db *DeploymentState) HistorySkipped() bool { + db.mu.Lock() + defer db.mu.Unlock() + return db.historySkipped +} + // RecordsHistory reports whether the loaded file state or selected write mode records DMS history. func (db *DeploymentState) RecordsHistory() bool { db.mu.Lock() defer db.mu.Unlock() + if db.historySkipped { + return false + } switch db.mode { case ReadingFile: _, history := db.Data.Features[FeatureDeploymentHistoryShadow] @@ -570,6 +594,7 @@ func (db *DeploymentState) reset() { func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode Mode, withRecovery WithRecovery, dmsDeployment OpenDmsArgs) error { db.Path = path db.mode = mode + db.historySkipped = false db.DeploymentID = dmsDeployment.DeploymentID db.VersionID = dmsDeployment.LastVersionID @@ -701,6 +726,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { db.Path = path db.mode = ReadingFile + db.historySkipped = false db.Data = data db.stateIDs = make(map[string]string) for key, entry := range data.State { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index c5bfea2783b..24bcfdb1177 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -89,6 +89,28 @@ func TestFinalizeWithoutResourceWrites(t *testing.T) { } } +func TestSkipHistoryKeepsShadowMarker(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(true), OpenDmsArgs{})) + db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryShadow: {}} + require.True(t, db.RecordsHistory()) + + db.SkipHistory() + assert.False(t, db.RecordsHistory()) + require.NoError(t, db.UpgradeToWrite()) + mustFinalize(t, &db) + // Telemetry reads it after Finalize. + assert.False(t, db.RecordsHistory()) + + // The skip covers one run: the marker is persisted, and the next Open records history again. + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.False(t, db.HistorySkipped()) + assert.True(t, db.RecordsHistory()) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryShadow) + mustFinalize(t, &db) +} + func TestDataForFileSerial(t *testing.T) { tests := []struct { name string diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index cfac15e5930..dde5e434b5c 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -374,6 +374,8 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { } log.Warnf(ctx, "Could not record deployment history for this destroy: %s", err) } + } else if b.DeploymentBundle.StateDB.HistorySkipped() { + log.Warnf(ctx, "Keeping the deployment history record, if any: it could not be read, so this destroy does not delete it") } destroyCore(ctx, b, plan) } else { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 68798a4bb01..e8a130c4066 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -405,7 +405,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle var err error dmsDeployment, err = prepareDeploymentHistoryShadow(ctx, b) if err != nil { - log.Warnf(ctx, "Could not read deployment history: %s", err) + log.Warnf(ctx, "Could not read deployment history, not recording it for this run: %s", err) } } @@ -666,7 +666,9 @@ func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceCl }) }) if err != nil { - return "", nil, err + // The id is still returned: the node exists, so the deployment is known even when its record + // cannot be read. + return deploymentID, nil, err } return deploymentID, deployment, nil } @@ -714,17 +716,32 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return nil } -// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. +// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. On a read +// failure it skips history for this run, still stamping the deployment id when the lookup found it. func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { + db := &b.DeploymentBundle.StateDB id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) - if err != nil { - return nil, err + // A node without a record is a half-created deployment, which CreateDeployment finalizes, so it + // takes the first-deploy path. + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + id, err = "", nil + } + var lastVersion int + if err == nil { + lastVersion, err = parseLastVersionID(deployment) } - lastVersion, err := parseLastVersionID(deployment) if err != nil { + db.SkipHistory() + // Without the stamp the plan would update every job and pipeline just to clear it. The + // version is left off: this run records none, and the last one is unknown. + if id != "" { + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.DeploymentID = id + }) + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(id)) + } return nil, err } - db := &b.DeploymentBundle.StateDB db.DeploymentID, db.VersionID = id, lastVersion bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { b.Config.Bundle.Deployment.DeploymentID = id