Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/upload-after-plan.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Delay bundle file and artifact uploads until planning, approval, and any DMS version creation succeed. ([#6704](https://github.com/databricks/cli/pull/6704))
33 changes: 31 additions & 2 deletions acceptance/bin/print_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@
... ]
>>> [x["method"] for x in filter_requests(seq, ["//idx"], True, False, unique=True)]
['GET', 'DELETE', 'GET']

>>> # Periodic lease renewals vary with deployment duration; inspect them explicitly.
>>> heartbeat = {"method": "POST", "path": "/api/2.0/bundle/deployments/123/versions/1/heartbeat"}
>>> complete = {"method": "POST", "path": "/api/2.0/bundle/deployments/123/versions/1/complete"}
>>> filter_requests([heartbeat, complete], [], False, False, include_dms=True) == [complete]
True
>>> filter_requests([heartbeat], [], False, False, include_dms=True, include_heartbeats=True) == [heartbeat]
True
"""

import argparse
Expand Down Expand Up @@ -137,7 +145,14 @@ def read_json_many(s):


def filter_requests(
requests, path_filters, include_get, should_sort, unique=False, method_filter=None, include_dms=False
requests,
path_filters,
include_get,
should_sort,
unique=False,
method_filter=None,
include_dms=False,
include_heartbeats=False,
):
"""Filter requests based on method and path filters."""
positive_filters = []
Expand Down Expand Up @@ -170,6 +185,15 @@ def filter_requests(

# Apply path filters
path = req.get("path", "")
# A slow cloud deployment can renew its lease several times while the local
# fake finishes before the first tick. Heartbeat tests opt in explicitly.
if (
not include_heartbeats
and req.get("method") == "POST"
and path.startswith(DMS_PATH + "/deployments/")
and path.endswith("/heartbeat")
):
continue
should_include = True

# Check positive filters - if any exist, at least one must match (OR logic)
Expand Down Expand Up @@ -222,6 +246,11 @@ def main():
help="Include deployment-history requests (excluded by default; see filter_requests)",
)
parser.add_argument("--keep", action="store_true", help="Keep out.requests.json file after processing")
parser.add_argument(
"--heartbeats",
action="store_true",
help="Include periodic DMS heartbeat requests (use with --dms)",
)
parser.add_argument("--sort", action="store_true", help="Sort requests before output")
parser.add_argument(
"--unique",
Expand Down Expand Up @@ -281,7 +310,7 @@ def main():

requests = read_json_many(data)
filtered_requests = filter_requests(
requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms
requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms, args.heartbeats
)

for req in filtered_requests:
Expand Down
4 changes: 0 additions & 4 deletions acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== Deploy 2 (planning fails, must not leave a WAL)
>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Error: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED)

Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID]
Expand All @@ -18,15 +17,13 @@ API message: Fault injected by test.

Error: planning failed

Files: 0 uploaded, 0 deleted

Exit code: 1

>>> assert_not_exists.py .databricks/bundle/default/resources.json.wal

=== Deploy 3 (planning fails again, must not leave a WAL)
>>> errcode [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Error: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED)

Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID]
Expand All @@ -36,7 +33,6 @@ API message: Fault injected by test.

Error: planning failed

Files: 0 uploaded, 0 deleted

Exit code: 1

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ Run 'bundle deploy' to deploy changes to your workspace
recreate dashboards.dashboard1

Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following dashboards.
This will result in changed IDs and permanent URLs of the dashboards that will be recreated:
Expand All @@ -19,14 +18,13 @@ Error: the deployment requires destructive actions, but the current console does
Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
To proceed, use --auto-approve after reviewing the plan above.

Files: 6 uploaded, 0 deleted

Exit code: 1

>>> [CLI] bundle deployment unbind dashboard1
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created dashboards.dashboard1
Files: 1 uploaded, 0 deleted
Files: 7 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> [CLI] lakeview get [DASHBOARD_ID]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ recreate pipelines.foo
Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the
Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will
Expand All @@ -20,18 +19,17 @@ Error: the deployment requires destructive actions, but the current console does
Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
To proceed, use --auto-approve after reviewing the plan above.

Files: 10 uploaded, 0 deleted

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the
Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will
restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
properties such as the 'catalog' or 'storage' are changed:
recreate resources.pipelines.foo
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files...
Recreated pipelines.foo
Files: 0 uploaded, 0 deleted
Files: 10 uploaded, 0 deleted
Resources: 1 created, 0 changed, 1 deleted, 0 unchanged

>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext
2 changes: 0 additions & 2 deletions acceptance/bundle/dms/declined-deploy/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -107,15 +107,13 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME]

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.foo
Error: the deployment requires destructive actions, but the current console does not support prompting.
Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
To proceed, use --auto-approve after reviewing the plan above.

Files: 2 uploaded, 0 deleted

=== Nothing was recorded for the declined deploy - no version, so none to abort
>>> print_requests.py --dms --get //api/2.0/bundle
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/dms/failed-recreate/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE

>>> musterr [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.foo
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files...
Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE)

Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas
Expand Down
20 changes: 20 additions & 0 deletions acceptance/bundle/dms/file-upload-failure/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
bundle:
name: dms-file-upload-failure
deployment:
lock:
enabled: false

experimental:
deployment_history: true

workspace:
artifact_path: /Workspace/Users/${workspace.current_user.userName}/dms-file-upload-failure-artifacts

permissions:
- user_name: ${workspace.current_user.userName}
level: CAN_MANAGE

resources:
jobs:
foo:
name: foo
3 changes: 3 additions & 0 deletions acceptance/bundle/dms/file-upload-failure/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 36 additions & 0 deletions acceptance/bundle/dms/file-upload-failure/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@

=== An upload failure completes the version without applying resources or opening a WAL
>>> fault.py POST /api/2.0/workspace-files/import-file/* 403 0 1 PERMISSION_DENIED

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-file-upload-failure/default/files...
Error: Failed to update, encountered possible permission error: /Workspace/Users/[USERNAME]/.bundle/dms-file-upload-failure/default/files
Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/dms-file-upload-failure/default/files as [USERNAME]. Cannot apply local deployment permissions.
For assistance, contact the owners of this project.
They can redeploy the project to apply the latest set of permissions.
Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.


>>> assert_not_exists.py .databricks/bundle/default/resources.json.wal

>>> print_requests.py --keep //jobs

>>> print_requests.py --dms //versions --oneline
{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}}

=== Retry succeeds, including permissions on the custom artifact directory
>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-file-upload-failure/default/files...
Created jobs.foo
Created jobs.foo.permissions
Files: 3 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> assert_not_exists.py .databricks/bundle/default/resources.json.wal

>>> print_requests.py --dms //versions --oneline
{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-file-upload-failure/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
11 changes: 11 additions & 0 deletions acceptance/bundle/dms/file-upload-failure/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
title "An upload failure completes the version without applying resources or opening a WAL"
trace fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 PERMISSION_DENIED
trace musterr $CLI bundle deploy
trace assert_not_exists.py .databricks/bundle/default/resources.json.wal
trace print_requests.py --keep //jobs | contains.py "!POST"
trace print_requests.py --dms //versions --oneline

title "Retry succeeds, including permissions on the custom artifact directory"
trace $CLI bundle deploy
trace assert_not_exists.py .databricks/bundle/default/resources.json.wal
trace print_requests.py --dms //versions --oneline
1 change: 1 addition & 0 deletions acceptance/bundle/dms/file-upload-failure/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME]

>>> musterr [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.foo
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files...
Recreated schemas.foo
Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED)

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/dms/successful-recreate/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME]

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.foo
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files...
Recreated schemas.foo
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
Expand Down
6 changes: 2 additions & 4 deletions acceptance/bundle/dms/version-never-created/output.txt
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@

=== The first version fails, so no deployment record exists - only the node naming its ID
>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files...
Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR)

Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1
HTTP Status: 500 Internal Server Error
API error_code: INTERNAL_ERROR
API message: Internal error

Files: 3 uploaded, 0 deleted

>>> print_requests.py --keep //files/ //artifacts/

>>> [CLI] bundle plan -o json
{
Expand Down Expand Up @@ -43,15 +43,13 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged

=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment
>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files...
Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR)

Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1
HTTP Status: 500 Internal Server Error
API error_code: INTERNAL_ERROR
API message: Internal error

Files: 0 uploaded, 0 deleted

>>> print_requests.py --dms //api/2.0/bundle --get --oneline
{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"}
Expand Down
1 change: 1 addition & 0 deletions acceptance/bundle/dms/version-never-created/script
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
title "The first version fails, so no deployment record exists - only the node naming its ID"
trace musterr $CLI bundle deploy
trace print_requests.py --keep //files/ //artifacts/ | contains.py "!POST" "!DELETE"
dms_plan
trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}'
trace print_requests.py --dms //api/2.0/bundle --get --oneline
Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/lifecycle/prevent-destroy/out.direct.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ recreate schemas.my_schema
Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.my_schema
Expand All @@ -41,6 +40,7 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating
restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
properties such as the 'catalog' or 'storage' are changed:
recreate resources.pipelines.my_pipelines
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...
Recreated pipelines.my_pipelines
Recreated schemas.my_schema
Files: 3 uploaded, 0 deleted
Expand All @@ -53,7 +53,6 @@ delete schemas.my_schema
Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
delete resources.schemas.my_schema
Expand All @@ -63,6 +62,7 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating
restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
properties such as the 'catalog' or 'storage' are changed:
delete resources.pipelines.my_pipelines
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...
Deleted pipelines.my_pipelines
Deleted schemas.my_schema
Files: 1 uploaded, 2 deleted
Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ recreate schemas.my_schema
Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
recreate resources.schemas.my_schema
Expand All @@ -125,6 +124,7 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating
restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
properties such as the 'catalog' or 'storage' are changed:
recreate resources.pipelines.my_pipelines
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...
Recreated pipelines.my_pipelines
Recreated schemas.my_schema
Files: 3 uploaded, 0 deleted
Expand All @@ -137,7 +137,6 @@ delete schemas.my_schema
Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...

This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
delete resources.schemas.my_schema
Expand All @@ -147,6 +146,7 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating
restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
properties such as the 'catalog' or 'storage' are changed:
delete resources.pipelines.my_pipelines
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files...
Deleted pipelines.my_pipelines
Deleted schemas.my_schema
Files: 1 uploaded, 2 deleted
Expand Down
Loading
Loading