From bd463dd84a442e912680ee5e851c344f1a9c64ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 16:46:56 +0200 Subject: [PATCH 1/3] acceptance: terraform-path references survive auto-migration Replay a terraform deployment whose config references other resources by terraform field paths, then check that the direct engine resolves them to the deployed values after auto-migration: clean plan, no changes on deploy, resolved names in state. Co-authored-by: Isaac --- .../auto/tf-path-references/databricks.yml | 38 + .../auto/tf-path-references/out.test.toml | 2 + .../auto/tf-path-references/output.txt | 45 ++ .../migrate/auto/tf-path-references/script | 25 + .../terraform-requests.json | 146 ++++ .../auto/tf-path-references/terraform.tfstate | 691 ++++++++++++++++++ 6 files changed, 947 insertions(+) create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/databricks.yml create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/out.test.toml create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/output.txt create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/script create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json create mode 100644 acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml b/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml new file mode 100644 index 00000000000..ef6cf213871 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml @@ -0,0 +1,38 @@ +bundle: + name: test-bundle + +resources: + pipelines: + src_pipeline: + name: source-pipeline + clusters: + - label: mycluster + num_workers: 1 + + jobs: + src_git: + name: src-git + git_source: + git_branch: main + git_url: https://github.com/databricks/cli + git_provider: gitHub + dst_git: + # "git_source[0].branch": git_source is a TF list-block ([0] required), + # "branch" is the TF field name for DABs "git_branch". + name: ${resources.jobs.src_git.git_source[0].branch} + + src_tasks: + name: src-tasks + tasks: + - task_key: main + libraries: + - whl: /path/to/wheel.whl + notebook_task: + notebook_path: /Workspace/notebook + dst_tasks: + # "task[0].library[0].whl": "task" → "tasks", "library" → "libraries" (two renames). + name: ${resources.jobs.src_tasks.task[0].library[0].whl} + + dst_pipeline: + # "cluster[0].label": "cluster" is TF name for DABs "clusters" on pipelines. + name: ${resources.pipelines.src_pipeline.cluster[0].label} diff --git a/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml b/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/tf-path-references/output.txt b/acceptance/bundle/migrate/auto/tf-path-references/output.txt new file mode 100644 index 00000000000..e2265a5da8e --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/output.txt @@ -0,0 +1,45 @@ + +=== Replay a prior terraform deployment +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Plan migrates in memory: the terraform-path references resolve to the deployed values +>>> [CLI] bundle plan +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged + +=== Deploy commits the migration without changing any resource +>>> [CLI] bundle deploy +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Migrated 6 resources to direct deployment engine. +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 6 unchanged + +=== Migrated state keeps the resolved names +>>> print_state.py +resources.jobs.dst_git main +resources.jobs.dst_pipeline mycluster +resources.jobs.dst_tasks /path/to/wheel.whl + +=== Redeploy is a no-op on direct +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 6 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.dst_git + delete resources.jobs.dst_pipeline + delete resources.jobs.dst_tasks + delete resources.jobs.src_git + delete resources.jobs.src_tasks + delete resources.pipelines.src_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.src_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 6 deleted diff --git a/acceptance/bundle/migrate/auto/tf-path-references/script b/acceptance/bundle/migrate/auto/tf-path-references/script new file mode 100644 index 00000000000..9e4861578b7 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/script @@ -0,0 +1,25 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# The config references other resources by Terraform field paths (see +# resource_deps/tf_path_renames). Terraform resolved them on the original deploy; after +# migration the direct engine must resolve them to the same values. +title "Replay a prior terraform deployment" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +rm -f "$OUT_REQUESTS" + +title "Plan migrates in memory: the terraform-path references resolve to the deployed values" +trace $CLI bundle plan + +title "Deploy commits the migration without changing any resource" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Migrated state keeps the resolved names" +trace print_state.py | jq -r '.state | to_entries[] | select(.key | startswith("resources.jobs.dst")) | "\(.key) \(.value.state.name)"' + +title "Redeploy is a no-op on direct" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json b/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json new file mode 100644 index 00000000000..3eacf36b0d5 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json @@ -0,0 +1,146 @@ +[ + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/artifacts/.internal" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files" + } + }, + { + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "clusters": [ + { + "label": "mycluster", + "num_workers": 1 + } + ], + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "source-pipeline" + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "git_source": { + "git_branch": "main", + "git_provider": "gitHub", + "git_url": "https://github.com/databricks/cli" + }, + "max_concurrent_runs": 1, + "name": "src-git", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "src-tasks", + "queue": { + "enabled": true + }, + "tasks": [ + { + "libraries": [ + { + "whl": "/path/to/wheel.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/notebook", + "source": "WORKSPACE" + }, + "task_key": "main" + } + ] + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "mycluster", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "main", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "/path/to/wheel.whl", + "queue": { + "enabled": true + } + } + } +] diff --git a/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate b/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate new file mode 100644 index 00000000000..3a1869177a1 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate @@ -0,0 +1,691 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 7, + "lineage": "b76425ce-df50-075e-317c-8d4f7d38b839", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "dst_git", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791211499496624000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "main", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791211499496624000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_job.src_git" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "dst_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791211499480065000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "mycluster", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791211499480065000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_pipeline.src_pipeline" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "dst_tasks", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791211499497748000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "/path/to/wheel.whl", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791211499497748000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_job.src_tasks" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "src_git", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [ + { + "branch": "main", + "commit": "", + "git_snapshot": [], + "job_source": [], + "provider": "gitHub", + "sparse_checkout": [], + "tag": "", + "url": "https://github.com/databricks/cli" + } + ], + "health": [], + "id": "8791211499447989000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "src-git", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791211499447989000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "src_tasks", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791211499448751000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "src-tasks", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [ + { + "ai_runtime_task": [], + "alert_task": [], + "clean_rooms_notebook_task": [], + "compute": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "", + "library": [ + { + "cran": [], + "egg": "", + "jar": "", + "maven": [], + "provider_config": [], + "pypi": [], + "requirements": "", + "whl": "/path/to/wheel.whl" + } + ], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [ + { + "base_parameters": null, + "notebook_path": "/Workspace/notebook", + "source": "WORKSPACE", + "warehouse_id": "" + } + ], + "notification_settings": [], + "pipeline_task": [], + "power_bi_task": [], + "python_operator_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "ALL_SUCCESS", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "main", + "timeout_seconds": 0, + "webhook_notifications": [] + } + ], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791211499448751000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "src_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": null, + "cause": null, + "channel": "CURRENT", + "cluster": [ + { + "apply_policy_default_values": false, + "autoscale": [], + "aws_attributes": [], + "azure_attributes": [], + "cluster_log_conf": [], + "custom_tags": null, + "driver_instance_pool_id": "", + "driver_node_type_id": "", + "enable_local_disk_encryption": false, + "gcp_attributes": [], + "init_scripts": [], + "instance_pool_id": "", + "label": "mycluster", + "node_type_id": "", + "num_workers": 1, + "policy_id": "", + "spark_conf": null, + "spark_env_vars": null, + "ssh_public_keys": null + } + ], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "tester@databricks.com", + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "development": false, + "edition": "ADVANCED", + "environment": [], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "7a00a847-e948-2520-7a00-a847e9482521", + "ingestion_definition": [], + "last_modified": 1791211499437, + "latest_updates": null, + "library": [], + "name": "source-pipeline", + "notification": [], + "photon": false, + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "restart_window": [], + "root_path": null, + "run_as": [], + "run_as_user_name": "tester@databricks.com", + "schema": null, + "serverless": false, + "serverless_compute_id": null, + "state": "IDLE", + "storage": "dbfs:/pipelines/7a00a847-e948-2520-7a00-a847e9482521", + "tags": null, + "target": null, + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/7a00a847-e948-2520-7a00-a847e9482521", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} From e58e2c351eb7c9b2190672fa0068c65290fdbf57 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 16:56:00 +0200 Subject: [PATCH 2/3] acceptance: record that terraform resource-type references fail on direct Bundles referencing resources as ${databricks_....} deployed on terraform, but the direct engine only supports terraform field paths inside ${resources.*}: plan and deploy fail, both directly and after auto-migration from a replayed terraform deployment. Co-authored-by: Isaac --- .../databricks.yml | 17 ++ .../tf-resource-type-references/out.test.toml | 2 + .../tf-resource-type-references/output.txt | 32 +++ .../auto/tf-resource-type-references/script | 18 ++ .../terraform-requests.json | 61 +++++ .../terraform.tfstate | 244 ++++++++++++++++++ .../tf_resource_type_ref/databricks.yml | 17 ++ .../tf_resource_type_ref/out.test.toml | 2 + .../tf_resource_type_ref/output.txt | 19 ++ .../resource_deps/tf_resource_type_ref/script | 6 + 10 files changed, 418 insertions(+) create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/script create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json create mode 100644 acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate create mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml create mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml create mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt create mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/script diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml b/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml new file mode 100644 index 00000000000..6d91f6515bd --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + pipelines: + src_pipeline: + name: source-pipeline + + jobs: + dst_job: + name: dst-job + tasks: + - task_key: main + pipeline_task: + # Terraform-style resource reference: databricks_pipeline is the terraform + # resource type for DABs "pipelines". + pipeline_id: ${databricks_pipeline.src_pipeline.id} diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml b/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt new file mode 100644 index 00000000000..45557fcc89f --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt @@ -0,0 +1,32 @@ + +=== Replay a prior terraform deployment +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Plan after auto-migration +>>> musterr [CLI] bundle plan +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + + +=== Deploy after auto-migration +>>> musterr [CLI] bundle deploy +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + +Files: 1 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +The following resources will be deleted: + delete resources.jobs.dst_job + delete resources.pipelines.src_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.src_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Migrated 2 resources to direct deployment engine. +Destroy: 2 deleted diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/script b/acceptance/bundle/migrate/auto/tf-resource-type-references/script new file mode 100644 index 00000000000..548dd21a149 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/script @@ -0,0 +1,18 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# The config references the pipeline by its terraform resource type (${databricks_pipeline...}), +# which terraform resolved on the original deploy. The direct engine does not support this syntax +# (only terraform field paths inside ${resources.*}), so after auto-migration plan and deploy fail. +title "Replay a prior terraform deployment" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +rm -f "$OUT_REQUESTS" + +title "Plan after auto-migration" +trace musterr $CLI bundle plan + +title "Deploy after auto-migration" +trace musterr $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json new file mode 100644 index 00000000000..b638e9f92d5 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json @@ -0,0 +1,61 @@ +[ + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/artifacts/.internal" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files" + } + }, + { + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "source-pipeline" + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "dst-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "pipeline_task": { + "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9" + }, + "task_key": "main" + } + ] + } + } +] diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate new file mode 100644 index 00000000000..564391bbe5a --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate @@ -0,0 +1,244 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 3, + "lineage": "feddf884-7b6a-3c90-968d-3dcc4761437e", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "dst_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791212044006401000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "dst-job", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [ + { + "ai_runtime_task": [], + "alert_task": [], + "clean_rooms_notebook_task": [], + "compute": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "", + "library": [], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "pipeline_task": [ + { + "full_refresh": false, + "full_refresh_selection": null, + "parameters": null, + "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "refresh_flow_selection": null, + "refresh_selection": null, + "reset_checkpoint_selection": null + } + ], + "power_bi_task": [], + "python_operator_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "ALL_SUCCESS", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "main", + "timeout_seconds": 0, + "webhook_notifications": [] + } + ], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791212044006401000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_pipeline.src_pipeline" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "src_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": null, + "cause": null, + "channel": "CURRENT", + "cluster": [], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "tester@databricks.com", + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "development": false, + "edition": "ADVANCED", + "environment": [], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "ingestion_definition": [], + "last_modified": 1791212043963, + "latest_updates": null, + "library": [], + "name": "source-pipeline", + "notification": [], + "photon": false, + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "restart_window": [], + "root_path": null, + "run_as": [], + "run_as_user_name": "tester@databricks.com", + "schema": null, + "serverless": false, + "serverless_compute_id": null, + "state": "IDLE", + "storage": "dbfs:/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "tags": null, + "target": null, + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml b/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml new file mode 100644 index 00000000000..6d91f6515bd --- /dev/null +++ b/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + pipelines: + src_pipeline: + name: source-pipeline + + jobs: + dst_job: + name: dst-job + tasks: + - task_key: main + pipeline_task: + # Terraform-style resource reference: databricks_pipeline is the terraform + # resource type for DABs "pipelines". + pipeline_id: ${databricks_pipeline.src_pipeline.id} diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml b/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt b/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt new file mode 100644 index 00000000000..5da823336ce --- /dev/null +++ b/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! + +>>> musterr [CLI] bundle plan +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/script b/acceptance/bundle/resource_deps/tf_resource_type_ref/script new file mode 100644 index 00000000000..b1f297e2cf1 --- /dev/null +++ b/acceptance/bundle/resource_deps/tf_resource_type_ref/script @@ -0,0 +1,6 @@ +# ${databricks_....} references (terraform resource types) are not supported on the +# direct engine, unlike terraform field paths inside ${resources.*} (see tf_path_renames). +# Validation does not flag them; plan and deploy fail. +trace $CLI bundle validate +trace musterr $CLI bundle plan +trace musterr $CLI bundle deploy From 999c1d5f07481e291d618f3995778c72938054ca Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 17:13:42 +0200 Subject: [PATCH 3/3] acceptance: keep only the terraform resource-type migration test tf-path-references duplicates resource_deps/tf_path_renames and tf_resource_type_ref fails the same way as the migration test; validate is now part of the migration test. Co-authored-by: Isaac --- .../auto/tf-path-references/databricks.yml | 38 - .../auto/tf-path-references/out.test.toml | 2 - .../auto/tf-path-references/output.txt | 45 -- .../migrate/auto/tf-path-references/script | 25 - .../terraform-requests.json | 146 ---- .../auto/tf-path-references/terraform.tfstate | 691 ------------------ .../tf-resource-type-references/output.txt | 10 + .../auto/tf-resource-type-references/script | 3 + .../tf_resource_type_ref/databricks.yml | 17 - .../tf_resource_type_ref/out.test.toml | 2 - .../tf_resource_type_ref/output.txt | 19 - .../resource_deps/tf_resource_type_ref/script | 6 - 12 files changed, 13 insertions(+), 991 deletions(-) delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/databricks.yml delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/out.test.toml delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/output.txt delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/script delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json delete mode 100644 acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate delete mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml delete mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml delete mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt delete mode 100644 acceptance/bundle/resource_deps/tf_resource_type_ref/script diff --git a/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml b/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml deleted file mode 100644 index ef6cf213871..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/databricks.yml +++ /dev/null @@ -1,38 +0,0 @@ -bundle: - name: test-bundle - -resources: - pipelines: - src_pipeline: - name: source-pipeline - clusters: - - label: mycluster - num_workers: 1 - - jobs: - src_git: - name: src-git - git_source: - git_branch: main - git_url: https://github.com/databricks/cli - git_provider: gitHub - dst_git: - # "git_source[0].branch": git_source is a TF list-block ([0] required), - # "branch" is the TF field name for DABs "git_branch". - name: ${resources.jobs.src_git.git_source[0].branch} - - src_tasks: - name: src-tasks - tasks: - - task_key: main - libraries: - - whl: /path/to/wheel.whl - notebook_task: - notebook_path: /Workspace/notebook - dst_tasks: - # "task[0].library[0].whl": "task" → "tasks", "library" → "libraries" (two renames). - name: ${resources.jobs.src_tasks.task[0].library[0].whl} - - dst_pipeline: - # "cluster[0].label": "cluster" is TF name for DABs "clusters" on pipelines. - name: ${resources.pipelines.src_pipeline.cluster[0].label} diff --git a/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml b/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml deleted file mode 100644 index 19b5b15d11f..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/tf-path-references/output.txt b/acceptance/bundle/migrate/auto/tf-path-references/output.txt deleted file mode 100644 index e2265a5da8e..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/output.txt +++ /dev/null @@ -1,45 +0,0 @@ - -=== Replay a prior terraform deployment ->>> replay_tfstate.py terraform-requests.json terraform.tfstate - -=== Plan migrates in memory: the terraform-path references resolve to the deployed values ->>> [CLI] bundle plan -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged - -=== Deploy commits the migration without changing any resource ->>> [CLI] bundle deploy -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Migrated 6 resources to direct deployment engine. -Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 6 unchanged - -=== Migrated state keeps the resolved names ->>> print_state.py -resources.jobs.dst_git main -resources.jobs.dst_pipeline mycluster -resources.jobs.dst_tasks /path/to/wheel.whl - -=== Redeploy is a no-op on direct ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 6 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.dst_git - delete resources.jobs.dst_pipeline - delete resources.jobs.dst_tasks - delete resources.jobs.src_git - delete resources.jobs.src_tasks - delete resources.pipelines.src_pipeline - -This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: - delete resources.pipelines.src_pipeline - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 6 deleted diff --git a/acceptance/bundle/migrate/auto/tf-path-references/script b/acceptance/bundle/migrate/auto/tf-path-references/script deleted file mode 100644 index 9e4861578b7..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/script +++ /dev/null @@ -1,25 +0,0 @@ -export DATABRICKS_BUNDLE_ENGINE= - -# The config references other resources by Terraform field paths (see -# resource_deps/tf_path_renames). Terraform resolved them on the original deploy; after -# migration the direct engine must resolve them to the same values. -title "Replay a prior terraform deployment" -trace replay_tfstate.py terraform-requests.json terraform.tfstate -rm -f "$OUT_REQUESTS" - -title "Plan migrates in memory: the terraform-path references resolve to the deployed values" -trace $CLI bundle plan - -title "Deploy commits the migration without changing any resource" -trace $CLI bundle deploy -rm -f "$OUT_REQUESTS" - -title "Migrated state keeps the resolved names" -trace print_state.py | jq -r '.state | to_entries[] | select(.key | startswith("resources.jobs.dst")) | "\(.key) \(.value.state.name)"' - -title "Redeploy is a no-op on direct" -trace $CLI bundle deploy -rm -f "$OUT_REQUESTS" - -trace $CLI bundle destroy --auto-approve -rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json b/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json deleted file mode 100644 index 3eacf36b0d5..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/terraform-requests.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state" - } - }, - { - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/artifacts/.internal" - } - }, - { - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files" - } - }, - { - "method": "POST", - "path": "/api/2.0/pipelines", - "body": { - "channel": "CURRENT", - "clusters": [ - { - "label": "mycluster", - "num_workers": 1 - } - ], - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edition": "ADVANCED", - "name": "source-pipeline" - } - }, - { - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "git_source": { - "git_branch": "main", - "git_provider": "gitHub", - "git_url": "https://github.com/databricks/cli" - }, - "max_concurrent_runs": 1, - "name": "src-git", - "queue": { - "enabled": true - } - } - }, - { - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "src-tasks", - "queue": { - "enabled": true - }, - "tasks": [ - { - "libraries": [ - { - "whl": "/path/to/wheel.whl" - } - ], - "notebook_task": { - "notebook_path": "/Workspace/notebook", - "source": "WORKSPACE" - }, - "task_key": "main" - } - ] - } - }, - { - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "mycluster", - "queue": { - "enabled": true - } - } - }, - { - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "main", - "queue": { - "enabled": true - } - } - }, - { - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "/path/to/wheel.whl", - "queue": { - "enabled": true - } - } - } -] diff --git a/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate b/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate deleted file mode 100644 index 3a1869177a1..00000000000 --- a/acceptance/bundle/migrate/auto/tf-path-references/terraform.tfstate +++ /dev/null @@ -1,691 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.5.5", - "serial": 7, - "lineage": "b76425ce-df50-075e-317c-8d4f7d38b839", - "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "databricks_job", - "name": "dst_git", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 2, - "attributes": { - "always_running": false, - "budget_policy_id": null, - "continuous": [], - "control_run_state": false, - "dbt_task": [], - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "description": null, - "edit_mode": "UI_LOCKED", - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment": [], - "existing_cluster_id": null, - "format": "MULTI_TASK", - "git_source": [], - "health": [], - "id": "8791211499496624000", - "job_cluster": [], - "library": [], - "max_concurrent_runs": 1, - "max_retries": 0, - "min_retry_interval_millis": 0, - "name": "main", - "new_cluster": [], - "notebook_task": [], - "notification_settings": [], - "parameter": [], - "parent_path": null, - "performance_target": null, - "pipeline_task": [], - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "python_wheel_task": [], - "queue": [ - { - "enabled": true - } - ], - "retry_on_timeout": false, - "run_as": [ - { - "group_name": "", - "service_principal_name": "", - "user_name": "tester@databricks.com" - } - ], - "run_job_task": [], - "schedule": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "tags": null, - "task": [], - "timeout_seconds": 0, - "timeouts": null, - "trigger": [], - "triggers": [], - "url": "[DATABRICKS_URL]/#job/8791211499496624000", - "usage_policy_id": null, - "webhook_notifications": [ - { - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", - "dependencies": [ - "databricks_job.src_git" - ] - } - ] - }, - { - "mode": "managed", - "type": "databricks_job", - "name": "dst_pipeline", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 2, - "attributes": { - "always_running": false, - "budget_policy_id": null, - "continuous": [], - "control_run_state": false, - "dbt_task": [], - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "description": null, - "edit_mode": "UI_LOCKED", - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment": [], - "existing_cluster_id": null, - "format": "MULTI_TASK", - "git_source": [], - "health": [], - "id": "8791211499480065000", - "job_cluster": [], - "library": [], - "max_concurrent_runs": 1, - "max_retries": 0, - "min_retry_interval_millis": 0, - "name": "mycluster", - "new_cluster": [], - "notebook_task": [], - "notification_settings": [], - "parameter": [], - "parent_path": null, - "performance_target": null, - "pipeline_task": [], - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "python_wheel_task": [], - "queue": [ - { - "enabled": true - } - ], - "retry_on_timeout": false, - "run_as": [ - { - "group_name": "", - "service_principal_name": "", - "user_name": "tester@databricks.com" - } - ], - "run_job_task": [], - "schedule": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "tags": null, - "task": [], - "timeout_seconds": 0, - "timeouts": null, - "trigger": [], - "triggers": [], - "url": "[DATABRICKS_URL]/#job/8791211499480065000", - "usage_policy_id": null, - "webhook_notifications": [ - { - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", - "dependencies": [ - "databricks_pipeline.src_pipeline" - ] - } - ] - }, - { - "mode": "managed", - "type": "databricks_job", - "name": "dst_tasks", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 2, - "attributes": { - "always_running": false, - "budget_policy_id": null, - "continuous": [], - "control_run_state": false, - "dbt_task": [], - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "description": null, - "edit_mode": "UI_LOCKED", - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment": [], - "existing_cluster_id": null, - "format": "MULTI_TASK", - "git_source": [], - "health": [], - "id": "8791211499497748000", - "job_cluster": [], - "library": [], - "max_concurrent_runs": 1, - "max_retries": 0, - "min_retry_interval_millis": 0, - "name": "/path/to/wheel.whl", - "new_cluster": [], - "notebook_task": [], - "notification_settings": [], - "parameter": [], - "parent_path": null, - "performance_target": null, - "pipeline_task": [], - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "python_wheel_task": [], - "queue": [ - { - "enabled": true - } - ], - "retry_on_timeout": false, - "run_as": [ - { - "group_name": "", - "service_principal_name": "", - "user_name": "tester@databricks.com" - } - ], - "run_job_task": [], - "schedule": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "tags": null, - "task": [], - "timeout_seconds": 0, - "timeouts": null, - "trigger": [], - "triggers": [], - "url": "[DATABRICKS_URL]/#job/8791211499497748000", - "usage_policy_id": null, - "webhook_notifications": [ - { - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", - "dependencies": [ - "databricks_job.src_tasks" - ] - } - ] - }, - { - "mode": "managed", - "type": "databricks_job", - "name": "src_git", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 2, - "attributes": { - "always_running": false, - "budget_policy_id": null, - "continuous": [], - "control_run_state": false, - "dbt_task": [], - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "description": null, - "edit_mode": "UI_LOCKED", - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment": [], - "existing_cluster_id": null, - "format": "MULTI_TASK", - "git_source": [ - { - "branch": "main", - "commit": "", - "git_snapshot": [], - "job_source": [], - "provider": "gitHub", - "sparse_checkout": [], - "tag": "", - "url": "https://github.com/databricks/cli" - } - ], - "health": [], - "id": "8791211499447989000", - "job_cluster": [], - "library": [], - "max_concurrent_runs": 1, - "max_retries": 0, - "min_retry_interval_millis": 0, - "name": "src-git", - "new_cluster": [], - "notebook_task": [], - "notification_settings": [], - "parameter": [], - "parent_path": null, - "performance_target": null, - "pipeline_task": [], - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "python_wheel_task": [], - "queue": [ - { - "enabled": true - } - ], - "retry_on_timeout": false, - "run_as": [ - { - "group_name": "", - "service_principal_name": "", - "user_name": "tester@databricks.com" - } - ], - "run_job_task": [], - "schedule": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "tags": null, - "task": [], - "timeout_seconds": 0, - "timeouts": null, - "trigger": [], - "triggers": [], - "url": "[DATABRICKS_URL]/#job/8791211499447989000", - "usage_policy_id": null, - "webhook_notifications": [ - { - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" - } - ] - }, - { - "mode": "managed", - "type": "databricks_job", - "name": "src_tasks", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 2, - "attributes": { - "always_running": false, - "budget_policy_id": null, - "continuous": [], - "control_run_state": false, - "dbt_task": [], - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "description": null, - "edit_mode": "UI_LOCKED", - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment": [], - "existing_cluster_id": null, - "format": "MULTI_TASK", - "git_source": [], - "health": [], - "id": "8791211499448751000", - "job_cluster": [], - "library": [], - "max_concurrent_runs": 1, - "max_retries": 0, - "min_retry_interval_millis": 0, - "name": "src-tasks", - "new_cluster": [], - "notebook_task": [], - "notification_settings": [], - "parameter": [], - "parent_path": null, - "performance_target": null, - "pipeline_task": [], - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "python_wheel_task": [], - "queue": [ - { - "enabled": true - } - ], - "retry_on_timeout": false, - "run_as": [ - { - "group_name": "", - "service_principal_name": "", - "user_name": "tester@databricks.com" - } - ], - "run_job_task": [], - "schedule": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "tags": null, - "task": [ - { - "ai_runtime_task": [], - "alert_task": [], - "clean_rooms_notebook_task": [], - "compute": [], - "condition_task": [], - "dashboard_task": [], - "dbt_cloud_task": [], - "dbt_platform_task": [], - "dbt_task": [], - "depends_on": [], - "description": "", - "disable_auto_optimization": false, - "disabled": false, - "email_notifications": [ - { - "no_alert_for_skipped_runs": false, - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ], - "environment_key": "", - "existing_cluster_id": "", - "for_each_task": [], - "gen_ai_compute_task": [], - "health": [], - "job_cluster_key": "", - "library": [ - { - "cran": [], - "egg": "", - "jar": "", - "maven": [], - "provider_config": [], - "pypi": [], - "requirements": "", - "whl": "/path/to/wheel.whl" - } - ], - "max_retries": 0, - "min_retry_interval_millis": 0, - "new_cluster": [], - "notebook_task": [ - { - "base_parameters": null, - "notebook_path": "/Workspace/notebook", - "source": "WORKSPACE", - "warehouse_id": "" - } - ], - "notification_settings": [], - "pipeline_task": [], - "power_bi_task": [], - "python_operator_task": [], - "python_wheel_task": [], - "retry_on_timeout": false, - "run_if": "ALL_SUCCESS", - "run_job_task": [], - "spark_jar_task": [], - "spark_python_task": [], - "spark_submit_task": [], - "sql_task": [], - "task_key": "main", - "timeout_seconds": 0, - "webhook_notifications": [] - } - ], - "timeout_seconds": 0, - "timeouts": null, - "trigger": [], - "triggers": [], - "url": "[DATABRICKS_URL]/#job/8791211499448751000", - "usage_policy_id": null, - "webhook_notifications": [ - { - "on_duration_warning_threshold_exceeded": [], - "on_failure": [], - "on_start": [], - "on_streaming_backlog_exceeded": [], - "on_success": [] - } - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" - } - ] - }, - { - "mode": "managed", - "type": "databricks_pipeline", - "name": "src_pipeline", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "allow_duplicate_names": false, - "budget_policy_id": null, - "catalog": null, - "cause": null, - "channel": "CURRENT", - "cluster": [ - { - "apply_policy_default_values": false, - "autoscale": [], - "aws_attributes": [], - "azure_attributes": [], - "cluster_log_conf": [], - "custom_tags": null, - "driver_instance_pool_id": "", - "driver_node_type_id": "", - "enable_local_disk_encryption": false, - "gcp_attributes": [], - "init_scripts": [], - "instance_pool_id": "", - "label": "mycluster", - "node_type_id": "", - "num_workers": 1, - "policy_id": "", - "spark_conf": null, - "spark_env_vars": null, - "ssh_public_keys": null - } - ], - "cluster_id": null, - "configuration": null, - "continuous": false, - "creator_user_name": "tester@databricks.com", - "deployment": [ - { - "deployment_id": "", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", - "version_id": "" - } - ], - "development": false, - "edition": "ADVANCED", - "environment": [], - "event_log": [], - "expected_last_modified": 0, - "filters": [], - "gateway_definition": [], - "health": null, - "id": "7a00a847-e948-2520-7a00-a847e9482521", - "ingestion_definition": [], - "last_modified": 1791211499437, - "latest_updates": null, - "library": [], - "name": "source-pipeline", - "notification": [], - "photon": false, - "provider_config": [ - { - "workspace_id": "900800700600" - } - ], - "restart_window": [], - "root_path": null, - "run_as": [], - "run_as_user_name": "tester@databricks.com", - "schema": null, - "serverless": false, - "serverless_compute_id": null, - "state": "IDLE", - "storage": "dbfs:/pipelines/7a00a847-e948-2520-7a00-a847e9482521", - "tags": null, - "target": null, - "timeouts": null, - "trigger": [], - "url": "[DATABRICKS_URL]/#joblist/pipelines/7a00a847-e948-2520-7a00-a847e9482521", - "usage_policy_id": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - } - ], - "check_results": null -} diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt index 45557fcc89f..a75cf4019df 100644 --- a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt @@ -2,6 +2,16 @@ === Replay a prior terraform deployment >>> replay_tfstate.py terraform-requests.json terraform.tfstate +=== Validation does not flag the reference +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! + === Plan after auto-migration >>> musterr [CLI] bundle plan Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/script b/acceptance/bundle/migrate/auto/tf-resource-type-references/script index 548dd21a149..0eada0a9bf6 100644 --- a/acceptance/bundle/migrate/auto/tf-resource-type-references/script +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/script @@ -7,6 +7,9 @@ title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate rm -f "$OUT_REQUESTS" +title "Validation does not flag the reference" +trace $CLI bundle validate + title "Plan after auto-migration" trace musterr $CLI bundle plan diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml b/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml deleted file mode 100644 index 6d91f6515bd..00000000000 --- a/acceptance/bundle/resource_deps/tf_resource_type_ref/databricks.yml +++ /dev/null @@ -1,17 +0,0 @@ -bundle: - name: test-bundle - -resources: - pipelines: - src_pipeline: - name: source-pipeline - - jobs: - dst_job: - name: dst-job - tasks: - - task_key: main - pipeline_task: - # Terraform-style resource reference: databricks_pipeline is the terraform - # resource type for DABs "pipelines". - pipeline_id: ${databricks_pipeline.src_pipeline.id} diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml b/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/resource_deps/tf_resource_type_ref/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt b/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt deleted file mode 100644 index 5da823336ce..00000000000 --- a/acceptance/bundle/resource_deps/tf_resource_type_ref/output.txt +++ /dev/null @@ -1,19 +0,0 @@ - ->>> [CLI] bundle validate -Name: test-bundle -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Validation OK! - ->>> musterr [CLI] bundle plan -Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" - - ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" - -Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resource_deps/tf_resource_type_ref/script b/acceptance/bundle/resource_deps/tf_resource_type_ref/script deleted file mode 100644 index b1f297e2cf1..00000000000 --- a/acceptance/bundle/resource_deps/tf_resource_type_ref/script +++ /dev/null @@ -1,6 +0,0 @@ -# ${databricks_....} references (terraform resource types) are not supported on the -# direct engine, unlike terraform field paths inside ${resources.*} (see tf_path_renames). -# Validation does not flag them; plan and deploy fail. -trace $CLI bundle validate -trace musterr $CLI bundle plan -trace musterr $CLI bundle deploy