diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml b/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml new file mode 100644 index 00000000000..6d91f6515bd --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + pipelines: + src_pipeline: + name: source-pipeline + + jobs: + dst_job: + name: dst-job + tasks: + - task_key: main + pipeline_task: + # Terraform-style resource reference: databricks_pipeline is the terraform + # resource type for DABs "pipelines". + pipeline_id: ${databricks_pipeline.src_pipeline.id} diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml b/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt new file mode 100644 index 00000000000..a75cf4019df --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt @@ -0,0 +1,42 @@ + +=== Replay a prior terraform deployment +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Validation does not flag the reference +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! + +=== Plan after auto-migration +>>> musterr [CLI] bundle plan +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + + +=== Deploy after auto-migration +>>> musterr [CLI] bundle deploy +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" + +Files: 1 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +The following resources will be deleted: + delete resources.jobs.dst_job + delete resources.pipelines.src_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.src_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Migrated 2 resources to direct deployment engine. +Destroy: 2 deleted diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/script b/acceptance/bundle/migrate/auto/tf-resource-type-references/script new file mode 100644 index 00000000000..0eada0a9bf6 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/script @@ -0,0 +1,21 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# The config references the pipeline by its terraform resource type (${databricks_pipeline...}), +# which terraform resolved on the original deploy. The direct engine does not support this syntax +# (only terraform field paths inside ${resources.*}), so after auto-migration plan and deploy fail. +title "Replay a prior terraform deployment" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +rm -f "$OUT_REQUESTS" + +title "Validation does not flag the reference" +trace $CLI bundle validate + +title "Plan after auto-migration" +trace musterr $CLI bundle plan + +title "Deploy after auto-migration" +trace musterr $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json new file mode 100644 index 00000000000..b638e9f92d5 --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json @@ -0,0 +1,61 @@ +[ + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/artifacts/.internal" + } + }, + { + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files" + } + }, + { + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "source-pipeline" + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "dst-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "pipeline_task": { + "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9" + }, + "task_key": "main" + } + ] + } + } +] diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate new file mode 100644 index 00000000000..564391bbe5a --- /dev/null +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate @@ -0,0 +1,244 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 3, + "lineage": "feddf884-7b6a-3c90-968d-3dcc4761437e", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "dst_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8791212044006401000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "dst-job", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [ + { + "ai_runtime_task": [], + "alert_task": [], + "clean_rooms_notebook_task": [], + "compute": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "", + "library": [], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "pipeline_task": [ + { + "full_refresh": false, + "full_refresh_selection": null, + "parameters": null, + "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "refresh_flow_selection": null, + "refresh_selection": null, + "reset_checkpoint_selection": null + } + ], + "power_bi_task": [], + "python_operator_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "ALL_SUCCESS", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "main", + "timeout_seconds": 0, + "webhook_notifications": [] + } + ], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8791212044006401000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "databricks_pipeline.src_pipeline" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "src_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": null, + "cause": null, + "channel": "CURRENT", + "cluster": [], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "tester@databricks.com", + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json", + "version_id": "" + } + ], + "development": false, + "edition": "ADVANCED", + "environment": [], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "ingestion_definition": [], + "last_modified": 1791212043963, + "latest_updates": null, + "library": [], + "name": "source-pipeline", + "notification": [], + "photon": false, + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "restart_window": [], + "root_path": null, + "run_as": [], + "run_as_user_name": "tester@databricks.com", + "schema": null, + "serverless": false, + "serverless_compute_id": null, + "state": "IDLE", + "storage": "dbfs:/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "tags": null, + "target": null, + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +}