acc: terraform resource-type references after auto-migration

acceptance/bundle/migrate/auto/tf-resource-type-references/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle
+
+resources:
+  pipelines:
+    src_pipeline:
+      name: source-pipeline
+
+  jobs:
+    dst_job:
+      name: dst-job
+      tasks:
+        - task_key: main
+          pipeline_task:
+            # Terraform-style resource reference: databricks_pipeline is the terraform
+            # resource type for DABs "pipelines".
+            pipeline_id: ${databricks_pipeline.src_pipeline.id}

acceptance/bundle/migrate/auto/tf-resource-type-references/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt

@@ -0,0 +1,42 @@
+
+=== Replay a prior terraform deployment
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+=== Validation does not flag the reference
+>>> [CLI] bundle validate
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Validation OK!
+
+=== Plan after auto-migration
+>>> musterr [CLI] bundle plan
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node ""
+
+
+=== Deploy after auto-migration
+>>> musterr [CLI] bundle deploy
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node ""
+
+Files: 1 uploaded, 0 deleted
+
+>>> [CLI] bundle destroy --auto-approve
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+The following resources will be deleted:
+  delete resources.jobs.dst_job
+  delete resources.pipelines.src_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.src_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Migrated 2 resources to direct deployment engine.
+Destroy: 2 deleted

acceptance/bundle/migrate/auto/tf-resource-type-references/script

@@ -0,0 +1,21 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+# The config references the pipeline by its terraform resource type (${databricks_pipeline...}),
+# which terraform resolved on the original deploy. The direct engine does not support this syntax
+# (only terraform field paths inside ${resources.*}), so after auto-migration plan and deploy fail.
+title "Replay a prior terraform deployment"
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+rm -f "$OUT_REQUESTS"
+
+title "Validation does not flag the reference"
+trace $CLI bundle validate
+
+title "Plan after auto-migration"
+trace musterr $CLI bundle plan
+
+title "Deploy after auto-migration"
+trace musterr $CLI bundle deploy
+rm -f "$OUT_REQUESTS"
+
+trace $CLI bundle destroy --auto-approve
+rm -f "$OUT_REQUESTS"

acceptance/bundle/migrate/auto/tf-resource-type-references/terraform-requests.json

@@ -0,0 +1,61 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.0/workspace/mkdirs",
+    "body": {
+      "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state"
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.0/workspace/mkdirs",
+    "body": {
+      "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/artifacts/.internal"
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.0/workspace/mkdirs",
+    "body": {
+      "path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files"
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.0/pipelines",
+    "body": {
+      "channel": "CURRENT",
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "edition": "ADVANCED",
+      "name": "source-pipeline"
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "dst-job",
+      "queue": {
+        "enabled": true
+      },
+      "tasks": [
+        {
+          "pipeline_task": {
+            "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9"
+          },
+          "task_key": "main"
+        }
+      ]
+    }
+  }
+]

acceptance/bundle/migrate/auto/tf-resource-type-references/terraform.tfstate

@@ -0,0 +1,244 @@
+{
+  "version": 4,
+  "terraform_version": "1.5.5",
+  "serial": 3,
+  "lineage": "feddf884-7b6a-3c90-968d-3dcc4761437e",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "databricks_job",
+      "name": "dst_job",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 2,
+          "attributes": {
+            "always_running": false,
+            "budget_policy_id": null,
+            "continuous": [],
+            "control_run_state": false,
+            "dbt_task": [],
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "description": null,
+            "edit_mode": "UI_LOCKED",
+            "email_notifications": [
+              {
+                "no_alert_for_skipped_runs": false,
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ],
+            "environment": [],
+            "existing_cluster_id": null,
+            "format": "MULTI_TASK",
+            "git_source": [],
+            "health": [],
+            "id": "8791212044006401000",
+            "job_cluster": [],
+            "library": [],
+            "max_concurrent_runs": 1,
+            "max_retries": 0,
+            "min_retry_interval_millis": 0,
+            "name": "dst-job",
+            "new_cluster": [],
+            "notebook_task": [],
+            "notification_settings": [],
+            "parameter": [],
+            "parent_path": null,
+            "performance_target": null,
+            "pipeline_task": [],
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "python_wheel_task": [],
+            "queue": [
+              {
+                "enabled": true
+              }
+            ],
+            "retry_on_timeout": false,
+            "run_as": [
+              {
+                "group_name": "",
+                "service_principal_name": "",
+                "user_name": "tester@databricks.com"
+              }
+            ],
+            "run_job_task": [],
+            "schedule": [],
+            "spark_jar_task": [],
+            "spark_python_task": [],
+            "spark_submit_task": [],
+            "tags": null,
+            "task": [
+              {
+                "ai_runtime_task": [],
+                "alert_task": [],
+                "clean_rooms_notebook_task": [],
+                "compute": [],
+                "condition_task": [],
+                "dashboard_task": [],
+                "dbt_cloud_task": [],
+                "dbt_platform_task": [],
+                "dbt_task": [],
+                "depends_on": [],
+                "description": "",
+                "disable_auto_optimization": false,
+                "disabled": false,
+                "email_notifications": [
+                  {
+                    "no_alert_for_skipped_runs": false,
+                    "on_duration_warning_threshold_exceeded": [],
+                    "on_failure": [],
+                    "on_start": [],
+                    "on_streaming_backlog_exceeded": [],
+                    "on_success": []
+                  }
+                ],
+                "environment_key": "",
+                "existing_cluster_id": "",
+                "for_each_task": [],
+                "gen_ai_compute_task": [],
+                "health": [],
+                "job_cluster_key": "",
+                "library": [],
+                "max_retries": 0,
+                "min_retry_interval_millis": 0,
+                "new_cluster": [],
+                "notebook_task": [],
+                "notification_settings": [],
+                "pipeline_task": [
+                  {
+                    "full_refresh": false,
+                    "full_refresh_selection": null,
+                    "parameters": null,
+                    "pipeline_id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9",
+                    "refresh_flow_selection": null,
+                    "refresh_selection": null,
+                    "reset_checkpoint_selection": null
+                  }
+                ],
+                "power_bi_task": [],
+                "python_operator_task": [],
+                "python_wheel_task": [],
+                "retry_on_timeout": false,
+                "run_if": "ALL_SUCCESS",
+                "run_job_task": [],
+                "spark_jar_task": [],
+                "spark_python_task": [],
+                "spark_submit_task": [],
+                "sql_task": [],
+                "task_key": "main",
+                "timeout_seconds": 0,
+                "webhook_notifications": []
+              }
+            ],
+            "timeout_seconds": 0,
+            "timeouts": null,
+            "trigger": [],
+            "triggers": [],
+            "url": "[DATABRICKS_URL]/#job/8791212044006401000",
+            "usage_policy_id": null,
+            "webhook_notifications": [
+              {
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ]
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==",
+          "dependencies": [
+            "databricks_pipeline.src_pipeline"
+          ]
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "databricks_pipeline",
+      "name": "src_pipeline",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "allow_duplicate_names": false,
+            "budget_policy_id": null,
+            "catalog": null,
+            "cause": null,
+            "channel": "CURRENT",
+            "cluster": [],
+            "cluster_id": null,
+            "configuration": null,
+            "continuous": false,
+            "creator_user_name": "tester@databricks.com",
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "development": false,
+            "edition": "ADVANCED",
+            "environment": [],
+            "event_log": [],
+            "expected_last_modified": 0,
+            "filters": [],
+            "gateway_definition": [],
+            "health": null,
+            "id": "7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9",
+            "ingestion_definition": [],
+            "last_modified": 1791212043963,
+            "latest_updates": null,
+            "library": [],
+            "name": "source-pipeline",
+            "notification": [],
+            "photon": false,
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "restart_window": [],
+            "root_path": null,
+            "run_as": [],
+            "run_as_user_name": "tester@databricks.com",
+            "schema": null,
+            "serverless": false,
+            "serverless_compute_id": null,
+            "state": "IDLE",
+            "storage": "dbfs:/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9",
+            "tags": null,
+            "target": null,
+            "timeouts": null,
+            "trigger": [],
+            "url": "[DATABRICKS_URL]/#joblist/pipelines/7a00a8c6-b18f-59a8-7a00-a8c6b18f59a9",
+            "usage_policy_id": null
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
+        }
+      ]
+    }
+  ],
+  "check_results": null
+}