bundle/direct: record resource operations with DMS

acceptance/bundle/deploy/record-deployment-history/jobs/databricks.delete.yml

@@ -0,0 +1,8 @@
+bundle:
+  name: test-rdh-jobs
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs: {}

acceptance/bundle/deploy/record-deployment-history/jobs/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-rdh-jobs
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo-job

acceptance/bundle/deploy/record-deployment-history/jobs/output.txt

@@ -0,0 +1,94 @@
+
+=== create: records OPERATION_ACTION_TYPE_CREATE
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-jobs/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.jobs.foo"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "resources.jobs.foo",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-rdh-jobs/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "foo-job",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+
+=== update: records OPERATION_ACTION_TYPE_UPDATE
+>>> update_file.py databricks.yml foo-job foo-job-renamed
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-jobs/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.jobs.foo"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_UPDATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "resources.jobs.foo",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-rdh-jobs/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "foo-job-renamed",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+
+=== delete: records OPERATION_ACTION_TYPE_DELETE (resource removed from config)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-jobs/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.jobs.foo"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_DELETE",
+    "resource_id": "[NUMID]",
+    "resource_key": "resources.jobs.foo",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}

acceptance/bundle/deploy/record-deployment-history/jobs/script

@@ -0,0 +1,13 @@
+title "create: records OPERATION_ACTION_TYPE_CREATE"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle
+
+title "update: records OPERATION_ACTION_TYPE_UPDATE"
+trace update_file.py databricks.yml foo-job foo-job-renamed
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle
+
+title "delete: records OPERATION_ACTION_TYPE_DELETE (resource removed from config)"
+cp databricks.delete.yml databricks.yml
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle

acceptance/bundle/deploy/record-deployment-history/jobs/test.toml

@@ -0,0 +1,2 @@
+# databricks.yml is rewritten in-place by the script (update + delete steps).
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/deploy/record-deployment-history/recreate/databricks.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: test-rdh-recreate
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  pipelines:
+    bar:
+      name: bar-pipeline
+      storage: /tmp/storage-a

acceptance/bundle/deploy/record-deployment-history/recreate/output.txt

@@ -0,0 +1,72 @@
+
+=== create the pipeline
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-recreate/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.pipelines.bar"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[UUID]",
+    "resource_key": "resources.pipelines.bar",
+    "state": {
+      "channel": "CURRENT",
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-rdh-recreate/default/state/metadata.json"
+      },
+      "edition": "ADVANCED",
+      "name": "bar-pipeline",
+      "storage": "/tmp/storage-a"
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+
+=== recreate: changing immutable 'storage' records OPERATION_ACTION_TYPE_RECREATE
+>>> update_file.py databricks.yml /tmp/storage-a /tmp/storage-b
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-recreate/default/files...
+
+This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will
+restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline
+properties such as the 'catalog' or 'storage' are changed:
+  recreate resources.pipelines.bar
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.pipelines.bar"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_RECREATE",
+    "resource_id": "[UUID]",
+    "resource_key": "resources.pipelines.bar",
+    "state": {
+      "channel": "CURRENT",
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-rdh-recreate/default/state/metadata.json"
+      },
+      "edition": "ADVANCED",
+      "name": "bar-pipeline",
+      "storage": "/tmp/storage-b"
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}

acceptance/bundle/deploy/record-deployment-history/recreate/script

@@ -0,0 +1,8 @@
+title "create the pipeline"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle
+
+title "recreate: changing immutable 'storage' records OPERATION_ACTION_TYPE_RECREATE"
+trace update_file.py databricks.yml /tmp/storage-a /tmp/storage-b
+trace $CLI bundle deploy --auto-approve
+trace print_requests.py //api/2.0/bundle

acceptance/bundle/deploy/record-deployment-history/recreate/test.toml

@@ -0,0 +1,2 @@
+# databricks.yml is rewritten in-place by the script (storage change).
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/deploy/record-deployment-history/resize/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: test-rdh-resize
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  clusters:
+    cl:
+      cluster_name: test-cluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+      lifecycle:
+        started: true

acceptance/bundle/deploy/record-deployment-history/resize/output.txt

@@ -0,0 +1,66 @@
+
+=== create the cluster (running via lifecycle.started)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-resize/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.clusters.cl"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[UUID]",
+    "resource_key": "resources.clusters.cl",
+    "state": {
+      "autotermination_minutes": 60,
+      "cluster_name": "test-cluster",
+      "lifecycle": {
+        "started": true
+      },
+      "node_type_id": "[NODE_TYPE_ID]",
+      "num_workers": 1,
+      "spark_version": "15.4.x-scala2.12"
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+
+=== resize: changing num_workers on a running cluster records OPERATION_ACTION_TYPE_RESIZE
+>>> update_file.py databricks.yml num_workers: 1 num_workers: 2
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-resize/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/abcd/versions/0/operations",
+  "q": {
+    "resource_key": "resources.clusters.cl"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_RESIZE",
+    "resource_id": "[UUID]",
+    "resource_key": "resources.clusters.cl",
+    "state": {
+      "autotermination_minutes": 60,
+      "cluster_name": "test-cluster",
+      "lifecycle": {
+        "started": true
+      },
+      "node_type_id": "[NODE_TYPE_ID]",
+      "num_workers": 2,
+      "spark_version": "15.4.x-scala2.12"
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}

acceptance/bundle/deploy/record-deployment-history/resize/script

@@ -0,0 +1,8 @@
+title "create the cluster (running via lifecycle.started)"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle
+
+title "resize: changing num_workers on a running cluster records OPERATION_ACTION_TYPE_RESIZE"
+trace update_file.py databricks.yml "num_workers: 1" "num_workers: 2"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle

acceptance/bundle/deploy/record-deployment-history/resize/test.toml

@@ -0,0 +1,2 @@
+# databricks.yml is rewritten in-place by the script (num_workers change).
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/deploy/record-deployment-history/test.toml

@@ -0,0 +1,8 @@
+Local = true
+Cloud = false
+RecordRequests = true
+
+# Operation recording only runs for the direct engine; the terraform engine
+# would record nothing and produce divergent output.
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/deploy/record-deployment-history/update-id/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-rdh-update-id
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  catalogs:
+    cat:
+      name: my_catalog_a