From 3854cff47bbee9647891fc70e0bf981a02ba7ba9 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 31 Jul 2026 11:55:23 +0000 Subject: [PATCH] acc: cover config-remote-sync write-back for split lists Co-authored-by: Isaac --- .../bundle/config-remote-sync/README.md | 30 ++++++++ .../split_isolation/databricks.yml.tmpl | 34 +++++++++ .../split_isolation/out.test.toml | 4 ++ .../split_isolation/output.txt | 56 +++++++++++++++ .../config-remote-sync/split_isolation/script | 39 +++++++++++ .../split_isolation/test.toml | 8 +++ .../split_keyed_edit/databricks.yml.tmpl | 30 ++++++++ .../split_keyed_edit/out.test.toml | 4 ++ .../split_keyed_edit/output.txt | 57 +++++++++++++++ .../split_keyed_edit/script | 53 ++++++++++++++ .../split_keyed_edit/test.toml | 8 +++ .../split_keyed_remove/databricks.yml.tmpl | 45 ++++++++++++ .../split_keyed_remove/out.test.toml | 4 ++ .../split_keyed_remove/output.txt | 54 ++++++++++++++ .../split_keyed_remove/script | 54 ++++++++++++++ .../split_keyed_remove/test.toml | 8 +++ .../split_keyed_twoblock/databricks.yml.tmpl | 35 ++++++++++ .../split_keyed_twoblock/out.test.toml | 4 ++ .../split_keyed_twoblock/output.txt | 70 +++++++++++++++++++ .../split_keyed_twoblock/script | 56 +++++++++++++++ .../split_keyed_twoblock/test.toml | 8 +++ .../split_multifile/databricks.yml.tmpl | 17 +++++ .../split_multifile/out.test.toml | 4 ++ .../split_multifile/output.txt | 40 +++++++++++ .../overrides/10-first.yml.tmpl | 10 +++ .../overrides/20-second.yml.tmpl | 9 +++ .../config-remote-sync/split_multifile/script | 45 ++++++++++++ .../split_multifile/test.toml | 14 ++++ .../split_positional/databricks.yml.tmpl | 30 ++++++++ .../split_positional/out.test.toml | 4 ++ .../split_positional/output.txt | 52 ++++++++++++++ .../split_positional/script | 53 ++++++++++++++ .../split_positional/test.toml | 8 +++ 33 files changed, 947 insertions(+) create mode 100644 acceptance/bundle/config-remote-sync/README.md create mode 100644 acceptance/bundle/config-remote-sync/split_isolation/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_isolation/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_isolation/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_isolation/script create mode 100644 acceptance/bundle/config-remote-sync/split_isolation/test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_edit/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_edit/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_edit/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_edit/script create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_edit/test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_remove/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_remove/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_remove/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_remove/script create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_remove/test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_twoblock/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_twoblock/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_twoblock/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_twoblock/script create mode 100644 acceptance/bundle/config-remote-sync/split_keyed_twoblock/test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/overrides/10-first.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/overrides/20-second.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/script create mode 100644 acceptance/bundle/config-remote-sync/split_multifile/test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_positional/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/split_positional/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/split_positional/output.txt create mode 100644 acceptance/bundle/config-remote-sync/split_positional/script create mode 100644 acceptance/bundle/config-remote-sync/split_positional/test.toml diff --git a/acceptance/bundle/config-remote-sync/README.md b/acceptance/bundle/config-remote-sync/README.md new file mode 100644 index 00000000000..24afbdb9eeb --- /dev/null +++ b/acceptance/bundle/config-remote-sync/README.md @@ -0,0 +1,30 @@ +# Split-list write-back tests + +The `split_*` directories cover `config-remote-sync` write-back +when a resource's list field is defined in more than one physical YAML region: a top-level +`resources...` block and a `targets..resources…` override, +possibly across several included files. + +Loading merges those regions into one in-memory list (keyed lists are also sorted by key), +but on disk they remain separate regions that have to be edited independently. A remote +change is expressed against the merged view, so write-back has to map it back to the right +`(file, block, position)`. + +**The goldens in these directories record current behavior, which is wrong.** They are +committed as-is so that the fixes which follow show up as a golden diff: corruption on one +side, correct output on the other. Each `script` names the scenario it covers next to the +step that exercises it. + +What the goldens currently show: + +| Directory | Current behavior | +| --- | --- | +| `split_keyed_edit` | Editing a task defined only in the target block appends a keyless `- timeout_seconds: 111` element to the top-level block instead of updating the task. The follow-up sync then reports `tasks[task_key='']: remove` for the element it just created. | +| `split_keyed_twoblock` | Editing a target-block task whose sibling is defined in both blocks appends another keyless element. Routing one field per block on the two-block element already works and is covered here too. | +| `split_keyed_remove` | Removing one task per block fails with `remove index key out of bounds (idx 2, len 2)`: a merged index is used against a block that has fewer elements. Removing a task defined in both blocks deletes it from the top-level block, leaving the target block's half orphaned. | +| `split_multifile` | An edit to a task defined in an included override file is written to the top-level `databricks.yml` instead, so the wrong file changes and the intended one does not. | +| `split_positional` | A field edit on a positionally-diffed pipeline cluster in the target block fails with `parent path /resources/pipelines/split_pipeline/clusters/1 does not exist`. A length change rewrites the whole list into one block. | +| `split_isolation` | A rename of a two-block task is applied by guessing: the task key, the merged `timeout_seconds`, and quoted scalars are all written into the top-level block, and the blank line between resources is consumed. | + +All of these run locally against the in-process test server (no `Cloud = true`), under both +deployment engines, so they execute in normal CI. diff --git a/acceptance/bundle/config-remote-sync/split_isolation/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/split_isolation/databricks.yml.tmpl new file mode 100644 index 00000000000..c3d096cd0d5 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_isolation/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# job_a's task list is split across both blocks and one task, "shared", is defined +# in both. job_b is an ordinary single-block job. +# +# The point of the fixture: whatever happens to job_a, an unrelated resource's +# unambiguous change must still be applied in the same run. The sync is +# unattended, so one hard-to-place change must never stop the rest. +resources: + jobs: + job_a: + tasks: + - task_key: shared + max_retries: 1 + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/shared + + job_b: + max_concurrent_runs: 1 + tasks: + - task_key: simple + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/simple + +targets: + dev: + mode: development + resources: + jobs: + job_a: + tasks: + - task_key: shared + timeout_seconds: 45 diff --git a/acceptance/bundle/config-remote-sync/split_isolation/out.test.toml b/acceptance/bundle/config-remote-sync/split_isolation/out.test.toml new file mode 100644 index 00000000000..4b5914daa2c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_isolation/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split_isolation/output.txt b/acceptance/bundle/config-remote-sync/split_isolation/output.txt new file mode 100644 index 00000000000..90bda10cfaa --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_isolation/output.txt @@ -0,0 +1,56 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Rename the two-block task on job_a, and edit job_b in the same run +=== Sync +Detected changes in 2 resource(s): + +Resource: resources.jobs.job_a + tasks[task_key='shared']: remove + tasks[task_key='shared_renamed']: add + +Resource: resources.jobs.job_b + max_concurrent_runs: replace + + + +=== job_b is updated, and job_a keeps exactly one copy of the task + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -12,11 +12,11 @@ + job_a: + tasks: +- - task_key: shared +- max_retries: 1 ++ - max_retries: 1 + notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/shared +- ++ notebook_path: '/Users/{{workspace_user_name}}/shared' ++ task_key: shared_renamed ++ timeout_seconds: 45 + job_b: +- max_concurrent_runs: 1 ++ max_concurrent_runs: 6 + tasks: + - task_key: simple + +>>> grep -c max_concurrent_runs: 6 databricks.yml +1 + +>>> grep -c task_key: shared_renamed databricks.yml +1 + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.jobs.job_a + delete resources.jobs.job_b + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_isolation/script b/acceptance/bundle/config-remote-sync/split_isolation/script new file mode 100644 index 00000000000..74f6cc50ee8 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_isolation/script @@ -0,0 +1,39 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +job_a_id="$(read_id.py job_a)" +job_b_id="$(read_id.py job_b)" + + +# A rename of the two-block task on job_a, and a plain scalar edit on job_b, in the +# SAME run. job_b's edit is independent of anything job_a does, so it must be +# applied whether or not job_a's rename can be placed. +title "Rename the two-block task on job_a, and edit job_b in the same run" +edit_resource.py jobs $job_a_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -18,4 +18,5 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/mu ++ - timeout_seconds: 111 + + targets: + +=== Edit mu, a top-level task that is not first after the merge sort +=== Sync +Detected changes in 1 resource(s): + +Resource: resources.jobs.split_job + tasks[task_key='']: remove + tasks[task_key='alpha'].timeout_seconds: add + tasks[task_key='mu'].timeout_seconds: add + + + +=== Only mu in the top-level block gains timeout_seconds: 222 + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -18,4 +18,5 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/mu ++ timeout_seconds: 222 + - timeout_seconds: 111 + + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.jobs.split_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_keyed_edit/script b/acceptance/bundle/config-remote-sync/split_keyed_edit/script new file mode 100644 index 00000000000..dcfc1028455 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_keyed_edit/script @@ -0,0 +1,53 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +job_id="$(read_id.py split_job)" + + +# alpha is the only task in the target block, so it must be written at index 0 of +# that block. Its merged index is also 0, but the top-level block's index 0 is +# zeta -- a naive merged-index write lands on zeta instead. +title "Edit alpha, defined only in the target block" +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml + +=== Remove the task defined in BOTH blocks +=== Sync +Detected changes in 1 resource(s): + +Resource: resources.jobs.twoblock_remove_job + tasks[task_key='both']: remove + + + +=== 'both' is gone from both blocks; 'keep' survives + +>>> diff.py databricks.yml.second-backup databricks.yml +--- databricks.yml.second-backup ++++ databricks.yml +@@ -22,8 +22,4 @@ + twoblock_remove_job: + tasks: +- - task_key: both +- max_retries: 2 +- notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/both + - task_key: keep + notebook_task: + +>>> grep -c task_key: both databricks.yml +1 + +>>> grep -c task_key: keep databricks.yml +1 + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.jobs.remove_job + delete resources.jobs.twoblock_remove_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_keyed_remove/script b/acceptance/bundle/config-remote-sync/split_keyed_remove/script new file mode 100644 index 00000000000..7ff87657059 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_keyed_remove/script @@ -0,0 +1,54 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +remove_job_id="$(read_id.py remove_job)" +twoblock_job_id="$(read_id.py twoblock_remove_job)" + + +# Remove one task from each block in a single run. Exactly gamma and alpha must +# disappear; beta must survive even though the removals shift indices in both +# blocks. +title "Remove gamma from the top-level block and alpha from the target block" +edit_resource.py jobs $remove_job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -21,4 +21,5 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/nu ++ - timeout_seconds: 303 + + targets: + +=== Edit one field per block on the two-block element, plus add a new field +=== Sync +Detected changes in 1 resource(s): + +Resource: resources.jobs.twoblock_job + tasks[task_key='']: remove + tasks[task_key='omega'].timeout_seconds: add + tasks[task_key='shared'].max_retries: replace + tasks[task_key='shared'].min_retry_interval_millis: add + tasks[task_key='shared'].timeout_seconds: replace + + + +=== max_retries updates top-level, timeout_seconds updates target block + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -15,7 +15,8 @@ + tasks: + - task_key: shared +- max_retries: 1 ++ max_retries: 9 + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/shared ++ min_retry_interval_millis: 5000 + - task_key: nu + notebook_task: +@@ -31,5 +32,5 @@ + tasks: + - task_key: shared +- timeout_seconds: 60 ++ timeout_seconds: 900 + - task_key: omega + notebook_task: + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.jobs.twoblock_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_keyed_twoblock/script b/acceptance/bundle/config-remote-sync/split_keyed_twoblock/script new file mode 100644 index 00000000000..e4efd642208 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_keyed_twoblock/script @@ -0,0 +1,56 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +job_id="$(read_id.py twoblock_job)" + + +# omega is defined only in the target block, but its sibling "shared" contributes +# to both blocks. Editing omega must land on omega and must leave the two-block +# element completely untouched. +title "Edit omega, whose sibling is defined in two blocks" +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -16,2 +16,3 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/base ++ timeout_seconds: 777 + +=== overrides/10-first.yml is untouched + +>>> diff.py overrides/10-first.yml.backup overrides/10-first.yml + +=== overrides/20-second.yml gains timeout_seconds: 777 + +>>> diff.py overrides/20-second.yml.backup overrides/20-second.yml + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.jobs.multifile_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_multifile/overrides/10-first.yml.tmpl b/acceptance/bundle/config-remote-sync/split_multifile/overrides/10-first.yml.tmpl new file mode 100644 index 00000000000..e0d765810fb --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_multifile/overrides/10-first.yml.tmpl @@ -0,0 +1,10 @@ +targets: + dev: + mode: development + resources: + jobs: + multifile_job: + tasks: + - task_key: from_first_file + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/first diff --git a/acceptance/bundle/config-remote-sync/split_multifile/overrides/20-second.yml.tmpl b/acceptance/bundle/config-remote-sync/split_multifile/overrides/20-second.yml.tmpl new file mode 100644 index 00000000000..d0e7406aa90 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_multifile/overrides/20-second.yml.tmpl @@ -0,0 +1,9 @@ +targets: + dev: + resources: + jobs: + multifile_job: + tasks: + - task_key: from_second_file + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/second diff --git a/acceptance/bundle/config-remote-sync/split_multifile/script b/acceptance/bundle/config-remote-sync/split_multifile/script new file mode 100644 index 00000000000..cfef6aea3cc --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_multifile/script @@ -0,0 +1,45 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml +envsubst < overrides/10-first.yml.tmpl > overrides/10-first.yml +envsubst < overrides/20-second.yml.tmpl > overrides/20-second.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +job_id="$(read_id.py multifile_job)" + + +# from_second_file is defined in overrides/20-second.yml. The edit must be written +# to that file only -- not to the top-level databricks.yml and not to the sibling +# override file, and without creating a duplicate subtree anywhere. +title "Edit a task defined in the second included file" +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml + +=== Remove the target-block cluster (length change) +=== Sync +Detected changes in 1 resource(s): + +Resource: resources.pipelines.split_pipeline + clusters: replace + + + +=== Applied correctly or left unchanged; default cluster never corrupted + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -15,5 +15,5 @@ + clusters: + - label: default +- num_workers: 2 ++ num_workers: 5 + libraries: + - notebook: + +>>> grep -c label: default databricks.yml +1 + +>>> [CLI] bundle destroy --auto-approve -t dev +The following resources will be deleted: + delete resources.pipelines.split_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.split_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/split_positional/script b/acceptance/bundle/config-remote-sync/split_positional/script new file mode 100644 index 00000000000..a1ad6715b29 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/split_positional/script @@ -0,0 +1,53 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve -t dev +} +trap cleanup EXIT + +$CLI bundle deploy -t dev +pipeline_id="$(read_id.py split_pipeline)" + + +# Length-preserving field edits on both elements of a positional list that is +# split across two blocks. Each edit must land in its own block. +title "Edit num_workers on the top-level cluster and on the target cluster" +edit_resource.py pipelines $pipeline_id <