From f3da03fb3da2f63c5cb5c167fb06db79105d97a5 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 28 May 2026 12:37:55 +0200 Subject: [PATCH 1/5] Update test fixtures based on cloud --- .../vector_search_index/databricks.yml.tmpl | 5 ++++- .../bind/vector_search_index/output.txt | 2 +- .../deployment/bind/vector_search_index/script | 2 +- .../configs/vector_search_index.yml.tmpl | 2 +- .../basic/databricks.yml.tmpl | 2 +- .../basic/out.requests.direct.json | 2 +- .../vector_search_indexes/basic/output.txt | 12 ++++++------ .../vector_search_indexes/basic/script | 2 +- .../drift/deleted_remotely/databricks.yml.tmpl | 2 +- .../drift/deleted_remotely/output.txt | 6 +++--- .../drift/deleted_remotely/script | 2 +- .../orphaned_endpoint/databricks.yml.tmpl | 2 +- .../drift/orphaned_endpoint/output.txt | 4 ++-- .../drift/orphaned_endpoint/script | 2 +- .../grants/select/databricks.yml.tmpl | 5 ++++- .../grants/select/out.requests.direct.json | 8 +++++++- .../recreate/index_type/databricks.yml.tmpl | 2 +- .../index_type/out.requests.create.direct.json | 4 ++-- .../out.requests.recreate.direct.json | 18 ++++++++++++------ .../recreate/index_type/output.txt | 10 ++++++---- .../recreate/index_type/script | 8 +++++--- .../recreate/with_endpoint/databricks.yml.tmpl | 7 +++++-- .../recreate/with_endpoint/output.txt | 14 ++++++++++---- .../recreate/with_endpoint/script | 2 +- 24 files changed, 78 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl index 3189b712d6b..4e3f82a7bf7 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl @@ -12,4 +12,7 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + embedding_vector_columns: + - name: vector + embedding_dimension: 8 diff --git a/acceptance/bundle/deployment/bind/vector_search_index/output.txt b/acceptance/bundle/deployment/bind/vector_search_index/output.txt index db61161ad00..498d3034d0d 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/output.txt +++ b/acceptance/bundle/deployment/bind/vector_search_index/output.txt @@ -5,7 +5,7 @@ "endpoint_type": "STANDARD" } ->>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"}} +>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}","embedding_vector_columns":[{"name":"vector","embedding_dimension":8}]}} { "name": "main.default.test_vs_index_[UNIQUE_NAME]", "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", diff --git a/acceptance/bundle/deployment/bind/vector_search_index/script b/acceptance/bundle/deployment/bind/vector_search_index/script index d43578d94c0..d28aa00a783 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/script +++ b/acceptance/bundle/deployment/bind/vector_search_index/script @@ -11,7 +11,7 @@ trap cleanup EXIT trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}' -trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"}]}\"}}" | jq '{name, endpoint_name, index_type, primary_key}' +trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"},{\\\"name\\\":\\\"vector\\\",\\\"type\\\":\\\"array\\\"}]}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":8}]}}" | jq '{name, endpoint_name, index_type, primary_key}' trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve diff --git a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl index bafb1a3cd15..b897ad202c6 100644 --- a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl @@ -8,7 +8,7 @@ resources: endpoint_type: STANDARD vector_search_indexes: foo: - name: test-index-$UNIQUE_NAME + name: main.default.test_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.bar.name} primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl index 768334ee2a6..7553b2260f4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl @@ -11,7 +11,7 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json index 0d6186c0ec2..519edb8fb58 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json @@ -8,7 +8,7 @@ }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DELTA_SYNC", - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } } diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index d75bfea61db..5d84aaa0770 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -21,8 +21,8 @@ Resources: URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: - Name: vs-index-[UNIQUE_NAME] - URL: (not deployed) + Name: main.default.vs_index_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... @@ -30,9 +30,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DELTA_SYNC", "primary_key": "id" @@ -51,8 +51,8 @@ Resources: URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: - Name: vs-index-[UNIQUE_NAME] - URL: (not deployed) + Name: main.default.vs_index_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID] >>> print_requests.py //vector-search/indexes diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/script b/acceptance/bundle/resources/vector_search_indexes/basic/script index 54304337425..2e513d8ea6f 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/script +++ b/acceptance/bundle/resources/vector_search_indexes/basic/script @@ -14,7 +14,7 @@ rm -f out.requests.txt trace $CLI bundle deploy # Get index details -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' trace $CLI bundle summary diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl index 986f9507e01..3ad7bd96ff0 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl @@ -11,7 +11,7 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index 58cd871d0ed..674e78f24b0 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -7,7 +7,7 @@ Updating deployment state... Deployment complete! === Simulate remote deletion ->>> [CLI] vector-search-indexes delete-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes delete-index main.default.vs_index_[UNIQUE_NAME] === Plan detects missing resource and proposes creation >>> [CLI] bundle plan @@ -22,9 +22,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DELTA_SYNC", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script index ae461b4758c..f92a79a1eff 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl index 1bf224ad2ec..0ca226d9d53 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl @@ -11,7 +11,7 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt index b8c384ba553..b18c77ec4fd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -28,9 +28,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DELTA_SYNC", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script index b67193c34c9..d3470f38414 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml endpoint_name="vs-endpoint-${UNIQUE_NAME}" -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl index 855d2f1f141..0d2e48d5c17 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl @@ -16,7 +16,10 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + embedding_vector_columns: + - name: vector + embedding_dimension: 8 grants: - principal: deco-test-user@databricks.com privileges: diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json index ecfc6b933aa..b142b41ceb0 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json @@ -3,7 +3,13 @@ "path": "/api/2.0/vector-search/indexes", "body": { "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + "embedding_vector_columns": [ + { + "embedding_dimension": 8, + "name": "vector" + } + ], + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl index 768334ee2a6..7553b2260f4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl @@ -11,7 +11,7 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json index 00d11a7272f..38460bec644 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json @@ -8,11 +8,11 @@ }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DELTA_SYNC", - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } } { "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" } diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json index c00a4caa79b..60b400f866d 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json @@ -1,29 +1,35 @@ { "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" } { "method": "DELETE", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" } { "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" } { "method": "POST", "path": "/api/2.0/vector-search/indexes", "body": { "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + "embedding_vector_columns": [ + { + "embedding_dimension": 8, + "name": "vector" + } + ], + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } } { "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" } diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt index 23ac9878b99..929b0d371f6 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -13,9 +13,11 @@ Deployment complete! >>> update_file.py databricks.yml delta_sync_index_spec: direct_access_index_spec: ->>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"}]}' +>>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' ->>> update_file.py databricks.yml pipeline_type: TRIGGERED +>>> update_file.py databricks.yml pipeline_type: TRIGGERED embedding_vector_columns: + - name: vector + embedding_dimension: 8 >>> [CLI] bundle plan recreate vector_search_indexes.my_index @@ -35,9 +37,9 @@ Deployment complete! >>> print_requests.py --get //vector-search/indexes ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script index 630628bd3b3..73919b286d7 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script @@ -25,8 +25,10 @@ print_requests create title "Change index_type (should trigger recreation)" trace update_file.py databricks.yml "index_type: DELTA_SYNC" "index_type: DIRECT_ACCESS" trace update_file.py databricks.yml "delta_sync_index_spec:" "direct_access_index_spec:" -trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}'" -trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "" +trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}'" +trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "embedding_vector_columns: + - name: vector + embedding_dimension: 8" trace $CLI bundle plan rm -f out.requests.txt @@ -34,5 +36,5 @@ trace $CLI bundle deploy --auto-approve print_requests recreate -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl index b6fde83e7fb..e6063af44dd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl @@ -11,9 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + embedding_vector_columns: + - name: vector + embedding_dimension: 8 diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index f5314dd9a0a..9fc7425cbf0 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -63,12 +63,18 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "remote_state": { "creator": "[USERNAME]", "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + "embedding_vector_columns": [ + { + "embedding_dimension": 8, + "name": "vector" + } + ], + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\u003cfloat\u003e\"}]}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", "index_type": "DIRECT_ACCESS", - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id", "status": { "ready": true @@ -98,9 +104,9 @@ Deployment complete! "endpoint_type": "STORAGE_OPTIMIZED" } ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index bbc14f45176..b6170f5b613 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -20,7 +20,7 @@ trace $CLI bundle deploy --auto-approve # Verify final state of both resources. endpoint_name="vs-endpoint-${UNIQUE_NAME}" -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' From 2a10ba53314b2c72d94a790a5fbc1ff08c6e3309 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 1 Jun 2026 00:52:44 +0200 Subject: [PATCH 2/5] Change recreate index_type test to embedding_dimension --- .../databricks.yml.tmpl | 10 ++++++---- .../out.requests.create.direct.json | 13 +++++++++---- .../out.requests.recreate.direct.json | 4 ++-- .../out.test.toml | 0 .../{index_type => embedding_dimension}/output.txt | 14 +++----------- .../{index_type => embedding_dimension}/script | 11 +++-------- 6 files changed, 23 insertions(+), 29 deletions(-) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/databricks.yml.tmpl (62%) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/out.requests.create.direct.json (53%) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/out.requests.recreate.direct.json (82%) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/out.test.toml (100%) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/output.txt (75%) rename acceptance/bundle/resources/vector_search_indexes/recreate/{index_type => embedding_dimension}/script (60%) diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl similarity index 62% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl index 7553b2260f4..e16993d7019 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl @@ -14,7 +14,9 @@ resources: name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json similarity index 53% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json index 38460bec644..880b58413d4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json @@ -2,12 +2,17 @@ "method": "POST", "path": "/api/2.0/vector-search/indexes", "body": { - "delta_sync_index_spec": { - "pipeline_type": "TRIGGERED", - "source_table": "main.default.source_[UNIQUE_NAME]" + "direct_access_index_spec": { + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json similarity index 82% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json index 60b400f866d..ffce16dd741 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json @@ -17,11 +17,11 @@ "direct_access_index_spec": { "embedding_vector_columns": [ { - "embedding_dimension": 8, + "embedding_dimension": 384, "name": "vector" } ], - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}" + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml similarity index 100% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt similarity index 75% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt index 929b0d371f6..0c04db43f89 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt @@ -1,5 +1,5 @@ -=== Initial deployment with DELTA_SYNC index_type +=== Initial deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -8,16 +8,8 @@ Deployment complete! >>> print_requests.py --get //vector-search/indexes -=== Change index_type (should trigger recreation) ->>> update_file.py databricks.yml index_type: DELTA_SYNC index_type: DIRECT_ACCESS - ->>> update_file.py databricks.yml delta_sync_index_spec: direct_access_index_spec: - ->>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' - ->>> update_file.py databricks.yml pipeline_type: TRIGGERED embedding_vector_columns: - - name: vector - embedding_dimension: 8 +=== Change embedding_dimension (should trigger recreation) +>>> update_file.py databricks.yml embedding_dimension: 768 embedding_dimension: 384 >>> [CLI] bundle plan recreate vector_search_indexes.my_index diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script similarity index 60% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script index 73919b286d7..6c994cd4586 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script @@ -16,19 +16,14 @@ print_requests() { rm -f out.requests.txt } -title "Initial deployment with DELTA_SYNC index_type" +title "Initial deployment" rm -f out.requests.txt trace $CLI bundle deploy print_requests create -title "Change index_type (should trigger recreation)" -trace update_file.py databricks.yml "index_type: DELTA_SYNC" "index_type: DIRECT_ACCESS" -trace update_file.py databricks.yml "delta_sync_index_spec:" "direct_access_index_spec:" -trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}'" -trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "embedding_vector_columns: - - name: vector - embedding_dimension: 8" +title "Change embedding_dimension (should trigger recreation)" +trace update_file.py databricks.yml "embedding_dimension: 768" "embedding_dimension: 384" trace $CLI bundle plan rm -f out.requests.txt From fa71800df8da5fa08684e25a2c1d965a5f5aaf7c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 1 Jun 2026 00:54:51 +0200 Subject: [PATCH 3/5] Use DIRECT_ACCESS instead of DELTA_SYNC --- .../bind/vector_search_index/databricks.yml.tmpl | 4 ++-- .../deployment/bind/vector_search_index/output.txt | 2 +- .../deployment/bind/vector_search_index/script | 2 +- .../invariant/configs/vector_search_index.yml.tmpl | 10 ++++++---- .../vector_search_indexes/basic/databricks.yml.tmpl | 10 ++++++---- .../basic/out.requests.direct.json | 13 +++++++++---- .../vector_search_indexes/basic/output.txt | 2 +- .../drift/deleted_remotely/databricks.yml.tmpl | 10 ++++++---- .../drift/deleted_remotely/output.txt | 2 +- .../drift/orphaned_endpoint/databricks.yml.tmpl | 10 ++++++---- .../drift/orphaned_endpoint/output.txt | 2 +- .../grants/select/databricks.yml.tmpl | 4 ++-- .../grants/select/out.requests.direct.json | 4 ++-- .../recreate/with_endpoint/databricks.yml.tmpl | 4 ++-- .../recreate/with_endpoint/output.txt | 4 ++-- 15 files changed, 48 insertions(+), 35 deletions(-) diff --git a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl index 4e3f82a7bf7..29692b4450c 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl @@ -12,7 +12,7 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + schema_json: '{"id":"integer","vector":"array"}' embedding_vector_columns: - name: vector - embedding_dimension: 8 + embedding_dimension: 768 diff --git a/acceptance/bundle/deployment/bind/vector_search_index/output.txt b/acceptance/bundle/deployment/bind/vector_search_index/output.txt index 498d3034d0d..f1b79186906 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/output.txt +++ b/acceptance/bundle/deployment/bind/vector_search_index/output.txt @@ -5,7 +5,7 @@ "endpoint_type": "STANDARD" } ->>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}","embedding_vector_columns":[{"name":"vector","embedding_dimension":8}]}} +>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"id\":\"integer\",\"vector\":\"array\"}","embedding_vector_columns":[{"name":"vector","embedding_dimension":768}]}} { "name": "main.default.test_vs_index_[UNIQUE_NAME]", "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", diff --git a/acceptance/bundle/deployment/bind/vector_search_index/script b/acceptance/bundle/deployment/bind/vector_search_index/script index d28aa00a783..3d07efacf93 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/script +++ b/acceptance/bundle/deployment/bind/vector_search_index/script @@ -11,7 +11,7 @@ trap cleanup EXIT trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}' -trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"},{\\\"name\\\":\\\"vector\\\",\\\"type\\\":\\\"array\\\"}]}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":8}]}}" | jq '{name, endpoint_name, index_type, primary_key}' +trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"id\\\":\\\"integer\\\",\\\"vector\\\":\\\"array\\\"}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":768}]}}" | jq '{name, endpoint_name, index_type, primary_key}' trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve diff --git a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl index b897ad202c6..3db7ae2d532 100644 --- a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl @@ -11,7 +11,9 @@ resources: name: main.default.test_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.bar.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl index 7553b2260f4..e16993d7019 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl @@ -14,7 +14,9 @@ resources: name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json index 519edb8fb58..c16d428a3f8 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json @@ -2,12 +2,17 @@ "method": "POST", "path": "/api/2.0/vector-search/indexes", "body": { - "delta_sync_index_spec": { - "pipeline_type": "TRIGGERED", - "source_table": "main.default.source_[UNIQUE_NAME]" + "direct_access_index_spec": { + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index 5d84aaa0770..227aa110ada 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -34,7 +34,7 @@ Deployment complete! { "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl index 3ad7bd96ff0..43999b39c69 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl @@ -14,7 +14,9 @@ resources: name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index 674e78f24b0..9b3d7aab90b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -26,7 +26,7 @@ Deployment complete! { "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl index 0ca226d9d53..d12336c7e94 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl @@ -14,7 +14,9 @@ resources: name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt index b18c77ec4fd..7d865eb14ea 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -32,7 +32,7 @@ Deployment complete! { "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl index 0d2e48d5c17..c0619e93eed 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl @@ -16,10 +16,10 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + schema_json: '{"id":"integer","vector":"array"}' embedding_vector_columns: - name: vector - embedding_dimension: 8 + embedding_dimension: 768 grants: - principal: deco-test-user@databricks.com privileges: diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json index b142b41ceb0..66c0fe79100 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json @@ -5,11 +5,11 @@ "direct_access_index_spec": { "embedding_vector_columns": [ { - "embedding_dimension": 8, + "embedding_dimension": 768, "name": "vector" } ], - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\"}]}" + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl index e6063af44dd..33438f1d351 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl @@ -16,7 +16,7 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array"}]}' + schema_json: '{"id":"integer","vector":"array"}' embedding_vector_columns: - name: vector - embedding_dimension: 8 + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 9fc7425cbf0..7838a374446 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -65,11 +65,11 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "direct_access_index_spec": { "embedding_vector_columns": [ { - "embedding_dimension": 8, + "embedding_dimension": 768, "name": "vector" } ], - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\u003cfloat\u003e\"}]}" + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", From 797aeaac87d7abc53d73ea6e58bf83d0f1b596c1 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 1 Jun 2026 00:55:28 +0200 Subject: [PATCH 4/5] Validate index name in testserver --- libs/testserver/vector_search_indexes.go | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index c739dbfdeb9..9c99772fc30 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -4,10 +4,16 @@ import ( "encoding/json" "fmt" "net/http" + "regexp" + "strings" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) +// indexNamePart matches each catalog.schema.table component the real backend +// accepts: only alphanumerics and underscores. +var indexNamePart = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + // fakeVectorSearchIndex captures the endpoint's UUID at index creation time. // On the real backend an index is bound to a specific endpoint instance, not // just the name: deleting and recreating an endpoint with the same name yields @@ -31,6 +37,16 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } + if !isValidIndexName(createReq.Name) { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Invalid index name. Must specify the full index name ... Only alphanumerics and underscores are allowed.", + }, + } + } + if _, exists := s.VectorSearchIndexes[createReq.Name]; exists { return Response{ StatusCode: http.StatusConflict, @@ -71,6 +87,22 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } +// isValidIndexName checks that name is in catalog.schema.table form with +// only alphanumerics and underscores per UC, mirroring the backend's +// validation rejection at create time. +func isValidIndexName(name string) bool { + parts := strings.Split(name, ".") + if len(parts) != 3 { + return false + } + for _, p := range parts { + if !indexNamePart.MatchString(p) { + return false + } + } + return true +} + // remapDeltaSyncSpec converts a request spec to a response spec. func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vectorsearch.DeltaSyncVectorIndexSpecResponse { if req == nil { From 87ed77839cf2b67c8a0961e38eab5cc2fa583ae2 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 1 Jun 2026 10:43:30 +0200 Subject: [PATCH 5/5] Update unit test with correct index naming --- bundle/direct/dresources/all_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f8aff6a93f6..f18a84d0efc 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -263,7 +263,7 @@ var testConfig map[string]any = map[string]any{ "vector_search_indexes": &resources.VectorSearchIndex{ CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "my-index", + Name: "main.default.my_index", EndpointName: "my-index-endpoint", PrimaryKey: "id", IndexType: vectorsearch.VectorIndexTypeDeltaSync,