diff --git a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl index 3189b712d6b..29692b4450c 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl @@ -12,4 +12,7 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/deployment/bind/vector_search_index/output.txt b/acceptance/bundle/deployment/bind/vector_search_index/output.txt index db61161ad00..f1b79186906 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/output.txt +++ b/acceptance/bundle/deployment/bind/vector_search_index/output.txt @@ -5,7 +5,7 @@ "endpoint_type": "STANDARD" } ->>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"}} +>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"id\":\"integer\",\"vector\":\"array\"}","embedding_vector_columns":[{"name":"vector","embedding_dimension":768}]}} { "name": "main.default.test_vs_index_[UNIQUE_NAME]", "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", diff --git a/acceptance/bundle/deployment/bind/vector_search_index/script b/acceptance/bundle/deployment/bind/vector_search_index/script index d43578d94c0..3d07efacf93 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/script +++ b/acceptance/bundle/deployment/bind/vector_search_index/script @@ -11,7 +11,7 @@ trap cleanup EXIT trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}' -trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"}]}\"}}" | jq '{name, endpoint_name, index_type, primary_key}' +trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"id\\\":\\\"integer\\\",\\\"vector\\\":\\\"array\\\"}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":768}]}}" | jq '{name, endpoint_name, index_type, primary_key}' trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve diff --git a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl index bafb1a3cd15..3db7ae2d532 100644 --- a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl @@ -8,10 +8,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: foo: - name: test-index-$UNIQUE_NAME + name: main.default.test_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.bar.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl index 768334ee2a6..e16993d7019 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl @@ -11,10 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json index 0d6186c0ec2..c16d428a3f8 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json @@ -2,13 +2,18 @@ "method": "POST", "path": "/api/2.0/vector-search/indexes", "body": { - "delta_sync_index_spec": { - "pipeline_type": "TRIGGERED", - "source_table": "main.default.source_[UNIQUE_NAME]" + "direct_access_index_spec": { + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", - "name": "vs-index-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id" } } diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index d75bfea61db..227aa110ada 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -21,8 +21,8 @@ Resources: URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: - Name: vs-index-[UNIQUE_NAME] - URL: (not deployed) + Name: main.default.vs_index_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... @@ -30,11 +30,11 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } @@ -51,8 +51,8 @@ Resources: URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: - Name: vs-index-[UNIQUE_NAME] - URL: (not deployed) + Name: main.default.vs_index_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID] >>> print_requests.py //vector-search/indexes diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/script b/acceptance/bundle/resources/vector_search_indexes/basic/script index 54304337425..2e513d8ea6f 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/script +++ b/acceptance/bundle/resources/vector_search_indexes/basic/script @@ -14,7 +14,7 @@ rm -f out.requests.txt trace $CLI bundle deploy # Get index details -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' trace $CLI bundle summary diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl index 986f9507e01..43999b39c69 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl @@ -11,10 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index 58cd871d0ed..9b3d7aab90b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -7,7 +7,7 @@ Updating deployment state... Deployment complete! === Simulate remote deletion ->>> [CLI] vector-search-indexes delete-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes delete-index main.default.vs_index_[UNIQUE_NAME] === Plan detects missing resource and proposes creation >>> [CLI] bundle plan @@ -22,11 +22,11 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script index ae461b4758c..f92a79a1eff 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl index 1bf224ad2ec..d12336c7e94 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl @@ -11,10 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt index b8c384ba553..7d865eb14ea 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -28,11 +28,11 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", + "index_type": "DIRECT_ACCESS", "primary_key": "id" } diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script index b67193c34c9..d3470f38414 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml endpoint_name="vs-endpoint-${UNIQUE_NAME}" -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl index 855d2f1f141..c0619e93eed 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl @@ -16,7 +16,10 @@ resources: primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 grants: - principal: deco-test-user@databricks.com privileges: diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json index ecfc6b933aa..66c0fe79100 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json @@ -3,7 +3,13 @@ "path": "/api/2.0/vector-search/indexes", "body": { "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl similarity index 54% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl index 768334ee2a6..e16993d7019 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/databricks.yml.tmpl @@ -11,10 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json new file mode 100644 index 00000000000..880b58413d4 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.create.direct.json @@ -0,0 +1,23 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "direct_access_index_spec": { + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "main.default.vs_index_[UNIQUE_NAME]", + "primary_key": "id" + } +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json new file mode 100644 index 00000000000..ffce16dd741 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.requests.recreate.direct.json @@ -0,0 +1,35 @@ +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "direct_access_index_spec": { + "embedding_vector_columns": [ + { + "embedding_dimension": 384, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "main.default.vs_index_[UNIQUE_NAME]", + "primary_key": "id" + } +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml similarity index 100% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt similarity index 75% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt index 23ac9878b99..0c04db43f89 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt @@ -1,5 +1,5 @@ -=== Initial deployment with DELTA_SYNC index_type +=== Initial deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -8,14 +8,8 @@ Deployment complete! >>> print_requests.py --get //vector-search/indexes -=== Change index_type (should trigger recreation) ->>> update_file.py databricks.yml index_type: DELTA_SYNC index_type: DIRECT_ACCESS - ->>> update_file.py databricks.yml delta_sync_index_spec: direct_access_index_spec: - ->>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"}]}' - ->>> update_file.py databricks.yml pipeline_type: TRIGGERED +=== Change embedding_dimension (should trigger recreation) +>>> update_file.py databricks.yml embedding_dimension: 768 embedding_dimension: 384 >>> [CLI] bundle plan recreate vector_search_indexes.my_index @@ -35,9 +29,9 @@ Deployment complete! >>> print_requests.py --get //vector-search/indexes ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script similarity index 62% rename from acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script rename to acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script index 630628bd3b3..6c994cd4586 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/script @@ -16,17 +16,14 @@ print_requests() { rm -f out.requests.txt } -title "Initial deployment with DELTA_SYNC index_type" +title "Initial deployment" rm -f out.requests.txt trace $CLI bundle deploy print_requests create -title "Change index_type (should trigger recreation)" -trace update_file.py databricks.yml "index_type: DELTA_SYNC" "index_type: DIRECT_ACCESS" -trace update_file.py databricks.yml "delta_sync_index_spec:" "direct_access_index_spec:" -trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}'" -trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "" +title "Change embedding_dimension (should trigger recreation)" +trace update_file.py databricks.yml "embedding_dimension: 768" "embedding_dimension: 384" trace $CLI bundle plan rm -f out.requests.txt @@ -34,5 +31,5 @@ trace $CLI bundle deploy --auto-approve print_requests recreate -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json deleted file mode 100644 index 00d11a7272f..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/vector-search/indexes", - "body": { - "delta_sync_index_spec": { - "pipeline_type": "TRIGGERED", - "source_table": "main.default.source_[UNIQUE_NAME]" - }, - "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", - "name": "vs-index-[UNIQUE_NAME]", - "primary_key": "id" - } -} -{ - "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" -} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json deleted file mode 100644 index c00a4caa79b..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" -} -{ - "method": "DELETE", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" -} -{ - "method": "POST", - "path": "/api/2.0/vector-search/indexes", - "body": { - "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" - }, - "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DIRECT_ACCESS", - "name": "vs-index-[UNIQUE_NAME]", - "primary_key": "id" - } -} -{ - "method": "GET", - "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" -} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl index b6fde83e7fb..33438f1d351 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl @@ -11,9 +11,12 @@ resources: endpoint_type: STANDARD vector_search_indexes: my_index: - name: vs-index-$UNIQUE_NAME + name: main.default.vs_index_$UNIQUE_NAME endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: - schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index f5314dd9a0a..7838a374446 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -63,12 +63,18 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "remote_state": { "creator": "[USERNAME]", "direct_access_index_spec": { - "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + "embedding_vector_columns": [ + { + "embedding_dimension": 768, + "name": "vector" + } + ], + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", "index_type": "DIRECT_ACCESS", - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "primary_key": "id", "status": { "ready": true @@ -98,9 +104,9 @@ Deployment complete! "endpoint_type": "STORAGE_OPTIMIZED" } ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME] { - "name": "vs-index-[UNIQUE_NAME]", + "name": "main.default.vs_index_[UNIQUE_NAME]", "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "index_type": "DIRECT_ACCESS", "primary_key": "id" diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index bbc14f45176..b6170f5b613 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -20,7 +20,7 @@ trace $CLI bundle deploy --auto-approve # Verify final state of both resources. endpoint_name="vs-endpoint-${UNIQUE_NAME}" -index_name="vs-index-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f8aff6a93f6..f18a84d0efc 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -263,7 +263,7 @@ var testConfig map[string]any = map[string]any{ "vector_search_indexes": &resources.VectorSearchIndex{ CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "my-index", + Name: "main.default.my_index", EndpointName: "my-index-endpoint", PrimaryKey: "id", IndexType: vectorsearch.VectorIndexTypeDeltaSync, diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index c739dbfdeb9..9c99772fc30 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -4,10 +4,16 @@ import ( "encoding/json" "fmt" "net/http" + "regexp" + "strings" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) +// indexNamePart matches each catalog.schema.table component the real backend +// accepts: only alphanumerics and underscores. +var indexNamePart = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + // fakeVectorSearchIndex captures the endpoint's UUID at index creation time. // On the real backend an index is bound to a specific endpoint instance, not // just the name: deleting and recreating an endpoint with the same name yields @@ -31,6 +37,16 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } + if !isValidIndexName(createReq.Name) { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Invalid index name. Must specify the full index name ... Only alphanumerics and underscores are allowed.", + }, + } + } + if _, exists := s.VectorSearchIndexes[createReq.Name]; exists { return Response{ StatusCode: http.StatusConflict, @@ -71,6 +87,22 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } +// isValidIndexName checks that name is in catalog.schema.table form with +// only alphanumerics and underscores per UC, mirroring the backend's +// validation rejection at create time. +func isValidIndexName(name string) bool { + parts := strings.Split(name, ".") + if len(parts) != 3 { + return false + } + for _, p := range parts { + if !indexNamePart.MatchString(p) { + return false + } + } + return true +} + // remapDeltaSyncSpec converts a request spec to a response spec. func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vectorsearch.DeltaSyncVectorIndexSpecResponse { if req == nil {