Skip to content

Update test fixtures based on cloud #5366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
janniklasrose wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Update test fixtures based on cloud #5366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,7 @@ resources:
primary_key: id
index_type: DIRECT_ACCESS
direct_access_index_spec:
schema_json: '{"columns":[{"name":"id","type":"integer"}]}'
schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array<float>"}]}'
embedding_vector_columns:
- name: vector
embedding_dimension: 8
2 changes: 1 addition & 1 deletion acceptance/bundle/deployment/bind/vector_search_index/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"endpoint_type": "STANDARD"
}

>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"}}
>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array<float>\"}]}","embedding_vector_columns":[{"name":"vector","embedding_dimension":8}]}}
{
"name": "main.default.test_vs_index_[UNIQUE_NAME]",
"endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]",
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/deployment/bind/vector_search_index/script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ trap cleanup EXIT

trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}'

trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"}]}\"}}" | jq '{name, endpoint_name, index_type, primary_key}'
trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"},{\\\"name\\\":\\\"vector\\\",\\\"type\\\":\\\"array<float>\\\"}]}\",\"embedding_vector_columns\":[{\"name\":\"vector\",\"embedding_dimension\":8}]}}" | jq '{name, endpoint_name, index_type, primary_key}'

trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
foo:
name: test-index-$UNIQUE_NAME
name: main.default.test_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.bar.name}
primary_key: id
index_type: DELTA_SYNC
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
my_index:
name: vs-index-$UNIQUE_NAME
name: main.default.vs_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
primary_key: id
index_type: DELTA_SYNC
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
},
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DELTA_SYNC",
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"primary_key": "id"
}
}
12 changes: 6 additions & 6 deletions acceptance/bundle/resources/vector_search_indexes/basic/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,18 @@ Resources:
URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID]
Vector Search Indexes:
my_index:
Name: vs-index-[UNIQUE_NAME]
URL: (not deployed)
Name: main.default.vs_index_[UNIQUE_NAME]
URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID]

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files...
Deploying resources...
Updating deployment state...
Deployment complete!

>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
{
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DELTA_SYNC",
"primary_key": "id"
Expand All @@ -51,8 +51,8 @@ Resources:
URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID]
Vector Search Indexes:
my_index:
Name: vs-index-[UNIQUE_NAME]
URL: (not deployed)
Name: main.default.vs_index_[UNIQUE_NAME]
URL: [DATABRICKS_URL]/explore/data/main/default/vs_index_[UNIQUE_NAME]?o=[NUMID]

>>> print_requests.py //vector-search/indexes

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/basic/script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ rm -f out.requests.txt
trace $CLI bundle deploy

# Get index details
index_name="vs-index-${UNIQUE_NAME}"
index_name="main.default.vs_index_${UNIQUE_NAME}"
trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}'

trace $CLI bundle summary
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
my_index:
name: vs-index-$UNIQUE_NAME
name: main.default.vs_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
primary_key: id
index_type: DELTA_SYNC
Expand Down
6 changes: 3 additions & 3 deletions acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ Updating deployment state...
Deployment complete!

=== Simulate remote deletion
>>> [CLI] vector-search-indexes delete-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes delete-index main.default.vs_index_[UNIQUE_NAME]

=== Plan detects missing resource and proposes creation
>>> [CLI] bundle plan
Expand All @@ -22,9 +22,9 @@ Deploying resources...
Updating deployment state...
Deployment complete!

>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
{
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DELTA_SYNC",
"primary_key": "id"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
envsubst < databricks.yml.tmpl > databricks.yml
index_name="vs-index-${UNIQUE_NAME}"
index_name="main.default.vs_index_${UNIQUE_NAME}"

cleanup() {
trace $CLI bundle destroy --auto-approve
Expand Down
2 changes: 1 addition & 1 deletion ...ptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
my_index:
name: vs-index-$UNIQUE_NAME
name: main.default.vs_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name}
primary_key: id
index_type: DELTA_SYNC
Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ Deploying resources...
Updating deployment state...
Deployment complete!

>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
{
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DELTA_SYNC",
"primary_key": "id"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
envsubst < databricks.yml.tmpl > databricks.yml
endpoint_name="vs-endpoint-${UNIQUE_NAME}"
index_name="vs-index-${UNIQUE_NAME}"
index_name="main.default.vs_index_${UNIQUE_NAME}"

cleanup() {
trace $CLI bundle destroy --auto-approve
Expand Down
5 changes: 4 additions & 1 deletion acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ resources:
primary_key: id
index_type: DIRECT_ACCESS
direct_access_index_spec:
schema_json: '{"columns":[{"name":"id","type":"integer"}]}'
schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array<float>"}]}'
embedding_vector_columns:
- name: vector
embedding_dimension: 8
grants:
- principal: deco-test-user@databricks.com
privileges:
Expand Down
8 changes: 7 additions & 1 deletion acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@
"path": "/api/2.0/vector-search/indexes",
"body": {
"direct_access_index_spec": {
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"
"embedding_vector_columns": [
{
"embedding_dimension": 8,
"name": "vector"
}
],
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array<float>\"}]}"
},
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DIRECT_ACCESS",
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
my_index:
name: vs-index-$UNIQUE_NAME
name: main.default.vs_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
primary_key: id
index_type: DELTA_SYNC
Expand Down
4 changes: 2 additions & 2 deletions ...undle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
},
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DELTA_SYNC",
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"primary_key": "id"
}
}
{
"method": "GET",
"path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]"
"path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]"
}
18 changes: 12 additions & 6 deletions ...dle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
{
"method": "GET",
"path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]"
"path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]"
}
{
"method": "DELETE",
"path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]"
"path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]"
}
{
"method": "GET",
"path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]"
"path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]"
}
{
"method": "POST",
"path": "/api/2.0/vector-search/indexes",
"body": {
"direct_access_index_spec": {
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"
"embedding_vector_columns": [
{
"embedding_dimension": 8,
"name": "vector"
}
],
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array<float>\"}]}"
},
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DIRECT_ACCESS",
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"primary_key": "id"
}
}
{
"method": "GET",
"path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]"
"path": "/api/2.0/vector-search/indexes/main.default.vs_index_[UNIQUE_NAME]"
}
10 changes: 6 additions & 4 deletions acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@ Deployment complete!

>>> update_file.py databricks.yml delta_sync_index_spec: direct_access_index_spec:

>>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"}]}'
>>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array<float>"}]}'

>>> update_file.py databricks.yml pipeline_type: TRIGGERED
>>> update_file.py databricks.yml pipeline_type: TRIGGERED embedding_vector_columns:
- name: vector
embedding_dimension: 8

>>> [CLI] bundle plan
recreate vector_search_indexes.my_index
Expand All @@ -35,9 +37,9 @@ Deployment complete!

>>> print_requests.py --get //vector-search/indexes

>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
{
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DIRECT_ACCESS",
"primary_key": "id"
Expand Down
8 changes: 5 additions & 3 deletions acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@ print_requests create
title "Change index_type (should trigger recreation)"
trace update_file.py databricks.yml "index_type: DELTA_SYNC" "index_type: DIRECT_ACCESS"
trace update_file.py databricks.yml "delta_sync_index_spec:" "direct_access_index_spec:"
trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}'"
trace update_file.py databricks.yml "pipeline_type: TRIGGERED" ""
trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array<float>\"}]}'"
trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "embedding_vector_columns:
- name: vector
embedding_dimension: 8"

trace $CLI bundle plan
rm -f out.requests.txt
trace $CLI bundle deploy --auto-approve

print_requests recreate

index_name="vs-index-${UNIQUE_NAME}"
index_name="main.default.vs_index_${UNIQUE_NAME}"
trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}'
7 changes: 5 additions & 2 deletions acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,12 @@ resources:
endpoint_type: STANDARD
vector_search_indexes:
my_index:
name: vs-index-$UNIQUE_NAME
name: main.default.vs_index_$UNIQUE_NAME
endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
primary_key: id
index_type: DIRECT_ACCESS
direct_access_index_spec:
schema_json: '{"columns":[{"name":"id","type":"integer"}]}'
schema_json: '{"columns":[{"name":"id","type":"integer"},{"name":"vector","type":"array<float>"}]}'
embedding_vector_columns:
- name: vector
embedding_dimension: 8
14 changes: 10 additions & 4 deletions acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,18 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged
"remote_state": {
"creator": "[USERNAME]",
"direct_access_index_spec": {
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"
"embedding_vector_columns": [
{
"embedding_dimension": 8,
"name": "vector"
}
],
"schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"},{\"name\":\"vector\",\"type\":\"array\u003cfloat\u003e\"}]}"
},
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"endpoint_uuid": "[UUID]",
"index_type": "DIRECT_ACCESS",
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"primary_key": "id",
"status": {
"ready": true
Expand Down Expand Up @@ -98,9 +104,9 @@ Deployment complete!
"endpoint_type": "STORAGE_OPTIMIZED"
}

>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME]
>>> [CLI] vector-search-indexes get-index main.default.vs_index_[UNIQUE_NAME]
{
"name": "vs-index-[UNIQUE_NAME]",
"name": "main.default.vs_index_[UNIQUE_NAME]",
"endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
"index_type": "DIRECT_ACCESS",
"primary_key": "id"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ trace $CLI bundle deploy --auto-approve

# Verify final state of both resources.
endpoint_name="vs-endpoint-${UNIQUE_NAME}"
index_name="vs-index-${UNIQUE_NAME}"
index_name="main.default.vs_index_${UNIQUE_NAME}"
trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}'
trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}'

Expand Down
Loading