From ebf1a912b54c32df19857482da6ade6270a70a8f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 27 May 2026 16:27:22 +0200 Subject: [PATCH 1/4] Update terraform vector search index reference commit hash --- bundle/direct/dresources/vector_search_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 48ee6f0f96..2d6fab3108 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -24,7 +24,7 @@ const deleteIndexTimeout = 15 * time.Minute // createIndexTimeout caps the wait for an index to become ready after creation. // Delta sync indexes do an initial sync from the source table, which can stretch // out for large tables. Matches the terraform provider's defaultIndexProvisionTimeout. -// https://github.com/databricks/terraform-provider-databricks/blob/c61a32300445f84efb2bb6827dee35e6e523f4ff/vectorsearch/resource_vector_search_index.go#L19 +// https://github.com/databricks/terraform-provider-databricks/blob/c79d82d9582ab6670468bbff303199906d47905f/vectorsearch/resource_vector_search_index.go#L19 const createIndexTimeout = 75 * time.Minute // VectorSearchIndexState tracks the UUID of the endpoint the index is attached From c8fd31311501d114564ad4f591de080a2ff716bf Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 28 May 2026 10:32:08 +0200 Subject: [PATCH 2/4] Use endpoint_id from vector search index response Drop the separate VectorSearchEndpoints.GetEndpointByEndpointName call from DoRead/DoCreate and read endpoint_id directly off the VectorIndex response. This removes the race noted in #5123 where the index's endpoint and the lookup-by-name could disagree if the endpoint was deleted/recreated between the two calls. Won't compile until the SDK gains EndpointId on VectorIndex. --- .../direct/dresources/vector_search_index.go | 58 ++++--------------- libs/testserver/fake_workspace.go | 4 +- libs/testserver/vector_search_indexes.go | 40 +++++-------- 3 files changed, 29 insertions(+), 73 deletions(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 2d6fab3108..ef30f3cd8a 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -3,7 +3,6 @@ package dresources import ( "context" "errors" - "fmt" "time" "github.com/databricks/cli/bundle/config/resources" @@ -44,8 +43,9 @@ func (s VectorSearchIndexState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// VectorSearchIndexRemote is remote state. endpoint_uuid is looked up from the -// endpoint service since the index API itself doesn't return it. +// VectorSearchIndexRemote is remote state. endpoint_uuid mirrors the index +// response's endpoint_id so OverrideChangeDesc can compare it against the +// saved state without a second API call. type VectorSearchIndexRemote struct { vectorsearch.VectorIndex EndpointUuid string `json:"endpoint_uuid,omitempty"` @@ -111,13 +111,9 @@ func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*Vec if err != nil { return nil, err } - endpointUuid, err := r.lookupEndpointUuid(ctx, index.EndpointName) - if err != nil { - return nil, err - } return &VectorSearchIndexRemote{ VectorIndex: *index, - EndpointUuid: endpointUuid, + EndpointUuid: index.EndpointId, }, nil } @@ -126,16 +122,8 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector if err != nil { return "", nil, err } - // Second API call (also done in DoRead): the index API does not return the - // endpoint UUID, but we need to persist it in state so a future plan can - // detect that the endpoint was replaced out-of-band (same name, different - // UUID -> orphan). - endpointUuid, err := r.lookupEndpointUuid(ctx, config.EndpointName) - if err != nil { - return "", nil, err - } - config.EndpointUuid = endpointUuid - return config.Name, &VectorSearchIndexRemote{VectorIndex: *index, EndpointUuid: endpointUuid}, nil + config.EndpointUuid = index.EndpointId + return config.Name, &VectorSearchIndexRemote{VectorIndex: *index, EndpointUuid: index.EndpointId}, nil } // No DoUpdate: vector search indexes have no update API. All SDK fields are @@ -193,17 +181,13 @@ func (r *ResourceVectorSearchIndex) WaitAfterDelete(ctx context.Context, id stri } // OverrideChangeDesc classifies endpoint_uuid drift: Recreate when the saved -// UUID differs from what's currently attached to the endpoint name, Skip -// otherwise. endpoint_uuid is never present in config, so without Skip a -// synthetic diff between empty newState and populated saved state would -// otherwise leak into the plan. +// UUID differs from the endpoint_id the index API now reports, Skip otherwise. +// endpoint_uuid is never present in config, so without Skip a synthetic diff +// between empty newState and populated saved state would leak into the plan. // -// Unlike vector_search_endpoint, this intentionally does NOT require -// remoteUuid != "". An empty remoteUuid here is the orphan signal: the index -// still exists by name but its backing endpoint has been deleted out-of-band. -// lookupEndpointUuid distinguishes this (404 -> "") from transient errors -// (propagated through DoRead/DoCreate), so reaching this branch with empty -// remoteUuid unambiguously means the endpoint is gone. +// An empty remoteUuid is treated as drift (rather than ignored, as the endpoint +// resource does for its own UUID). If the backend ever clears endpoint_id when +// the endpoint is deleted out-of-band, this surfaces the orphan as a recreate. func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchIndexRemote) error { if path.String() != "endpoint_uuid" { return nil @@ -222,21 +206,3 @@ func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *st } return nil } - -// lookupEndpointUuid returns the current UUID of the endpoint with the given -// name. A 404 is converted to ("", nil) so the caller can distinguish a -// genuinely missing endpoint (the orphan signal) from a transient or -// permission error, which is propagated. -func (r *ResourceVectorSearchIndex) lookupEndpointUuid(ctx context.Context, endpointName string) (string, error) { - if endpointName == "" { - return "", nil - } - info, err := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, endpointName) - if err != nil { - if apierr.IsMissing(err) { - return "", nil - } - return "", fmt.Errorf("looking up vector search endpoint %q: %w", endpointName, err) - } - return info.Id, nil -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ff70f6b050..217f4d9506 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -152,7 +152,7 @@ type FakeWorkspace struct { RegisteredModels map[string]catalog.RegisteredModelInfo ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo - VectorSearchIndexes map[string]fakeVectorSearchIndex + VectorSearchIndexes map[string]vectorsearch.VectorIndex SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value @@ -297,7 +297,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - VectorSearchIndexes: map[string]fakeVectorSearchIndex{}, + VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, Repos: map[string]workspace.RepoInfo{}, SecretScopes: map[string]workspace.SecretScope{}, Secrets: map[string]map[string]string{}, diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index c739dbfdeb..8063e9118b 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -8,18 +8,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) -// fakeVectorSearchIndex captures the endpoint's UUID at index creation time. -// On the real backend an index is bound to a specific endpoint instance, not -// just the name: deleting and recreating an endpoint with the same name yields -// a different UUID, and the existing index keeps pointing at the OLD UUID -// (i.e. is orphaned). Tracking this here lets tests reason about that drift. -// The field is omitted from JSON responses since the real API doesn't return -// it on the index path; the CLI looks it up via GetEndpointByEndpointName. -type fakeVectorSearchIndex struct { - vectorsearch.VectorIndex - EndpointUuid string `json:"-"` -} - func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { defer s.LockUnlock()() @@ -48,20 +36,22 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } - index := fakeVectorSearchIndex{ - VectorIndex: vectorsearch.VectorIndex{ - Creator: s.CurrentUser().UserName, - EndpointName: createReq.EndpointName, - IndexType: createReq.IndexType, - Name: createReq.Name, - PrimaryKey: createReq.PrimaryKey, - DeltaSyncIndexSpec: remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec), - DirectAccessIndexSpec: createReq.DirectAccessIndexSpec, - Status: &vectorsearch.VectorIndexStatus{ - Ready: true, - }, + // EndpointId is captured at index creation time. On the real backend an + // index is bound to a specific endpoint instance, not just the name: + // deleting and recreating an endpoint with the same name yields a + // different UUID. Storing it lets tests reason about that drift. + index := vectorsearch.VectorIndex{ + Creator: s.CurrentUser().UserName, + EndpointId: endpoint.Id, + EndpointName: createReq.EndpointName, + IndexType: createReq.IndexType, + Name: createReq.Name, + PrimaryKey: createReq.PrimaryKey, + DeltaSyncIndexSpec: remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec), + DirectAccessIndexSpec: createReq.DirectAccessIndexSpec, + Status: &vectorsearch.VectorIndexStatus{ + Ready: true, }, - EndpointUuid: endpoint.Id, } s.VectorSearchIndexes[createReq.Name] = index From d9c89faf7b1e396c873bbb70ed7da22a3b23ec5f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 28 May 2026 10:38:53 +0200 Subject: [PATCH 3/4] Use VectorSearchEndpointPermissionLevel for VS endpoint permissions The SDK exposes a dedicated permission level type for vector search endpoints; switch VectorSearchEndpoint.Permissions to use it via the PermissionT[L] generic, matching the typing already in place for jobs, apps, model serving, etc. Regenerated jsonschema.json and added the annotations placeholder for the new VectorSearchEndpointPermission. --- .../apply_bundle_permissions_test.go | 8 ++-- bundle/config/resources/permission_types.go | 2 + .../resources/vector_search_endpoint.go | 2 +- bundle/internal/schema/annotations.yml | 13 +++++ bundle/schema/jsonschema.json | 48 ++++++++++++++++++- 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 3fa97c41a7..cd253ce4d9 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -147,12 +147,12 @@ func TestApplyBundlePermissions(t *testing.T) { require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.AppPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) } func TestWarningOnOverlapPermission(t *testing.T) { diff --git a/bundle/config/resources/permission_types.go b/bundle/config/resources/permission_types.go index 3029ee40b8..769a61376c 100644 --- a/bundle/config/resources/permission_types.go +++ b/bundle/config/resources/permission_types.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) // Each resource defines its own permission type so that the JSON schema names them distinctly. @@ -32,4 +33,5 @@ type ( ModelServingEndpointPermission PermissionT[serving.ServingEndpointPermissionLevel] PipelinePermission PermissionT[pipelines.PipelinePermissionLevel] SqlWarehousePermission PermissionT[sql.WarehousePermissionLevel] + VectorSearchEndpointPermission PermissionT[vectorsearch.VectorSearchEndpointPermissionLevel] ) diff --git a/bundle/config/resources/vector_search_endpoint.go b/bundle/config/resources/vector_search_endpoint.go index 900e91ccc5..912e39898d 100644 --- a/bundle/config/resources/vector_search_endpoint.go +++ b/bundle/config/resources/vector_search_endpoint.go @@ -15,7 +15,7 @@ type VectorSearchEndpoint struct { BaseResource vectorsearch.CreateEndpoint - Permissions []Permission `json:"permissions,omitempty"` + Permissions []VectorSearchEndpointPermission `json:"permissions,omitempty"` } func (e *VectorSearchEndpoint) UnmarshalJSON(b []byte) error { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 6a1070ddcd..d18e607731 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1051,6 +1051,19 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: "usage_policy_id": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission: + "group_name": + "description": |- + PLACEHOLDER + "level": + "description": |- + PLACEHOLDER + "service_principal_name": + "description": |- + PLACEHOLDER + "user_name": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: "delta_sync_index_spec": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c2026fc43b..225b7bcb93 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2041,7 +2041,7 @@ "$ref": "#/$defs/string" }, "permissions": { - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission" }, "target_qps": { "$ref": "#/$defs/int64" @@ -2064,6 +2064,35 @@ } ] }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorSearchEndpointPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.VectorSearchIndex": { "oneOf": [ { @@ -11955,6 +11984,9 @@ "vectorsearch.VectorIndexType": { "type": "string" }, + "vectorsearch.VectorSearchEndpointPermissionLevel": { + "type": "string" + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "oneOf": [ { @@ -12704,6 +12736,20 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "config.ArtifactFile": { From 41ae831c8d2fc9c1c5f0c2ecb8e5e42ce325072e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 28 May 2026 10:41:26 +0200 Subject: [PATCH 4/4] testserver: echo columns_to_sync/columns_to_index on VS index reads Production RemapState already round-trips both fields from the GET response, but the testserver was dropping them. Bring the fake in line with the real backend so acceptance tests see the same shape on read. --- libs/testserver/vector_search_indexes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index 8063e9118b..3500b2fcc7 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -67,6 +67,8 @@ func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vect return nil } return &vectorsearch.DeltaSyncVectorIndexSpecResponse{ + ColumnsToIndex: req.ColumnsToIndex, + ColumnsToSync: req.ColumnsToSync, EmbeddingSourceColumns: req.EmbeddingSourceColumns, EmbeddingVectorColumns: req.EmbeddingVectorColumns, EmbeddingWritebackTable: req.EmbeddingWritebackTable,