diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 3fa97c41a7..cd253ce4d9 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -147,12 +147,12 @@ func TestApplyBundlePermissions(t *testing.T) { require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.AppPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) } func TestWarningOnOverlapPermission(t *testing.T) { diff --git a/bundle/config/resources/permission_types.go b/bundle/config/resources/permission_types.go index 3029ee40b8..769a61376c 100644 --- a/bundle/config/resources/permission_types.go +++ b/bundle/config/resources/permission_types.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) // Each resource defines its own permission type so that the JSON schema names them distinctly. @@ -32,4 +33,5 @@ type ( ModelServingEndpointPermission PermissionT[serving.ServingEndpointPermissionLevel] PipelinePermission PermissionT[pipelines.PipelinePermissionLevel] SqlWarehousePermission PermissionT[sql.WarehousePermissionLevel] + VectorSearchEndpointPermission PermissionT[vectorsearch.VectorSearchEndpointPermissionLevel] ) diff --git a/bundle/config/resources/vector_search_endpoint.go b/bundle/config/resources/vector_search_endpoint.go index 900e91ccc5..912e39898d 100644 --- a/bundle/config/resources/vector_search_endpoint.go +++ b/bundle/config/resources/vector_search_endpoint.go @@ -15,7 +15,7 @@ type VectorSearchEndpoint struct { BaseResource vectorsearch.CreateEndpoint - Permissions []Permission `json:"permissions,omitempty"` + Permissions []VectorSearchEndpointPermission `json:"permissions,omitempty"` } func (e *VectorSearchEndpoint) UnmarshalJSON(b []byte) error { diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 48ee6f0f96..ef30f3cd8a 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -3,7 +3,6 @@ package dresources import ( "context" "errors" - "fmt" "time" "github.com/databricks/cli/bundle/config/resources" @@ -24,7 +23,7 @@ const deleteIndexTimeout = 15 * time.Minute // createIndexTimeout caps the wait for an index to become ready after creation. // Delta sync indexes do an initial sync from the source table, which can stretch // out for large tables. Matches the terraform provider's defaultIndexProvisionTimeout. -// https://github.com/databricks/terraform-provider-databricks/blob/c61a32300445f84efb2bb6827dee35e6e523f4ff/vectorsearch/resource_vector_search_index.go#L19 +// https://github.com/databricks/terraform-provider-databricks/blob/c79d82d9582ab6670468bbff303199906d47905f/vectorsearch/resource_vector_search_index.go#L19 const createIndexTimeout = 75 * time.Minute // VectorSearchIndexState tracks the UUID of the endpoint the index is attached @@ -44,8 +43,9 @@ func (s VectorSearchIndexState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// VectorSearchIndexRemote is remote state. endpoint_uuid is looked up from the -// endpoint service since the index API itself doesn't return it. +// VectorSearchIndexRemote is remote state. endpoint_uuid mirrors the index +// response's endpoint_id so OverrideChangeDesc can compare it against the +// saved state without a second API call. type VectorSearchIndexRemote struct { vectorsearch.VectorIndex EndpointUuid string `json:"endpoint_uuid,omitempty"` @@ -111,13 +111,9 @@ func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*Vec if err != nil { return nil, err } - endpointUuid, err := r.lookupEndpointUuid(ctx, index.EndpointName) - if err != nil { - return nil, err - } return &VectorSearchIndexRemote{ VectorIndex: *index, - EndpointUuid: endpointUuid, + EndpointUuid: index.EndpointId, }, nil } @@ -126,16 +122,8 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector if err != nil { return "", nil, err } - // Second API call (also done in DoRead): the index API does not return the - // endpoint UUID, but we need to persist it in state so a future plan can - // detect that the endpoint was replaced out-of-band (same name, different - // UUID -> orphan). - endpointUuid, err := r.lookupEndpointUuid(ctx, config.EndpointName) - if err != nil { - return "", nil, err - } - config.EndpointUuid = endpointUuid - return config.Name, &VectorSearchIndexRemote{VectorIndex: *index, EndpointUuid: endpointUuid}, nil + config.EndpointUuid = index.EndpointId + return config.Name, &VectorSearchIndexRemote{VectorIndex: *index, EndpointUuid: index.EndpointId}, nil } // No DoUpdate: vector search indexes have no update API. All SDK fields are @@ -193,17 +181,13 @@ func (r *ResourceVectorSearchIndex) WaitAfterDelete(ctx context.Context, id stri } // OverrideChangeDesc classifies endpoint_uuid drift: Recreate when the saved -// UUID differs from what's currently attached to the endpoint name, Skip -// otherwise. endpoint_uuid is never present in config, so without Skip a -// synthetic diff between empty newState and populated saved state would -// otherwise leak into the plan. +// UUID differs from the endpoint_id the index API now reports, Skip otherwise. +// endpoint_uuid is never present in config, so without Skip a synthetic diff +// between empty newState and populated saved state would leak into the plan. // -// Unlike vector_search_endpoint, this intentionally does NOT require -// remoteUuid != "". An empty remoteUuid here is the orphan signal: the index -// still exists by name but its backing endpoint has been deleted out-of-band. -// lookupEndpointUuid distinguishes this (404 -> "") from transient errors -// (propagated through DoRead/DoCreate), so reaching this branch with empty -// remoteUuid unambiguously means the endpoint is gone. +// An empty remoteUuid is treated as drift (rather than ignored, as the endpoint +// resource does for its own UUID). If the backend ever clears endpoint_id when +// the endpoint is deleted out-of-band, this surfaces the orphan as a recreate. func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchIndexRemote) error { if path.String() != "endpoint_uuid" { return nil @@ -222,21 +206,3 @@ func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *st } return nil } - -// lookupEndpointUuid returns the current UUID of the endpoint with the given -// name. A 404 is converted to ("", nil) so the caller can distinguish a -// genuinely missing endpoint (the orphan signal) from a transient or -// permission error, which is propagated. -func (r *ResourceVectorSearchIndex) lookupEndpointUuid(ctx context.Context, endpointName string) (string, error) { - if endpointName == "" { - return "", nil - } - info, err := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, endpointName) - if err != nil { - if apierr.IsMissing(err) { - return "", nil - } - return "", fmt.Errorf("looking up vector search endpoint %q: %w", endpointName, err) - } - return info.Id, nil -} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 6a1070ddcd..d18e607731 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1051,6 +1051,19 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: "usage_policy_id": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission: + "group_name": + "description": |- + PLACEHOLDER + "level": + "description": |- + PLACEHOLDER + "service_principal_name": + "description": |- + PLACEHOLDER + "user_name": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: "delta_sync_index_spec": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c2026fc43b..225b7bcb93 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2041,7 +2041,7 @@ "$ref": "#/$defs/string" }, "permissions": { - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission" }, "target_qps": { "$ref": "#/$defs/int64" @@ -2064,6 +2064,35 @@ } ] }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorSearchEndpointPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.VectorSearchIndex": { "oneOf": [ { @@ -11955,6 +11984,9 @@ "vectorsearch.VectorIndexType": { "type": "string" }, + "vectorsearch.VectorSearchEndpointPermissionLevel": { + "type": "string" + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "oneOf": [ { @@ -12704,6 +12736,20 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "config.ArtifactFile": { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ff70f6b050..217f4d9506 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -152,7 +152,7 @@ type FakeWorkspace struct { RegisteredModels map[string]catalog.RegisteredModelInfo ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo - VectorSearchIndexes map[string]fakeVectorSearchIndex + VectorSearchIndexes map[string]vectorsearch.VectorIndex SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value @@ -297,7 +297,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - VectorSearchIndexes: map[string]fakeVectorSearchIndex{}, + VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, Repos: map[string]workspace.RepoInfo{}, SecretScopes: map[string]workspace.SecretScope{}, Secrets: map[string]map[string]string{}, diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index c739dbfdeb..3500b2fcc7 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -8,18 +8,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) -// fakeVectorSearchIndex captures the endpoint's UUID at index creation time. -// On the real backend an index is bound to a specific endpoint instance, not -// just the name: deleting and recreating an endpoint with the same name yields -// a different UUID, and the existing index keeps pointing at the OLD UUID -// (i.e. is orphaned). Tracking this here lets tests reason about that drift. -// The field is omitted from JSON responses since the real API doesn't return -// it on the index path; the CLI looks it up via GetEndpointByEndpointName. -type fakeVectorSearchIndex struct { - vectorsearch.VectorIndex - EndpointUuid string `json:"-"` -} - func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { defer s.LockUnlock()() @@ -48,20 +36,22 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { } } - index := fakeVectorSearchIndex{ - VectorIndex: vectorsearch.VectorIndex{ - Creator: s.CurrentUser().UserName, - EndpointName: createReq.EndpointName, - IndexType: createReq.IndexType, - Name: createReq.Name, - PrimaryKey: createReq.PrimaryKey, - DeltaSyncIndexSpec: remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec), - DirectAccessIndexSpec: createReq.DirectAccessIndexSpec, - Status: &vectorsearch.VectorIndexStatus{ - Ready: true, - }, + // EndpointId is captured at index creation time. On the real backend an + // index is bound to a specific endpoint instance, not just the name: + // deleting and recreating an endpoint with the same name yields a + // different UUID. Storing it lets tests reason about that drift. + index := vectorsearch.VectorIndex{ + Creator: s.CurrentUser().UserName, + EndpointId: endpoint.Id, + EndpointName: createReq.EndpointName, + IndexType: createReq.IndexType, + Name: createReq.Name, + PrimaryKey: createReq.PrimaryKey, + DeltaSyncIndexSpec: remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec), + DirectAccessIndexSpec: createReq.DirectAccessIndexSpec, + Status: &vectorsearch.VectorIndexStatus{ + Ready: true, }, - EndpointUuid: endpoint.Id, } s.VectorSearchIndexes[createReq.Name] = index @@ -77,6 +67,8 @@ func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vect return nil } return &vectorsearch.DeltaSyncVectorIndexSpecResponse{ + ColumnsToIndex: req.ColumnsToIndex, + ColumnsToSync: req.ColumnsToSync, EmbeddingSourceColumns: req.EmbeddingSourceColumns, EmbeddingVectorColumns: req.EmbeddingVectorColumns, EmbeddingWritebackTable: req.EmbeddingWritebackTable,