diff --git a/.agent/rules/testing.md b/.agent/rules/testing.md index f0bdff1366d..62b5d6ef126 100644 --- a/.agent/rules/testing.md +++ b/.agent/rules/testing.md @@ -97,8 +97,6 @@ If the only reason for divergence is a server-side default that one engine sets **RULE: If a test's `out.test.toml` is still in the older `[EnvMatrix]` block format, a regen rewrites it to the inline form and the post-test `git diff --exit-code` check fails** ("out.test.toml files that are out of date"). Regenerate just those files with `go test ./acceptance -run "^TestAccept$" -only-out-test-toml`, then commit. -**RULE: Be aware of `GOOSOnPR` when adding or editing an acceptance test — an inherited opt-out may be skipping it on windows/macOS PR runs.** By default every test runs everywhere. For speed, some directories set `GOOSOnPR.windows = false` / `GOOSOnPR.darwin = false` in `test.toml`, which recursively opts their tests out of windows/macOS PR runs (they still run on Linux for PRs and on every OS on push to main). Check the resolved value in `out.test.toml`. This is right for the platform-independent majority, but if your test genuinely depends on the OS — path separators, `.exe`, CRLF, symlinks, file locking, process/signal semantics, venv layout (`Scripts` vs `bin`), wheel build — and it sits under such a directory, override the inherited skip with `GOOSOnPR.windows = true` / `GOOSOnPR.darwin = true`, or a Windows/macOS-only regression can merge unseen. `MSYS_NO_PATHCONV`, a leading `//`, or a CRLF-normalizing `sed` are portability scaffolding (they make output OS-identical), not evidence of OS-specific behavior. See `GOOSOnPR` in `acceptance/internal/config.go`. - ### Reference - Tests live in `acceptance/` with a nested directory structure. diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index ec476ab82fb..fd903afa5e0 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -2a91f7354f73e4b52d212181bfc7591f563ccc50 \ No newline at end of file +007006859ac9df636ebfa08d5d25a1d5b59f15e6 \ No newline at end of file diff --git a/.codegen/cli.json b/.codegen/cli.json index 5454698e1f9..d9612c29be9 100644 --- a/.codegen/cli.json +++ b/.codegen/cli.json @@ -144,14 +144,14 @@ "fields": { "name": { "description": "Name of the column.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "type_text": { "description": "Data type of the column (e.g., \"string\", \"int\", \"array\u003cfloat\u003e\").", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -163,11 +163,11 @@ "endpoint": { "description": "The Endpoint resource to create. Fields other than `endpoint.name` carry the desired\nconfiguration; `endpoint.name` is server-assigned from `parent` and `endpoint_id`.", "ref": "aisearch.Endpoint", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "endpoint_id": { "description": "The user-supplied short name for the Endpoint, per AIP-133. The server composes the\nfull `Endpoint.name` as `{parent}/endpoints/{endpoint_id}`. AIP-133 does not list\n`endpoint_id` as a fields-may-be-required entry, so we annotate it OPTIONAL on the\nwire; the server still rejects empty values with INVALID_PARAMETER_VALUE.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -179,11 +179,11 @@ "index": { "description": "The Index resource to create. Fields other than `index.name` carry the desired\nconfiguration; `index.name` is server-assigned from `parent` and `index_id`.", "ref": "aisearch.Index", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "index_id": { "description": "The user-supplied Unity Catalog table name for the Index, per AIP-133. The server\ncomposes the full `Index.name` as `{parent}/indexes/{index_id}`. AIP-133 does not\nlist `index_id` as a fields-may-be-required entry, so we annotate it OPTIONAL on the\nwire; the server still rejects empty values with INVALID_PARAMETER_VALUE.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -195,11 +195,11 @@ "fields": { "key": { "description": "Key field for an AI Search endpoint tag.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "value": { "description": "[Optional] Value field for an AI Search endpoint tag.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -211,14 +211,14 @@ "fields": { "failed_primary_keys": { "description": "Primary keys of rows that failed to process.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "success_row_count": { "description": "Count of rows processed successfully.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -233,9 +233,9 @@ "FAILURE" ], "enum_launch_stages": { - "FAILURE": "PUBLIC_BETA", - "PARTIAL_SUCCESS": "PUBLIC_BETA", - "SUCCESS": "PUBLIC_BETA" + "FAILURE": "PUBLIC_PREVIEW", + "PARTIAL_SUCCESS": "PUBLIC_PREVIEW", + "SUCCESS": "PUBLIC_PREVIEW" } }, "aisearch.DeleteEndpointRequest": {}, @@ -245,35 +245,35 @@ "fields": { "columns_to_sync": { "description": "[Optional] Select the columns to sync with the index. If left blank, all columns\nfrom the source table are synced. The primary key column and embedding source or\nvector column are always synced.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "embedding_source_columns": { "description": "The columns that contain the embedding source.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "embedding_vector_columns": { "description": "The columns that contain the embedding vectors.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "embedding_writeback_table": { "description": "[Optional] Name of the Delta table to sync the index contents and computed embeddings to.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "pipeline_id": { "description": "The ID of the pipeline that is used to sync the index.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -281,11 +281,11 @@ "pipeline_type": { "description": "Pipeline execution mode. Required on create — the backend rejects an unset value.\nStorage Optimized endpoints accept only `TRIGGERED`; Standard endpoints accept both.\nNo explicit `stage` — a REQUIRED field staged below its service would be dropped from\ncombined specs while remaining in `required`, tripping the OpenAPI required-vs-properties\nconsistency check. The field inherits the service's launch stage.", "ref": "aisearch.PipelineType", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "source_table": { "description": "The full name of the source Delta table.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -297,21 +297,21 @@ "fields": { "embedding_source_columns": { "description": "The columns that contain the embedding source.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "embedding_vector_columns": { "description": "The columns that contain the embedding vectors.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "schema_json": { "description": "The schema of the index in JSON format. Supported types are `integer`, `long`,\n`float`, `double`, `boolean`, `string`, `date`, `timestamp`. Supported types for\nvector columns: `array\u003cfloat\u003e`, `array\u003cdouble\u003e`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -323,15 +323,15 @@ "fields": { "embedding_model_endpoint": { "description": "Name of the embedding model endpoint, used by default for both ingestion and querying.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "model_endpoint_name_for_query": { "description": "Name of the embedding model endpoint which, if specified, is used for querying (not ingestion).", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "name": { "description": "Name of the source column.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -340,11 +340,11 @@ "fields": { "embedding_dimension": { "description": "Dimension of the embedding vector.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "name": { "description": "Name of the column.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -353,35 +353,35 @@ "fields": { "budget_policy_id": { "description": "The user-selected budget policy id for the endpoint.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "create_time": { "description": "Time the endpoint was created.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "creator": { "description": "Creator of the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "custom_tags": { "description": "The custom tags assigned to the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "effective_budget_policy_id": { "description": "The budget policy id applied to the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -389,7 +389,7 @@ "endpoint_status": { "description": "Current status of the endpoint", "ref": "aisearch.EndpointStatus", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -397,39 +397,39 @@ "endpoint_type": { "description": "Type of endpoint. Required on create and immutable thereafter.", "ref": "aisearch.EndpointType", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE" ] }, "id": { "description": "Unique identifier of the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "index_count": { "description": "Number of indexes on the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "last_updated_user": { "description": "User who last updated the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "name": { "description": "Name of the AI Search endpoint. Server-assigned full resource path\n(`workspaces/{workspace}/endpoints/{endpoint}`) on output. On create, the\nuser-supplied short name is conveyed via `CreateEndpointRequest.endpoint_id`;\nthe server composes the full `name` and returns it on the response.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "replica_count": { "description": "The client-supplied desired number of replicas for the endpoint, applied at\ncreate/update time. Mutually exclusive with `target_qps`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -437,14 +437,14 @@ "scaling_info": { "description": "Scaling information for the endpoint", "ref": "aisearch.EndpointScalingInfo", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "target_qps": { "description": "Target QPS for the endpoint. Mutually exclusive with `replica_count`. Best-effort;\nthe system does not guarantee this QPS will be achieved.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -452,21 +452,21 @@ "throughput_info": { "description": "Throughput information for the endpoint", "ref": "aisearch.EndpointThroughputInfo", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "update_time": { "description": "Time the endpoint was last updated.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "usage_policy_id": { "description": "The usage policy id applied to the endpoint.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -478,7 +478,7 @@ "fields": { "requested_target_qps": { "description": "The requested QPS target for the endpoint. Best-effort; the system does not\nguarantee this QPS will be achieved.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -486,7 +486,7 @@ "state": { "description": "The current state of the scaling change request.", "ref": "aisearch.ScalingChangeState", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -498,7 +498,7 @@ "fields": { "message": { "description": "Human-readable detail about the endpoint's current state or the reason for a state transition.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -506,7 +506,7 @@ "state": { "description": "Current lifecycle state of the endpoint. See `State` for the meaning of each value.", "ref": "aisearch.EndpointStatusState", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -524,12 +524,12 @@ "DELETED" ], "enum_launch_stages": { - "DELETED": "PUBLIC_BETA", - "OFFLINE": "PUBLIC_BETA", - "ONLINE": "PUBLIC_BETA", - "PROVISIONING": "PUBLIC_BETA", - "RED_STATE": "PUBLIC_BETA", - "YELLOW_STATE": "PUBLIC_BETA" + "DELETED": "PUBLIC_PREVIEW", + "OFFLINE": "PUBLIC_PREVIEW", + "ONLINE": "PUBLIC_PREVIEW", + "PROVISIONING": "PUBLIC_PREVIEW", + "RED_STATE": "PUBLIC_PREVIEW", + "YELLOW_STATE": "PUBLIC_PREVIEW" } }, "aisearch.EndpointThroughputInfo": { @@ -537,7 +537,7 @@ "fields": { "change_request_message": { "description": "Additional information about the throughput change request", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -545,56 +545,56 @@ "change_request_state": { "description": "The state of the most recent throughput change request", "ref": "aisearch.ThroughputChangeRequestState", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "current_concurrency": { "description": "The current concurrency (total CPU) allocated to the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "current_concurrency_utilization_percentage": { "description": "The current utilization of concurrency as a percentage (0-100)", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "current_num_replicas": { "description": "The current number of replicas allocated to the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "maximum_concurrency_allowed": { "description": "The maximum concurrency allowed for this endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "minimal_concurrency_allowed": { "description": "The minimum concurrency allowed for this endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "requested_concurrency": { "description": "The requested concurrency (total CPU) for the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "requested_num_replicas": { "description": "The requested number of replicas for the endpoint", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -608,8 +608,8 @@ "STANDARD" ], "enum_launch_stages": { - "STANDARD": "PUBLIC_BETA", - "STORAGE_OPTIMIZED": "PUBLIC_BETA" + "STANDARD": "PUBLIC_PREVIEW", + "STORAGE_OPTIMIZED": "PUBLIC_PREVIEW" } }, "aisearch.FacetResultData": { @@ -617,14 +617,14 @@ "fields": { "facet_array": { "description": "Facet rows; each row is `[facet_column_name, value_or_range, count]`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "facet_row_count": { "description": "Number of facet rows returned.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -638,7 +638,7 @@ "fields": { "creator": { "description": "Creator of the index.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -646,7 +646,7 @@ "delta_sync_index_spec": { "description": "Specification for a Delta Sync index. Set when `index_type` is `DELTA_SYNC`.", "ref": "aisearch.DeltaSyncIndexSpec", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE", "OPTIONAL" @@ -655,7 +655,7 @@ "direct_access_index_spec": { "description": "Specification for a Direct Access index. Set when `index_type` is `DIRECT_ACCESS`.", "ref": "aisearch.DirectAccessIndexSpec", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE", "OPTIONAL" @@ -663,7 +663,7 @@ }, "endpoint": { "description": "Name of the endpoint associated with the index. Ignored on create — the endpoint is\ntaken from `CreateIndexRequest.parent`; populated only on output.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -671,7 +671,7 @@ "index_subtype": { "description": "The subtype of the index. Set on create and immutable thereafter.", "ref": "aisearch.IndexSubtype", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE", "OPTIONAL" @@ -680,18 +680,18 @@ "index_type": { "description": "Type of index. Required on create and immutable thereafter.", "ref": "aisearch.IndexType", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE" ] }, "name": { "description": "Name of the AI Search index. Server-assigned full resource path\n(`workspaces/{workspace}/endpoints/{endpoint}/indexes/{index}`) on output, where\n`{index}` is the index's Unity Catalog table name. On create, the user-supplied UC\ntable name is conveyed via `CreateIndexRequest.index_id`; the server composes the\nfull `name` and returns it on the response.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "primary_key": { "description": "Primary key of the index. Set on create and immutable thereafter.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "IMMUTABLE" ] @@ -699,7 +699,7 @@ "status": { "description": "Current status of the index.", "ref": "aisearch.IndexStatus", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -711,28 +711,28 @@ "fields": { "index_url": { "description": "Index API URL used to perform operations on the index.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "indexed_row_count": { "description": "Number of rows indexed.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "message": { "description": "Human-readable detail about the index's current state.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "ready": { "description": "Whether the index is ready for search.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -747,9 +747,9 @@ "HYBRID" ], "enum_launch_stages": { - "FULL_TEXT": "PUBLIC_BETA", - "HYBRID": "PUBLIC_BETA", - "VECTOR": "PUBLIC_BETA" + "FULL_TEXT": "PUBLIC_PREVIEW", + "HYBRID": "PUBLIC_PREVIEW", + "VECTOR": "PUBLIC_PREVIEW" }, "enum_descriptions": { "FULL_TEXT": "An index that uses full-text search without vector embeddings.", @@ -764,8 +764,8 @@ "DIRECT_ACCESS" ], "enum_launch_stages": { - "DELTA_SYNC": "PUBLIC_BETA", - "DIRECT_ACCESS": "PUBLIC_BETA" + "DELTA_SYNC": "PUBLIC_PREVIEW", + "DIRECT_ACCESS": "PUBLIC_PREVIEW" }, "enum_descriptions": { "DELTA_SYNC": "An index that automatically syncs with a source Delta Table,", @@ -776,14 +776,14 @@ "fields": { "page_size": { "description": "Best-effort upper bound on the number of results to return. Honored as an upper\nbound by the shim: `page_size` only narrows the legacy backend's response, never\nwidens it, so the practical cap is `min(page_size, legacy_fixed_page_size)`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "page_token": { "description": "Page token from a previous response. If not provided, returns the first page.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -795,14 +795,14 @@ "fields": { "endpoints": { "description": "The endpoints in the workspace.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "next_page_token": { "description": "A token that can be used to get the next page of results. Empty when there are no more results.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -813,14 +813,14 @@ "fields": { "page_size": { "description": "Best-effort upper bound on the number of results to return. Honored as an upper\nbound by the shim: `page_size` only narrows the legacy backend's response, never\nwidens it, so the practical cap is `min(page_size, legacy_fixed_page_size)`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "page_token": { "description": "Page token from a previous response. If not provided, returns the first page.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -832,14 +832,14 @@ "fields": { "indexes": { "description": "The indexes on the endpoint. The field is named `indexes` (not the irregular plural\n`indices`) to satisfy core::0132, which derives the response field name from the\nListIndexes method. core::0158::response-plural-first-field independently computes the\nresource plural as `indices` and is satisfied via a scoped field exception below.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "next_page_token": { "description": "A token that can be used to get the next page of results. Empty when there are no more results.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -853,8 +853,8 @@ "CONTINUOUS" ], "enum_launch_stages": { - "CONTINUOUS": "PUBLIC_BETA", - "TRIGGERED": "PUBLIC_BETA" + "CONTINUOUS": "PUBLIC_PREVIEW", + "TRIGGERED": "PUBLIC_PREVIEW" }, "enum_descriptions": { "CONTINUOUS": "the pipeline processes new data as it arrives in the source table to", @@ -866,60 +866,60 @@ "fields": { "columns": { "description": "Column names to include in each result row.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "columns_to_rerank": { "description": "Columns whose values are sent to the reranker.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "facets": { "description": "Facets to compute over the matched results (e.g. `\"category TOP 5\"`).", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "filters_json": { "description": "JSON string describing query filters (e.g. `{\"id \u003e\": 5}`).", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "max_results": { "description": "Maximum number of results to return (the legacy `num_results`). Defaults to 10.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "query_columns": { "description": "Text columns to search for `query_text`. When empty, all text columns are searched.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "query_text": { "description": "Query text. Required for Delta Sync indexes that compute embeddings from a model endpoint.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "query_type": { "description": "Query type: `ANN`, `HYBRID`, or `FULL_TEXT`. Defaults to `ANN`.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "query_vector": { "description": "Query vector. Required for Direct Access indexes and Delta Sync indexes with self-managed\nvectors.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -927,21 +927,21 @@ "reranker": { "description": "If set, results are reranked before being returned.", "ref": "aisearch.RerankerConfig", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "score_threshold": { "description": "Score threshold for the approximate nearest-neighbor search. Defaults to 0.0.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "sort_columns": { "description": "Sort clauses, e.g. `[\"rating DESC\", \"price ASC\"]`. Overrides relevance ordering.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -954,7 +954,7 @@ "facet_result": { "description": "Facet aggregation rows, when facets were requested.", "ref": "aisearch.FacetResultData", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -962,7 +962,7 @@ "manifest": { "description": "Metadata describing the result columns.", "ref": "aisearch.ResultManifest", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -970,7 +970,7 @@ "result": { "description": "The matched result rows.", "ref": "aisearch.ResultData", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -982,7 +982,7 @@ "fields": { "primary_keys": { "description": "Primary keys of the rows to remove.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -992,7 +992,7 @@ "result": { "description": "Per-row outcome of the delete.", "ref": "aisearch.DataModificationResult", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1000,7 +1000,7 @@ "status": { "description": "Overall status of the delete.", "ref": "aisearch.DataModificationStatus", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1012,7 +1012,7 @@ "fields": { "model": { "description": "Reranker identifier: \"databricks_reranker\" for the base model, or a Model Serving\nendpoint name when `model_type` is MODEL_TYPE_FINETUNED.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -1020,7 +1020,7 @@ "model_type": { "description": "Discriminator for how `model` is interpreted.", "ref": "aisearch.RerankerConfigModelType", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -1028,7 +1028,7 @@ "parameters": { "description": "Parameters controlling reranking.", "ref": "aisearch.RerankerConfigRerankerParameters", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -1042,8 +1042,8 @@ "MODEL_TYPE_FINETUNED" ], "enum_launch_stages": { - "MODEL_TYPE_BASE": "PUBLIC_BETA", - "MODEL_TYPE_FINETUNED": "PUBLIC_BETA" + "MODEL_TYPE_BASE": "PUBLIC_PREVIEW", + "MODEL_TYPE_FINETUNED": "PUBLIC_PREVIEW" } }, "aisearch.RerankerConfigRerankerParameters": { @@ -1051,7 +1051,7 @@ "fields": { "columns_to_rerank": { "description": "Columns whose values are concatenated and sent to the reranker.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -1063,14 +1063,14 @@ "fields": { "data_array": { "description": "Result rows; each row is a list of column values aligned with the manifest columns.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "row_count": { "description": "Number of rows in the result set.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1082,28 +1082,28 @@ "fields": { "column_count": { "description": "Number of columns in the result set.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "columns": { "description": "Information about each column in the result set.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "facet_column_count": { "description": "Number of columns in the facet result.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "facet_columns": { "description": "Information about each facet column.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1118,9 +1118,9 @@ "SCALING_CHANGE_IN_PROGRESS" ], "enum_launch_stages": { - "SCALING_CHANGE_APPLIED": "PUBLIC_BETA", - "SCALING_CHANGE_IN_PROGRESS": "PUBLIC_BETA", - "SCALING_CHANGE_UNSPECIFIED": "PUBLIC_BETA" + "SCALING_CHANGE_APPLIED": "PUBLIC_PREVIEW", + "SCALING_CHANGE_IN_PROGRESS": "PUBLIC_PREVIEW", + "SCALING_CHANGE_UNSPECIFIED": "PUBLIC_PREVIEW" } }, "aisearch.ScanIndexRequest": { @@ -1128,14 +1128,14 @@ "fields": { "page_size": { "description": "Maximum number of rows to return in this page.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] }, "page_token": { "description": "Page token from a previous response; if unset, scanning starts from the beginning.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -1147,14 +1147,14 @@ "fields": { "data": { "description": "The rows in this page, each a struct of column name to value.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "next_page_token": { "description": "Token for the next page; empty when the scan is exhausted.", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1178,12 +1178,12 @@ "CHANGE_ADJUSTED" ], "enum_launch_stages": { - "CHANGE_ADJUSTED": "PUBLIC_BETA", - "CHANGE_FAILED": "PUBLIC_BETA", - "CHANGE_IN_PROGRESS": "PUBLIC_BETA", - "CHANGE_REACHED_MAXIMUM": "PUBLIC_BETA", - "CHANGE_REACHED_MINIMUM": "PUBLIC_BETA", - "CHANGE_SUCCESS": "PUBLIC_BETA" + "CHANGE_ADJUSTED": "PUBLIC_PREVIEW", + "CHANGE_FAILED": "PUBLIC_PREVIEW", + "CHANGE_IN_PROGRESS": "PUBLIC_PREVIEW", + "CHANGE_REACHED_MAXIMUM": "PUBLIC_PREVIEW", + "CHANGE_REACHED_MINIMUM": "PUBLIC_PREVIEW", + "CHANGE_SUCCESS": "PUBLIC_PREVIEW" } }, "aisearch.UpdateEndpointRequest": { @@ -1191,11 +1191,11 @@ "endpoint": { "description": "The Endpoint resource to update. `endpoint.name` carries the full resource path.", "ref": "aisearch.Endpoint", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" }, "update_mask": { "description": "The list of fields to update.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -1204,7 +1204,7 @@ "fields": { "inputs_json": { "description": "JSON document describing the rows to upsert.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -1214,7 +1214,7 @@ "result": { "description": "Per-row outcome of the upsert.", "ref": "aisearch.DataModificationResult", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -1222,7 +1222,7 @@ "status": { "description": "Overall status of the upsert.", "ref": "aisearch.DataModificationStatus", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -2311,7 +2311,7 @@ "enum_launch_stages": { "LARGE": "GA", "MEDIUM": "GA", - "XLARGE": "PRIVATE_PREVIEW" + "XLARGE": "GA" } }, "apps.ComputeState": { @@ -3539,7 +3539,8 @@ ] }, "creator_user_name": { - "description": "The policy creator user name to be filtered on.\nIf unspecified, all policies will be returned.", + "description": "Deprecated: Do not use this field in new integrations. Creator filtering will be removed in a\nfuture version.\nThe policy creator user name to be filtered on.\nIf unspecified, all policies will be returned.", + "deprecated": true, "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" @@ -3980,13 +3981,9 @@ "bundledeployments.CreateDeploymentRequest": { "fields": { "deployment": { - "description": "The deployment to create.", + "description": "The deployment to create. The caller must set `initial_parent_path`. Other fields are ignored\non input and populated by the service.", "ref": "bundledeployments.Deployment", "launch_stage": "PRIVATE_PREVIEW" - }, - "deployment_id": { - "description": "The ID to use for the deployment, which will become the final\ncomponent of the deployment's resource name\n(i.e. `deployments/{deployment_id}`).", - "launch_stage": "PRIVATE_PREVIEW" } } }, @@ -4043,7 +4040,7 @@ ] }, "destroy_time": { - "description": "When the deployment was destroyed (i.e. `bundle destroy` completed).\nUnset if the deployment has not been destroyed.\nNamed destroy_time (not delete_time) because this tracks the\n`databricks bundle destroy` command, not the API-level deletion.", + "description": "When deletion was recorded. Unset if deletion has not been recorded.\nThis response metadata does not determine the deployment's lifecycle status.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OUTPUT_ONLY" @@ -4057,7 +4054,7 @@ ] }, "display_name": { - "description": "Human-readable name for the deployment. Output only: it is denormalized from\nthe latest version, not set directly on the deployment.", + "description": "Human-readable name for the deployment, up to 256 characters. Output only: clients update it\nby setting `display_name` when creating a version.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OUTPUT_ONLY" @@ -4071,6 +4068,13 @@ "OUTPUT_ONLY" ] }, + "initial_parent_path": { + "description": "The workspace path of the existing folder where the deployment is initially created. Must be\nabsolute and canonical, with single separators, no `.` or `..` segments, and no trailing slash\nunless the path is `/`. It may contain at most 24 path segments, excluding an optional leading\n`/Workspace` segment. The complete path may contain up to 1,024 characters, and each segment\nmay contain up to 511 characters. This field is input only and is not returned in create, get,\nor list responses.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "INPUT_ONLY" + ] + }, "last_version_id": { "description": "The version_id of the most recent deployment version.", "launch_stage": "PRIVATE_PREVIEW", @@ -4235,7 +4239,7 @@ "bundledeployments.ListDeploymentsRequest": { "fields": { "page_size": { - "description": "The maximum number of deployments to return. The service may return\nfewer than this value.\nIf unspecified, at most 50 deployments will be returned.\nThe maximum value is 1000; values above 1000 will be coerced to 1000.", + "description": "The maximum number of deployments to return. The service may return\nfewer than this value.\nIf unspecified, at most 20 deployments will be returned.\nThe maximum value is 100; values above 100 will be coerced to 100.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" @@ -4337,7 +4341,7 @@ "bundledeployments.ListVersionsRequest": { "fields": { "page_size": { - "description": "The maximum number of versions to return. The service may return\nfewer than this value.\nIf unspecified, at most 50 versions will be returned.\nThe maximum value is 1000; values above 1000 will be coerced to 1000.", + "description": "The maximum number of versions to return. The service may return\nfewer than this value.\nIf unspecified, at most 20 versions will be returned.\nThe maximum value is 100; values above 100 will be coerced to 100.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" @@ -4525,6 +4529,13 @@ "behaviors": [ "OPTIONAL" ] + }, + "update_time": { + "description": "When the last operation that updated this resource's recorded state was applied. Pairs with\nlast_action_type and last_version_id (all three advance together on that write).", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OUTPUT_ONLY" + ] } } }, @@ -4583,7 +4594,7 @@ ] }, "display_name": { - "description": "Display name for the deployment, captured at the time of this version.", + "description": "Display name for the deployment, captured at the time of this version. Up to 256 characters.\nWhen present, creating the version updates the deployment display name. An empty value clears\nit; an absent value leaves the current deployment display name unchanged.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "IMMUTABLE" @@ -5615,6 +5626,7 @@ "POWER_BI", "DYNAMICS365", "CONFLUENCE", + "JDBC", "META_MARKETING", "HUBSPOT", "ZENDESK", @@ -5633,6 +5645,7 @@ "HIVE_METASTORE": "GA", "HTTP": "GA", "HUBSPOT": "GA", + "JDBC": "PUBLIC_PREVIEW", "META_MARKETING": "PUBLIC_BETA", "MYSQL": "GA", "ORACLE": "GA", @@ -5837,6 +5850,13 @@ "description": "A map of key-value properties attached to the securable.", "launch_stage": "GA" }, + "parent": { + "description": "Parent schema for schema-level connections, in format \"schemas/{catalog}.{schema}\".\nAbsent for metastore-level (L1) connections.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + }, "properties": { "description": "A map of key-value properties attached to the securable.", "launch_stage": "GA" @@ -6243,7 +6263,10 @@ }, "browse_only": { "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "OUTPUT_ONLY" + ] }, "catalog_name": { "description": "The name of the catalog where the schema and the registered model reside", @@ -8444,6 +8467,13 @@ "page_token": { "description": "Opaque pagination token to go to next page based on previous query.", "launch_stage": "GA" + }, + "parent": { + "description": "Optional. Parent schema filter for listing schema-level connections, in format \"schemas/{catalog}.{schema}\".", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] } } }, @@ -8495,15 +8525,15 @@ "fields": { "page_size": { "description": "Specifies the maximum number of privilege assignments to return (page length).\nEvery EffectivePrivilegeAssignment present in a single page response is guaranteed to contain all the effective\nprivileges granted on (or inherited by) the requested Securable for the respective principal.\n\nIf not set, a server-configured default is used.\nIf set to\n- lesser than 0: invalid parameter error\n- 0: page length is set to a server configured value\n- lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able to return at\nleast one complete EffectivePrivilegeAssignment in a single page response)\n- greater than (or equal to) 150: page length is the minimum of this value and a server configured value", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW" }, "page_token": { "description": "Opaque pagination token to go to next page based on previous query.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW" }, "principal": { "description": "If provided, only the effective permissions for the specified principal (user or group) are returned.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -8514,14 +8544,14 @@ "fields": { "effective_privilege_assignments": { "description": "The effective privilege assignments for the securable (and optional principal).", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "next_page_token": { "description": "Opaque token to retrieve the next page of results. Absent if there are no more pages.\n__page_token__ should be set to this value for the next request (for the next page of results).", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -8784,15 +8814,15 @@ "fields": { "page_size": { "description": "Specifies the maximum number of privilege assignments to return (page length).\nEvery PrivilegeAssignment present in a single page response is guaranteed to contain all the privileges granted on\nthe requested Securable for the respective principal.\n\nIf not set, page length is the server configured value.\nIf set to\n- lesser than 0: invalid parameter error\n- 0: page length is set to a server configured value\n- lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able to return at\nleast one complete PrivilegeAssignment in a single page response)\n- greater than (or equal to) 150: page length is the minimum of this value and a server configured value", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW" }, "page_token": { "description": "Opaque pagination token to go to next page based on previous query.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW" }, "principal": { "description": "If provided, only the permissions for the specified principal (user or group) are returned.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OPTIONAL" ] @@ -8803,13 +8833,13 @@ "fields": { "next_page_token": { "description": "Opaque token to retrieve the next page of results. Absent if there are no more pages.\n__page_token__ should be set to this value for the next request (for the next page of results).", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] }, "privilege_assignments": { - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_PREVIEW", "behaviors": [ "OUTPUT_ONLY" ] @@ -10255,7 +10285,7 @@ "MODIFY": "GA", "MODIFY_CLEAN_ROOM": "GA", "READ_FILES": "GA", - "READ_METADATA": "PUBLIC_PREVIEW", + "READ_METADATA": "GA", "READ_PRIVATE_FILES": "GA", "READ_VOLUME": "GA", "REFRESH": "GA", @@ -10433,7 +10463,10 @@ }, "browse_only": { "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "OUTPUT_ONLY" + ] }, "catalog_name": { "description": "The name of the catalog where the schema and the registered model reside", @@ -10982,14 +11015,24 @@ "SPECIAL_DESTINATION_EXTERNAL_LOCATION_OWNER", "SPECIAL_DESTINATION_CONNECTION_OWNER", "SPECIAL_DESTINATION_CREDENTIAL_OWNER", - "SPECIAL_DESTINATION_METASTORE_OWNER" + "SPECIAL_DESTINATION_METASTORE_OWNER", + "SPECIAL_DESTINATION_SCHEMA_OWNER", + "SPECIAL_DESTINATION_TABLE_OWNER", + "SPECIAL_DESTINATION_VOLUME_OWNER", + "SPECIAL_DESTINATION_FUNCTION_OWNER", + "SPECIAL_DESTINATION_REGISTERED_MODEL_OWNER" ], "enum_launch_stages": { "SPECIAL_DESTINATION_CATALOG_OWNER": "PUBLIC_PREVIEW", "SPECIAL_DESTINATION_CONNECTION_OWNER": "PUBLIC_PREVIEW", "SPECIAL_DESTINATION_CREDENTIAL_OWNER": "PUBLIC_PREVIEW", "SPECIAL_DESTINATION_EXTERNAL_LOCATION_OWNER": "PUBLIC_PREVIEW", - "SPECIAL_DESTINATION_METASTORE_OWNER": "PUBLIC_PREVIEW" + "SPECIAL_DESTINATION_FUNCTION_OWNER": "PUBLIC_BETA", + "SPECIAL_DESTINATION_METASTORE_OWNER": "PUBLIC_PREVIEW", + "SPECIAL_DESTINATION_REGISTERED_MODEL_OWNER": "PUBLIC_BETA", + "SPECIAL_DESTINATION_SCHEMA_OWNER": "PUBLIC_BETA", + "SPECIAL_DESTINATION_TABLE_OWNER": "PUBLIC_BETA", + "SPECIAL_DESTINATION_VOLUME_OWNER": "PUBLIC_BETA" } }, "catalog.SseEncryptionDetails": { @@ -12042,7 +12085,10 @@ }, "browse_only": { "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "OUTPUT_ONLY" + ] }, "catalog_name": { "description": "The name of the catalog where the schema and the registered model reside", @@ -12619,6 +12665,10 @@ "OUTPUT_ONLY" ] }, + "enable_shared_output": { + "description": "Whether allow task to write to shared output schema.\nWhen enabled, clean room task runs triggered by the current collaborator\ncan write to the run-scoped shared output schema which is accessible by all collaborators.", + "launch_stage": "PUBLIC_PREVIEW" + }, "local_collaborator_alias": { "description": "The alias of the collaborator tied to the local clean room.", "launch_stage": "GA", @@ -12715,6 +12765,14 @@ "OPTIONAL" ] }, + "jar_analysis": { + "description": "Jar analysis details available to all collaborators of the clean room.\nPresent if and only if **asset_type** is **JAR_ANALYSIS**", + "ref": "cleanrooms.CleanRoomAssetJarAnalysis", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, "name": { "description": "A fully qualified name that uniquely identifies the asset within the clean room.\nThis is also the name displayed in the clean room UI.\n\nFor UC securable assets (tables, volumes, etc.), the format is *shared_catalog*.*shared_schema*.*asset_name*\n\nFor notebooks, the name is the notebook file name.\nFor jar analyses, the name is the jar analysis name.", "launch_stage": "GA" @@ -12790,10 +12848,12 @@ "NOTEBOOK_FILE", "VOLUME", "VIEW", - "FOREIGN_TABLE" + "FOREIGN_TABLE", + "JAR_ANALYSIS" ], "enum_launch_stages": { "FOREIGN_TABLE": "GA", + "JAR_ANALYSIS": "PUBLIC_PREVIEW", "NOTEBOOK_FILE": "GA", "TABLE": "GA", "VIEW": "GA", @@ -12819,8 +12879,83 @@ } } }, + "cleanrooms.CleanRoomAssetJarAnalysis": { + "fields": { + "central_jar_file_paths": { + "description": "The full paths in central to the jar files that are added to the library during execution (e.g. /Volumes/creator/schema/volume/folder/my_jar_file.jar)\nOnly returned for the owner collaborator.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "description": { + "description": "Optional description of the jar analysis shown to all collaborators.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "environment_version": { + "description": "The serverless environment version used to execute the JAR analysis (e.g. \"4\").\nDefaults to \"4-scala-preview\" if not specified.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "etag": { + "description": "Server generated etag that represents the jar analysis version.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "main_class_name": { + "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "review_state": { + "description": "Top-level status derived from all reviews.", + "ref": "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewState", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "reviews": { + "description": "All existing approvals or rejections.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "runner_collaborator_aliases": { + "description": "Collaborators that can run the jar.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "cleanrooms.CleanRoomAssetNotebook": { "fields": { + "description": { + "description": "Optional description of the notebook shown to all collaborators.", + "launch_stage": "GA", + "behaviors": [ + "OPTIONAL" + ] + }, + "environment_version": { + "description": "The serverless environment version used to execute the notebook (e.g. \"4\").\nDefaults to \"2\" if not specified.", + "launch_stage": "GA", + "behaviors": [ + "OPTIONAL" + ] + }, "etag": { "description": "Server generated etag that represents the notebook version.", "launch_stage": "GA", @@ -13027,6 +13162,53 @@ } } }, + "cleanrooms.CleanRoomJarAnalysisReview": { + "description": "This only applies to a JAR Analysis as a first-class asset in the Clean Room, and not to Volumes", + "fields": { + "comment": { + "description": "review comment", + "launch_stage": "PUBLIC_PREVIEW" + }, + "created_at_millis": { + "description": "timestamp of when the review was submitted", + "launch_stage": "PUBLIC_PREVIEW" + }, + "review_state": { + "description": "review outcome", + "ref": "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewState", + "launch_stage": "PUBLIC_PREVIEW" + }, + "review_sub_reason": { + "description": "specified when the review was not explicitly made by a user", + "ref": "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewSubReason", + "launch_stage": "PUBLIC_PREVIEW" + }, + "reviewer_collaborator_alias": { + "description": "collaborator alias of the reviewer", + "launch_stage": "PUBLIC_PREVIEW" + } + } + }, + "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewState": { + "enum": [ + "APPROVED", + "REJECTED", + "PENDING" + ], + "enum_launch_stages": { + "APPROVED": "PUBLIC_PREVIEW", + "PENDING": "PUBLIC_PREVIEW", + "REJECTED": "PUBLIC_PREVIEW" + } + }, + "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewSubReason": { + "enum": [ + "AUTO_APPROVED" + ], + "enum_launch_stages": { + "AUTO_APPROVED": "PUBLIC_PREVIEW" + } + }, "cleanrooms.CleanRoomNotebookReview": { "fields": { "comment": { @@ -13112,6 +13294,14 @@ "description": "Duration of the task run, in milliseconds.", "launch_stage": "GA" }, + "shared_output_schema_expiration_time": { + "description": "Expiration time of the shared output schema of the task run (if any), in epoch milliseconds.", + "launch_stage": "PUBLIC_PREVIEW" + }, + "shared_output_schema_name": { + "description": "Name of the shared output schema associated with the clean rooms notebook task run.\nThis schema is accessible by all collaborators when enable_shared_output is true.", + "launch_stage": "PUBLIC_PREVIEW" + }, "start_time": { "description": "When the task run started, in epoch milliseconds.", "launch_stage": "GA" @@ -13192,6 +13382,20 @@ "IMMUTABLE" ] }, + "enable_shared_output": { + "description": "Whether to enable shared output for the central clean room.\nWhen enabled, clean room task runs can write to the run-scoped shared output schema\nwhich is accessible by all collaborators.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "package_provider_collaborator_alias": { + "description": "Alias of the provider collaborator. If set, packaged clean rooms mode is enabled.\nThe consumer's experience is restricted: they can view notebook names and READMEs,\nadd their own data assets, and trigger runs, but cannot view notebook code,\nprovider data assets, or notebook run output.", + "launch_stage": "GA", + "behaviors": [ + "IMMUTABLE" + ] + }, "region": { "description": "Region of the central clean room.", "launch_stage": "GA", @@ -13215,6 +13419,87 @@ "PROVISIONING": "GA" } }, + "cleanrooms.CleanRoomTaskRun": { + "description": "Stores information about a single task run.", + "fields": { + "analysis_details": { + "description": "Information about the analysis run (etag, updated at)", + "ref": "cleanrooms.CleanRoomTaskRunCleanRoomTaskAnalysisDetails", + "launch_stage": "GA" + }, + "collaborator_job_run_info": { + "description": "Job run info of the task in the runner's local workspace.\nThis field is only included in the LIST API if the task was run within the same workspace the API is being called.\nIf the task run was in a different workspace under the same metastore, only the workspace_id is included.", + "ref": "cleanrooms.CollaboratorJobRunInfo", + "launch_stage": "GA" + }, + "name": { + "description": "Name of the executable.", + "launch_stage": "GA" + }, + "output_info": { + "description": "Information about run output", + "ref": "cleanrooms.CleanRoomTaskRunOutputInfo", + "launch_stage": "GA" + }, + "run_duration": { + "description": "Duration of the task run, in milliseconds.", + "launch_stage": "GA" + }, + "shared_output_info": { + "description": "Information about shared output accessible by all collaborators.\nThis field is only populated when enable_shared_output is true.", + "ref": "cleanrooms.CleanRoomTaskRunOutputInfo", + "launch_stage": "PUBLIC_PREVIEW" + }, + "start_time": { + "description": "When the task run started, in epoch milliseconds.", + "launch_stage": "GA" + }, + "task_run_state": { + "description": "State of the task run.", + "ref": "jobs.CleanRoomTaskRunState", + "launch_stage": "GA" + }, + "task_type": { + "description": "The type of Clean Room task.", + "ref": "cleanrooms.CleanRoomTaskType", + "launch_stage": "GA" + } + } + }, + "cleanrooms.CleanRoomTaskRunCleanRoomTaskAnalysisDetails": { + "fields": { + "etag": { + "description": "Etag of the asset executed in this task run, used to identify the asset version.", + "launch_stage": "GA" + }, + "updated_at": { + "description": "The timestamp of when the asset was last updated.", + "launch_stage": "GA" + } + } + }, + "cleanrooms.CleanRoomTaskRunOutputInfo": { + "fields": { + "output_schema_expiration_time": { + "description": "Expiration time of the output schema of the task run (if any), in epoch milliseconds.", + "launch_stage": "GA" + }, + "output_schema_name": { + "description": "Name of the output schema associated with the clean room task run.", + "launch_stage": "GA" + } + } + }, + "cleanrooms.CleanRoomTaskType": { + "enum": [ + "NOTEBOOK", + "JAR" + ], + "enum_launch_stages": { + "JAR": "GA", + "NOTEBOOK": "GA" + } + }, "cleanrooms.CollaboratorJobRunInfo": { "fields": { "collaborator_alias": { @@ -13262,6 +13547,10 @@ }, "cleanrooms.CreateCleanRoomAssetReviewRequest": { "fields": { + "jar_analysis_review": { + "ref": "cleanrooms.JarAnalysisVersionReview", + "launch_stage": "PUBLIC_BETA" + }, "notebook_review": { "ref": "cleanrooms.NotebookVersionReview", "launch_stage": "PUBLIC_BETA" @@ -13270,6 +13559,15 @@ }, "cleanrooms.CreateCleanRoomAssetReviewResponse": { "fields": { + "jar_analysis_review_state": { + "description": "top-level status derived from all reviews", + "ref": "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewState", + "launch_stage": "PUBLIC_BETA" + }, + "jar_analysis_reviews": { + "description": "All existing jar analysis approvals or rejections", + "launch_stage": "PUBLIC_BETA" + }, "notebook_review_state": { "description": "Top-level status derived from all reviews", "ref": "cleanrooms.CleanRoomNotebookReviewNotebookReviewState", @@ -13323,6 +13621,23 @@ "cleanrooms.GetCleanRoomAssetRevisionRequest": {}, "cleanrooms.GetCleanRoomAutoApprovalRuleRequest": {}, "cleanrooms.GetCleanRoomRequest": {}, + "cleanrooms.JarAnalysisVersionReview": { + "fields": { + "comment": { + "description": "Review comment", + "launch_stage": "PUBLIC_BETA" + }, + "etag": { + "description": "Etag identifying the jar analysis version, with its value being a hash of an internally-generated UUID", + "launch_stage": "PUBLIC_BETA" + }, + "review_state": { + "description": "Review outcome", + "ref": "cleanrooms.CleanRoomJarAnalysisReviewJarAnalysisReviewState", + "launch_stage": "PUBLIC_BETA" + } + } + }, "cleanrooms.ListCleanRoomAssetRevisionsRequest": { "fields": { "page_size": { @@ -13431,6 +13746,39 @@ } } }, + "cleanrooms.ListCleanRoomTaskRunsRequest": { + "fields": { + "name": { + "description": "Executable name.", + "launch_stage": "GA" + }, + "page_size": { + "description": "The maximum number of task runs to return. Maximum value of 100.", + "launch_stage": "GA" + }, + "page_token": { + "description": "Opaque pagination token to go to next page based on previous query.", + "launch_stage": "GA" + }, + "task_type": { + "description": "Filter by the type of Clean Room task.", + "ref": "cleanrooms.CleanRoomTaskType", + "launch_stage": "GA" + } + } + }, + "cleanrooms.ListCleanRoomTaskRunsResponse": { + "fields": { + "next_page_token": { + "description": "Opaque token to retrieve the next page of results. Absent if there are no more pages.\npage_token should be set to this value for the next request (for the next page of results).", + "launch_stage": "GA" + }, + "runs": { + "description": "Task runs in the clean room.", + "launch_stage": "GA" + } + } + }, "cleanrooms.ListCleanRoomsRequest": { "fields": { "page_size": { @@ -13608,7 +13956,7 @@ }, "capacity_reservation_group": { "description": "The Azure capacity reservation group resource ID to use for launching VMs.\nWhen specified, VMs will be launched using the provided capacity reservation.\n\nCapacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not\nmanaged by databricks). Ensure the databricks-login-prod Enterprise Application is granted the following four permissions:\n1. Microsoft.Compute/capacityReservationGroups/read\n2. Microsoft.Compute/capacityReservationGroups/deploy/action\n3. Microsoft.Compute/capacityReservationGroups/capacityReservations/read\n4. Microsoft.Compute/capacityReservationGroups/capacityReservations/deploy/action\n\nFormat: `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/capacityReservationGroups/{capacityReservationGroupName}`", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "first_on_demand": { "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", @@ -13796,6 +14144,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -13989,6 +14342,11 @@ "description": "Tags that are added by Databricks regardless of any `custom_tags`, including:\n\n- Vendor: Databricks\n\n- Creator: \u003cusername_of_creator\u003e\n\n- ClusterName: \u003cname_of_cluster\u003e\n\n- ClusterId: \u003cid_of_cluster\u003e\n\n- Name: \u003cDatabricks internal use\u003e", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -14445,6 +14803,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -14717,6 +15080,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -15094,6 +15462,24 @@ }, "compute.DeletePolicyResponse": {}, "compute.DeleteResponse": {}, + "compute.DependencyMode": { + "description": "Controls dependency configuration for the cluster.\n\n* `DEPENDENCY_MODE_AUTO`: Databricks will choose the most appropriate dependency mode based on your compute configuration.\n* `DEPENDENCY_MODE_ENVIRONMENTS`: Enables a unified dependency management experience across classic and serverless, resulting in increased stability and performance. Supported only on DBR 19+ in Standard access mode.\n* `DEPENDENCY_MODE_CLUSTER_LIBRARIES`: Legacy mode: dependencies come from cluster libraries and init scripts.", + "enum": [ + "DEPENDENCY_MODE_ENVIRONMENTS", + "DEPENDENCY_MODE_CLUSTER_LIBRARIES", + "DEPENDENCY_MODE_AUTO" + ], + "enum_launch_stages": { + "DEPENDENCY_MODE_AUTO": "PUBLIC_BETA", + "DEPENDENCY_MODE_CLUSTER_LIBRARIES": "PUBLIC_BETA", + "DEPENDENCY_MODE_ENVIRONMENTS": "PUBLIC_BETA" + }, + "enum_descriptions": { + "DEPENDENCY_MODE_AUTO": "\u003cDatabricks\u003e will choose the most appropriate dependency mode based on your compute configuration.", + "DEPENDENCY_MODE_CLUSTER_LIBRARIES": "Legacy mode: dependencies come from cluster libraries and init scripts.", + "DEPENDENCY_MODE_ENVIRONMENTS": "Enables a unified dependency management experience across classic and serverless, resulting in increased stability and performance. Supported only on DBR 19+ in Standard access mode." + } + }, "compute.DestroyContext": { "fields": { "clusterId": { @@ -15246,6 +15632,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -15515,6 +15906,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -16645,6 +17041,10 @@ "ref": "compute.InstancePoolAzureAttributesAvailability", "launch_stage": "GA" }, + "capacity_reservation_group": { + "description": "The Azure capacity reservation group resource ID to use for launching VMs in this pool.\nWhen specified, VMs will be launched using the provided capacity reservation.\n\nNOTE: Omitting this field will clear any existing configured capacity reservation group on the pool.\n\nCapacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not\nmanaged by databricks). Ensure the databricks-login-prod Enterprise Application is granted the following four permissions:\n1. Microsoft.Compute/capacityReservationGroups/read\n2. Microsoft.Compute/capacityReservationGroups/deploy/action\n3. Microsoft.Compute/capacityReservationGroups/capacityReservations/read\n4. Microsoft.Compute/capacityReservationGroups/capacityReservations/deploy/action\n\nFormat: `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/capacityReservationGroups/{capacityReservationGroupName}`", + "launch_stage": "GA" + }, "spot_bid_max_price": { "description": "With variable pricing, you have option to set a max price, in US dollars (USD)\nFor example, the value 2 would be a max price of $2.00 USD per hour.\nIf you set the max price to be -1, the VM won't be evicted based on price.\nThe price for the VM will be the current price for spot or the price for a standard VM,\nwhich ever is less, as long as there is capacity and quota available.", "launch_stage": "GA" @@ -18184,6 +18584,11 @@ "ref": "compute.DataSecurityMode", "launch_stage": "GA" }, + "dependency_mode": { + "description": "Controls dependency configuration for the cluster.", + "ref": "compute.DependencyMode", + "launch_stage": "PUBLIC_BETA" + }, "docker_image": { "description": "Custom docker image BYOC", "ref": "compute.DockerImage", @@ -21361,7 +21766,7 @@ "launch_stage": "PRIVATE_PREVIEW" }, "size": { - "description": "Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR\nor PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).", + "description": "Size parameter for the target type, for types that take one (e.g. vector\ndimension, varchar length). Required when the chosen pg_type needs a size.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" @@ -22219,15 +22624,15 @@ "failover_group": { "description": "The failover group to create.", "ref": "disasterrecovery.FailoverGroup", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "failover_group_id": { "description": "Client-provided identifier for the failover group. Used to construct the\nresource name as {parent}/failover-groups/{failover_group_id}.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "validate_only": { "description": "When true, validates the request without creating the failover group.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22239,15 +22644,15 @@ "stable_url": { "description": "The stable URL to create.", "ref": "disasterrecovery.StableUrl", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "stable_url_id": { "description": "Client-provided identifier for the stable URL. Used to construct the\nresource name as {parent}/stable-urls/{stable_url_id}.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "validate_only": { "description": "When true, validates the request without creating the stable URL.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22258,7 +22663,7 @@ "fields": { "etag": { "description": "Opaque version string for optimistic locking. If provided, must match the\ncurrent etag. If omitted, the delete proceeds without an etag check.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22271,7 +22676,7 @@ "fields": { "etag": { "description": "Opaque version string for optimistic locking. If provided, must match the\ncurrent etag. If omitted, the failover proceeds regardless of current state.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22279,11 +22684,11 @@ "failover_type": { "description": "The type of failover to perform.", "ref": "disasterrecovery.FailoverFailoverGroupRequestFailoverType", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "target_primary_region": { "description": "The target primary region. Must be one of the participating regions and different\nfrom the current effective_primary_region. Serves as an idempotency check.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22293,7 +22698,7 @@ "FORCED" ], "enum_launch_stages": { - "FORCED": "PUBLIC_PREVIEW" + "FORCED": "GA" } }, "disasterrecovery.FailoverGroup": { @@ -22301,28 +22706,28 @@ "fields": { "create_time": { "description": "Time at which this failover group was created.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "effective_primary_region": { "description": "Current effective primary region. Replication flows FROM workspaces in this region.\nChanges after a successful failover.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "etag": { "description": "Opaque version string for optimistic locking. Server-generated and returned in responses.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "initial_primary_region": { "description": "Initial primary region. Used only in Create requests to set the starting\nprimary region. Not returned in responses.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE", "INPUT_ONLY" @@ -22330,21 +22735,21 @@ }, "name": { "description": "Fully qualified resource name in the format\naccounts/{account_id}/failover-groups/{failover_group_id}.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE" ] }, "regions": { "description": "List of all regions participating in this failover group.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE" ] }, "replication_point": { "description": "The latest point in time to which data has been replicated.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] @@ -22352,7 +22757,7 @@ "state": { "description": "Aggregate state of the failover group.", "ref": "disasterrecovery.FailoverGroupState", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] @@ -22360,21 +22765,21 @@ "unity_catalog_assets": { "description": "Unity Catalog replication configuration.", "ref": "disasterrecovery.UcReplicationConfig", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "update_time": { "description": "Time at which this failover group was last modified.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "workspace_sets": { "description": "Workspace sets, each containing workspaces that replicate to each other.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22391,14 +22796,14 @@ "DELETION_FAILED" ], "enum_launch_stages": { - "ACTIVE": "PUBLIC_PREVIEW", - "CREATING": "PUBLIC_PREVIEW", - "CREATION_FAILED": "PUBLIC_PREVIEW", - "DELETING": "PUBLIC_PREVIEW", - "DELETION_FAILED": "PUBLIC_PREVIEW", - "FAILING_OVER": "PUBLIC_PREVIEW", - "FAILOVER_FAILED": "PUBLIC_PREVIEW", - "INITIAL_REPLICATION": "PUBLIC_PREVIEW" + "ACTIVE": "GA", + "CREATING": "GA", + "CREATION_FAILED": "GA", + "DELETING": "GA", + "DELETION_FAILED": "GA", + "FAILING_OVER": "GA", + "FAILOVER_FAILED": "GA", + "INITIAL_REPLICATION": "GA" } }, "disasterrecovery.GetFailoverGroupRequest": {}, @@ -22407,14 +22812,14 @@ "fields": { "page_size": { "description": "Maximum number of failover groups to return per page:\n- when set to a value greater than 0, the page length is the minimum of this value\nand a server configured value;\n- when set to 0 or unset, the page length is set to a server configured value\n(recommended);\n- when set to a value less than 0, an invalid parameter error is returned.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "page_token": { "description": "Page token received from a previous ListFailoverGroups call.\nProvide this to retrieve the subsequent page.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22426,11 +22831,11 @@ "fields": { "failover_groups": { "description": "The failover groups for this account.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "next_page_token": { "description": "A token that can be sent as page_token to retrieve the next page.\nIf omitted, there are no subsequent pages.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22438,14 +22843,14 @@ "fields": { "page_size": { "description": "Maximum number of stable URLs to return per page:\n- when set to a value greater than 0, the page length is the minimum of this value\nand a server configured value;\n- when set to 0 or unset, the page length is set to a server configured value\n(recommended);\n- when set to a value less than 0, an invalid parameter error is returned.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "page_token": { "description": "Page token received from a previous ListStableUrls call.\nProvide this to retrieve the subsequent page.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22457,11 +22862,11 @@ "fields": { "next_page_token": { "description": "A token that can be sent as page_token to retrieve the next page.\nIf omitted, there are no subsequent pages.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "stable_urls": { "description": "The stable URLs for this account.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22470,11 +22875,11 @@ "fields": { "name": { "description": "Resource name for this location.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "uri_by_region": { "description": "URI for each region. Each entry maps a region name to a storage URI.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22483,27 +22888,34 @@ "fields": { "region": { "description": "The region name.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "uri": { "description": "The storage URI for this region.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, "disasterrecovery.StableUrl": { "description": "A stable URL provides a failover-aware endpoint for accessing a workspace.\nIts lifecycle is independent of any failover group.", "fields": { + "effective_workspace_id": { + "description": "The workspace this stable URL currently routes to. Set to\n`initial_workspace_id` at creation, advanced to the failover group's primary\nwhile attached (including across a failover), and preserved when the stable\nURL is detached from its failover group. Read this to see where an unattached\nstable URL points: after a failover followed by a detach it reflects the\npost-failover primary, not `initial_workspace_id`.", + "launch_stage": "GA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, "failover_group_name": { "description": "Fully qualified resource name of the FailoverGroup this stable URL is\ncurrently linked to, in the format\n`accounts/{account_id}/failover-groups/{failover_group_id}`. Empty when\nthe stable URL is not attached to any failover group.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "initial_workspace_id": { "description": "The workspace this stable URL is initially bound to. Used only in Create\nrequests to associate the stable URL with a workspace. Not returned in\nresponses.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE", "INPUT_ONLY" @@ -22511,14 +22923,22 @@ }, "name": { "description": "Fully qualified resource name.\nFormat: accounts/{account_id}/stable-urls/{stable_url_id}.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE" ] }, + "stable_workspace_id": { + "description": "The stable workspace ID for this stable URL. Generated on creation and\nimmutable thereafter; identifies the URL across failovers and is the same\nvalue embedded in the `url` (as the `w=` query parameter for SPOG URLs,\nor in the `conn-\u003cid\u003e` hostname for Private-Link URLs).", + "launch_stage": "GA", + "behaviors": [ + "IMMUTABLE", + "OUTPUT_ONLY" + ] + }, "url": { "description": "The stable URL endpoint. Generated on creation and\nimmutable thereafter. For non-Private-Link workspaces this is\n`https://\u003cspog_host\u003e/?w=\u003cconnection_id\u003e`. For Private-Link workspaces\nthis is the per-connection hostname.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "IMMUTABLE", "OUTPUT_ONLY" @@ -22531,7 +22951,7 @@ "fields": { "name": { "description": "The name of the UC catalog to replicate.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22540,15 +22960,15 @@ "fields": { "catalogs": { "description": "UC catalogs to replicate.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "data_replication_workspace_set": { "description": "The workspace set whose workspaces will be used for data replication\nof all UC catalogs' underlying storage.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "location_mappings": { "description": "Location mappings - storage URI per region for each location.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22559,7 +22979,7 @@ "fields": { "etag": { "description": "Optional opaque version string for optimistic locking, obtained from a prior read of\nthe failover group. If provided, the update is rejected unless it matches the failover\ngroup's current etag. If omitted, the update proceeds without an optimistic-lock check.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -22567,11 +22987,11 @@ "failover_group": { "description": "The failover group with updated fields. The name field identifies the resource\nand is populated from the URL path.", "ref": "disasterrecovery.FailoverGroup", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "update_mask": { "description": "Comma-separated list of fields to update.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -22580,25 +23000,25 @@ "fields": { "name": { "description": "Resource name for this workspace set.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "replicate_workspace_assets": { "description": "Whether to enable control plane DR (notebooks, jobs, clusters, etc.) for this set.\nDefaults to false.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "stable_url_names": { "description": "Resource names of stable URLs associated with this workspace set.\nFormat: accounts/{account_id}/stable-urls/{stable_url_id}.\nThe referenced stable URLs must already exist (via CreateStableUrl).", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "workspace_ids": { "description": "Workspace IDs in this set. The system derives and validates regions.\nAll workspaces must be in the Mission Critical tier.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -23155,7 +23575,7 @@ "files.FileInfo": { "fields": { "file_size": { - "description": "The length of the file in bytes. This field is omitted for directories.", + "description": "The length of the file in bytes. Set to 0 for directories.", "launch_stage": "GA" }, "is_dir": { @@ -25595,7 +26015,7 @@ } }, "iamv2.WorkspaceAssignmentDetail": { - "description": "The details of a principal's assignment to a workspace.", + "description": "The direct assignment of a provisioned account-level principal (user, service\nprincipal, or group) to a workspace, together with the entitlements that\nassignment grants in the workspace.\n\nA WorkspaceAssignmentDetail exists only for principals that are directly\nassigned to the workspace; principals that merely inherit workspace access\nthrough a group are not represented here (see WorkspaceAccessDetail /\nWorkspaceIdentityDetail for the effective, direct-or-indirect view). Creating\nthe resource assigns the principal to the workspace; deleting it removes the\nassignment. The `entitlements` field is the only client-settable field and\ndefines the entitlements granted directly on this assignment; `effective_entitlements`\nis the read-only union of those plus any granted via group membership.\n\nA direct assignment always carries at least one directly-assigned entitlement:\nthe assignment is what grants the entitlement, so a WorkspaceAssignmentDetail\nwith an empty `entitlements` set is not a valid state. Both create and update\nrequire a non-empty `entitlements` set (an empty set is rejected); to remove a\nprincipal's assignment entirely, delete the resource.\n\nThis resource replaces workspace assignment previously managed through the\nworkspace SCIM and permission-assignment APIs, and is intended for account\nand workspace admins.", "fields": { "account_id": { "description": "The account ID parent of the workspace where the principal is assigned", @@ -25604,11 +26024,20 @@ "OUTPUT_ONLY" ] }, + "effective_entitlements": { + "description": "The principal's full effective entitlements granted in this workspace: every entitlement it holds\nwhether granted directly or via group membership. Populated on Get; empty on List.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OUTPUT_ONLY", + "UNORDERED_LIST" + ] + }, "entitlements": { "description": "Entitlements granted directly to the principal on this workspace. The only client-settable\nfield: create and update manage exactly this set (including entitlements the principal also\nholds via a group).\nNot populated by ListWorkspaceAssignmentDetails (omitted for scalability); call\nGetWorkspaceAssignmentDetail to read the entitlements for a single principal.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ - "OPTIONAL" + "OPTIONAL", + "UNORDERED_LIST" ] }, "principal_id": { @@ -25646,7 +26075,7 @@ "description": "AiRuntimeTask: multi-node GPU compute task definition for Databricks AI\nRuntime workloads.\n\nJobs-framework-level concepts (retries, per-task timeout, idempotency\ntoken, usage/budget policy, permissions) live on the surrounding\nTaskSettings / run-submit request and are intentionally NOT duplicated\nhere. Users compose `ai_runtime_task` with the standard Jobs/DABs task\nwrapper to get those.", "fields": { "code_source_path": { - "description": "Optional workspace or UC volume path of the uploaded code-source\narchive. The CLI packages the user's local code directory into an\narchive and populates this. Customers calling the Jobs API directly\nshould upload their archive to the workspace or a UC volume first and\nsupply the resulting path here.\n\nWhen set, the training node exposes the value via the `$CODE_SOURCE`\nenvironment variable.", + "description": "Workspace or UC volume path of the code-source archive, unpacked on\neach node and exposed through `$CODE_SOURCE`. Set by first-party\ntooling; not for direct callers.", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" @@ -25838,6 +26267,10 @@ "description": "The creator user name. This field won’t be included in the response if the user has already been deleted.", "launch_stage": "GA" }, + "deployment_id": { + "description": "ID of the deployment that produced the job when this run was created. Used to look up\ndeployment metadata from the Deployment Metadata service. Only set for job runs of jobs\nwith a `BUNDLE` deployment.", + "launch_stage": "PRIVATE_PREVIEW" + }, "description": { "description": "Description of the run", "launch_stage": "GA" @@ -25963,6 +26396,10 @@ "trigger_info": { "ref": "jobs.TriggerInfo", "launch_stage": "GA" + }, + "version_id": { + "description": "ID of the deployment version that produced the job when this run was created. Identifies\na specific snapshot of the deployment in the Deployment Metadata service. Only set for\njob runs of jobs with a `BUNDLE` deployment.", + "launch_stage": "PRIVATE_PREVIEW" } } }, @@ -26174,7 +26611,7 @@ } }, "jobs.ComputeSpecAcceleratorType": { - "description": "Customer-facing AcceleratorType: hardware accelerator type for the\nAiRuntime workload. Per-node accelerator count is encoded in the value\nname (e.g. `GPU_8xH100` means 8 H100s per node).", + "description": "Hardware accelerator type for the AiRuntime workload. Per-node\naccelerator count is encoded in the value name (e.g. `GPU_8xH100` means\n8 H100s per node).", "enum": [ "GPU_1xA10", "GPU_1xH100", @@ -26641,7 +27078,7 @@ "description": "DeploymentSpec: configuration for one deployment within an AiRuntimeTask.\nEach entry in `AiRuntimeTask.deployments` describes a group of nodes that\nshare the same command and compute. Many single-program training\nalgorithms use a single entry where every node runs the same command;\nrole-split workloads (driver + worker, parameter server, separate eval\nnode, etc.) use multiple entries.", "fields": { "command_path": { - "description": "Workspace path of the bash script to execute on each node in this\ndeployment. The CLI uploads the user's script and populates this.\nCustomers calling the Jobs API directly should upload their script to\nthe workspace first and supply the resulting path here.", + "description": "Workspace path of the script to run on each node in this deployment.\nUpload the script to this path and supply the path here. When the task\nruns, the file at this path is run on each node; if it fails, the task\nfails with its exit code.\n\nExample script contents:\n\n# Plain Python:\npython train.py --epochs 10\n\n# Multi-GPU via accelerate:\naccelerate launch train.py --config config.yaml\n\n# Distributed via torchrun:\ntorchrun --nproc_per_node=8 train.py", "launch_stage": "PRIVATE_PREVIEW" }, "compute": { @@ -27127,6 +27564,10 @@ "description": "If new_cluster, a description of a cluster that is created for each task.", "ref": "compute.ClusterSpec", "launch_stage": "GA" + }, + "serverless_compute_id": { + "description": "The ID of the serverless compute object to bind this cluster to. At most one\nJobCluster per job may set this field; the rate limit defined on the referenced\nserverless compute applies across all tasks bound to this cluster.", + "launch_stage": "PRIVATE_PREVIEW" } } }, @@ -27831,11 +28272,13 @@ "enum": [ "HOURS", "DAYS", - "WEEKS" + "WEEKS", + "MINUTES" ], "enum_launch_stages": { "DAYS": "GA", "HOURS": "GA", + "MINUTES": "GA", "WEEKS": "GA" } }, @@ -28318,7 +28761,7 @@ } }, "jobs.ResolvedValuesAiRuntimeTaskResolvedValues": { - "description": "Resolved env_vars for an AiRuntimeTask after dynamic-value substitution.\nMirrors the task's `resolved_parameters_field` (env_vars) so Jobs can\nexpand `{{tasks.\u003ckey\u003e.values.\u003cname\u003e}}` references before submission." + "description": "Resolved values for an AiRuntimeTask after dynamic-value substitution, so\nJobs can expand `{{tasks.\u003ckey\u003e.values.\u003cname\u003e}}` references before\nsubmission." }, "jobs.Run": { "description": "Run was retrieved successfully", @@ -28345,6 +28788,10 @@ "description": "The creator user name. This field won’t be included in the response if the user has already been deleted.", "launch_stage": "GA" }, + "deployment_id": { + "description": "ID of the deployment that produced the job when this run was created. Used to look up\ndeployment metadata from the Deployment Metadata service. Only set for job runs of jobs\nwith a `BUNDLE` deployment.", + "launch_stage": "PRIVATE_PREVIEW" + }, "description": { "description": "Description of the run", "launch_stage": "GA" @@ -28478,6 +28925,10 @@ "trigger_info": { "ref": "jobs.TriggerInfo", "launch_stage": "GA" + }, + "version_id": { + "description": "ID of the deployment version that produced the job when this run was created. Identifies\na specific snapshot of the deployment in the Deployment Metadata service. Only set for\njob runs of jobs with a `BUNDLE` deployment.", + "launch_stage": "PRIVATE_PREVIEW" } } }, @@ -29685,6 +30136,11 @@ "ref": "jobs.JobNotificationSettings", "launch_stage": "GA" }, + "performance_target": { + "description": "The performance mode on a serverless one-time run. This field determines the level of compute performance or cost-efficiency for the run.\nThe performance target does not apply to tasks that run on Serverless GPU compute.\n\n* `STANDARD`: Enables cost-efficient execution of serverless workloads.\n* `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance.", + "ref": "jobs.PerformanceTarget", + "launch_stage": "GA" + }, "queue": { "description": "The queue settings of the one-time run.", "ref": "jobs.QueueSettings", @@ -33238,6 +33694,14 @@ "tags": { "description": "A collection of tags to set on the experiment. Maximum tag size and number of tags per request\ndepends on the storage backend. All storage backends are guaranteed to support tag keys up\nto 250 bytes in size and tag values up to 5000 bytes in size. All storage backends are also\nguaranteed to support up to 20 tags per request.", "launch_stage": "GA" + }, + "trace_location": { + "description": "The location where the experiment's traces are stored. When set, the\nunderlying storage is provisioned and the experiment's traces are routed\nto it. When unset, traces are stored in the default MLflow backend. This\nfield cannot be updated after the experiment is created.", + "ref": "ml.ExperimentTraceLocation", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] } } }, @@ -33904,7 +34368,7 @@ } }, "ml.DirectSchemas": { - "description": "Schema definitions provided directly on the Stream, as opposed to referencing a schema registry.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions provided directly on the Stream, as opposed to referencing a schema registry.\nTo resolve schemas from a registry instead, use SchemaRegistryConfig.", "fields": { "key_schema": { "description": "Schema for the message key. This is only used for Kafka streams.\nFor Kafka, at least one of payload_schema or key_schema must be specified.", @@ -33962,6 +34426,14 @@ "tags": { "description": "Tags: Additional metadata key-value pairs.", "launch_stage": "GA" + }, + "trace_location": { + "description": "The location where the experiment's traces are stored. Unset when traces\nare stored in the default MLflow backend. This field cannot be updated\nafter the experiment is created.", + "ref": "ml.ExperimentTraceLocation", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] } } }, @@ -34075,6 +34547,16 @@ } } }, + "ml.ExperimentTraceLocation": { + "description": "The storage location for an experiment's traces.", + "fields": { + "uc_trace_location": { + "description": "A Unity Catalog schema where the experiment's traces are stored as\nDelta tables.", + "ref": "ml.UcTraceLocation", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "ml.Feature": { "fields": { "catalog_name": { @@ -35639,22 +36121,6 @@ } } }, - "ml.LongRollingWindow": { - "description": "A long (multi-day) rolling window served via the hybrid batch + streaming path. The batch\npipeline maintains daily partial aggregates for the bulk of the window while the streaming\npipeline maintains the most recent day(s), and serving merges them on read. Distinct from\nRollingWindow so the control plane can explicitly identify long rolling window features rather\nthan inferring hybrid behavior from window_duration.", - "fields": { - "delay": { - "description": "The delay applied to the end of the rolling window (must be non-negative).\nFor example, delay=1d shifts the window end 1 day before the evaluation time.", - "launch_stage": "PRIVATE_PREVIEW", - "behaviors": [ - "OPTIONAL" - ] - }, - "window_duration": { - "description": "The duration of the rolling window. Must be positive and span more than two days, so that both\nthe batch (N-1 day) and stale-path (N-2 day) partial aggregates are well defined. The duration\nneed not be a whole number of days (e.g. 3 days 15 minutes is allowed).", - "launch_stage": "PRIVATE_PREVIEW" - } - } - }, "ml.MaterializedFeature": { "description": "A materialized feature represents a feature that is continuously computed and stored.", "fields": { @@ -36641,8 +37107,11 @@ ] }, "window_duration": { - "description": "The duration of the rolling window (must be positive).", - "launch_stage": "PRIVATE_PREVIEW" + "description": "The duration of the rolling window. Must be positive when set; absent means lifetime\n(aggregate over the entity's entire history).", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] } } }, @@ -36774,6 +37243,25 @@ } } }, + "ml.SawtoothWindow": { + "description": "A sawtooth window served via the hybrid batch + streaming path. The batch pipeline maintains\ndaily partial aggregates for the bulk of the window while the streaming pipeline maintains the\nmost recent day(s), and serving merges them on read. Same field shape as RollingWindow, but a\ndistinct type so the control plane can explicitly identify hybrid (sawtooth) features rather\nthan inferring hybrid behavior from window_duration.", + "fields": { + "delay": { + "description": "The delay applied to the end of the window (must be non-negative).\nFor example, delay=1d shifts the window end 1 day before the evaluation time.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "window_duration": { + "description": "The duration of the window. Must be positive and span more than two days when set, so that both\nthe batch (N-1 day) and stale-path (N-2 day) partial aggregates are well defined. The duration\nneed not be a whole number of days (e.g. 3 days 15 minutes is allowed). Absent means lifetime\n(aggregate over the entity's entire history).", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "ml.ScalarDataType": { "description": "Scalar data types for request-time field definitions.\nOnly flat (non-nested) types are supported.", "enum": [ @@ -36829,6 +37317,79 @@ } } }, + "ml.SchemaLocator": { + "description": "Schema locator for one side (payload or key) of a message.\nIdentifies which schema to use in the schema registry and the serialization format.", + "fields": { + "confluent_schema": { + "description": "Confluent Schema Registry schema locator.", + "ref": "ml.SchemaLocatorConfluentSchema", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "format": { + "description": "Serialization format for this schema.", + "ref": "ml.SchemaLocatorFormat", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, + "ml.SchemaLocatorConfluentSchema": { + "description": "Confluent Schema Registry schema locator.\nThe value to provide for `subject` depends on the naming strategy configured in your registry:\n- TopicNameStrategy (default): \"{topic}-key\" or \"{topic}-value\"\ne.g. for topic \"transactions\" use \"transactions-value\" for the payload and \"transactions-key\" for the key.\n- RecordNameStrategy: the fully-qualified record name\ne.g. \"com.example.Payment\" for Avro, the bare message name (without package) for Protobuf,\nor the `title` field value for JSON.\n- TopicRecordNameStrategy: \"{topic}-{fully-qualified-record-name}\"\ne.g. \"transactions-com.example.Payment\".", + "fields": { + "subject": { + "description": "The Confluent schema registry subject name.", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, + "ml.SchemaLocatorFormat": { + "description": "Supported serialization formats for a schema registry schema.", + "enum": [ + "FORMAT_AVRO", + "FORMAT_PROTOBUF", + "FORMAT_JSON" + ], + "enum_launch_stages": { + "FORMAT_AVRO": "PRIVATE_PREVIEW", + "FORMAT_JSON": "PRIVATE_PREVIEW", + "FORMAT_PROTOBUF": "PRIVATE_PREVIEW" + } + }, + "ml.SchemaRegistryConfig": { + "description": "Configuration for resolving a Stream's schema from an external schema registry (e.g. Confluent).", + "fields": { + "api_secret_ref": { + "description": "Reference to the schema registry API secret in a Databricks secret scope.", + "ref": "ml.SecretScopeReference", + "launch_stage": "PRIVATE_PREVIEW" + }, + "key_schema_locator": { + "description": "Schema locator for the message key. Only used for Kafka streams.\nAt least one of payload_schema_locator or key_schema_locator must be set.", + "ref": "ml.SchemaLocator", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "payload_schema_locator": { + "description": "Schema locator for the message payload. For Kafka this is the value.\nAt least one of payload_schema_locator or key_schema_locator must be set.", + "ref": "ml.SchemaLocator", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "uc_connection": { + "description": "A Schema Registry UC Connection object.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "ml.SearchExperiments": { "fields": { "filter": { @@ -37149,8 +37710,11 @@ "launch_stage": "PRIVATE_PREVIEW" }, "window_duration": { - "description": "The duration of the sliding window.", - "launch_stage": "PRIVATE_PREVIEW" + "description": "The duration of the sliding window. Must be positive when set; absent means lifetime\n(aggregate over the entity's entire history).", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] } } }, @@ -37236,7 +37800,7 @@ "launch_stage": "PRIVATE_PREVIEW" }, "schema_config": { - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "ref": "ml.StreamSchemaConfig", "launch_stage": "PRIVATE_PREVIEW" }, @@ -37282,7 +37846,7 @@ } }, "ml.StreamSchemaConfig": { - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream.\nFeature store supports both direct schemas and schema registries.", "fields": { "direct_schemas": { "description": "Schema definitions provided directly on the Stream.", @@ -37291,12 +37855,27 @@ "behaviors": [ "OPTIONAL" ] + }, + "schema_registry_config": { + "description": "Resolve schemas from an external schema registry.", + "ref": "ml.SchemaRegistryConfig", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] } } }, "ml.StreamSource": { "description": "A Stream entity used as a data source for a feature.", "fields": { + "dataframe_schema": { + "description": "Schema of the resulting dataframe after transformations, in Spark StructType\nJSON format (from df.schema.json()).\nAny subsequent functions operate against this dataframe.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, "filter_condition": { "description": "The filter condition applied to the source data before aggregation.", "launch_stage": "PRIVATE_PREVIEW", @@ -37307,6 +37886,13 @@ "full_name": { "description": "Three-part full name of the Stream (catalog.schema.stream).", "launch_stage": "PRIVATE_PREVIEW" + }, + "transformation_sql": { + "description": "The pipeline runs these SQL statements immediately after conversion into\nthe schema specified on the Stream object.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] } } }, @@ -37427,16 +38013,16 @@ "OPTIONAL" ] }, - "long_rolling": { - "description": "A long (multi-day) rolling window served via the hybrid batch + streaming path.", - "ref": "ml.LongRollingWindow", + "rolling": { + "ref": "ml.RollingWindow", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" ] }, - "rolling": { - "ref": "ml.RollingWindow", + "sawtooth": { + "description": "A sawtooth window served via the hybrid batch + streaming path.", + "ref": "ml.SawtoothWindow", "launch_stage": "PRIVATE_PREVIEW", "behaviors": [ "OPTIONAL" @@ -37533,6 +38119,39 @@ } } }, + "ml.UcTraceLocation": { + "description": "A Unity Catalog trace storage location. Traces are stored as Delta tables\nin the specified catalog and schema.", + "fields": { + "catalog": { + "description": "The name of the Unity Catalog catalog.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "effective_table_prefix": { + "description": "The trace-table prefix actually in effect: `table_prefix` if it was set on\ncreation, otherwise the server-generated default.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "schema": { + "description": "The name of the Unity Catalog schema within `catalog`.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "table_prefix": { + "description": "The prefix for the trace tables, which are named\n`{catalog}.{schema}.{table_prefix}_otel_*`. May only contain letters,\ndigits, and underscores, and may be at most 238 characters. When unset, a\nserver-generated prefix derived from the experiment ID is used and this\nfield stays empty on read; the resolved value is always available in\n`effective_table_prefix`.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + } + } + }, "ml.UpdateComment": { "description": "Details required to edit a comment on a model version.", "fields": { @@ -37813,6 +38432,33 @@ "DELETED_ONLY": "GA" } }, + "networking.AwsVpcEndpointInfo": { + "fields": { + "aws_account_id": { + "description": "The AWS account ID in which this VPC endpoint lives.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OUTPUT_ONLY" + ] + }, + "aws_endpoint_service_id": { + "description": "The ID of the Databricks VPC endpoint service that this endpoint connects to.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OUTPUT_ONLY" + ] + }, + "aws_vpc_endpoint_id": { + "description": "The ID of the underlying VPC endpoint in AWS. Provided by the customer when\nregistering an existing AWS VPC endpoint.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + } + } + }, "networking.AzurePrivateEndpointInfo": { "fields": { "private_endpoint_name": { @@ -37859,6 +38505,15 @@ "OUTPUT_ONLY" ] }, + "aws_vpc_endpoint_info": { + "description": "Info for an AWS VPC endpoint.", + "ref": "networking.AwsVpcEndpointInfo", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OPTIONAL" + ] + }, "azure_private_endpoint_info": { "description": "Info for an Azure private endpoint.", "ref": "networking.AzurePrivateEndpointInfo", @@ -37891,6 +38546,15 @@ "OUTPUT_ONLY" ] }, + "gcp_psc_endpoint_info": { + "description": "Info for a GCP Private Service Connect endpoint.", + "ref": "networking.GcpPscEndpointInfo", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OPTIONAL" + ] + }, "name": { "description": "The resource name of the endpoint, which uniquely identifies the endpoint.", "launch_stage": "GA", @@ -37947,6 +38611,47 @@ "SERVICE_DIRECT": "GA" } }, + "networking.GcpPscEndpointInfo": { + "fields": { + "endpoint_region": { + "description": "The GCP region of the PSC connection endpoint. Provided by the customer when registering an\nexisting PSC endpoint. GCP supports only same-region PSC, so this must match the workspace\nregion.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "project_id": { + "description": "The GCP consumer project ID in which this PSC endpoint is created. Provided by the customer\nwhen registering an existing PSC endpoint.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "psc_connection_id": { + "description": "The ID of the underlying Private Service Connect connection in the GCP consumer project,\nassigned by GCP when the PSC connection is created.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OUTPUT_ONLY" + ] + }, + "psc_endpoint": { + "description": "The name of this PSC connection in the GCP consumer project. Provided by the customer when\nregistering an existing PSC endpoint.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE" + ] + }, + "service_attachment_id": { + "description": "The ID of the Databricks service attachment this PSC endpoint connects to.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "IMMUTABLE", + "OUTPUT_ONLY" + ] + } + } + }, "networking.GetEndpointRequest": {}, "networking.ListEndpointsRequest": { "fields": { @@ -38796,7 +39501,7 @@ }, "kafka_options": { "ref": "pipelines.KafkaOptions", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "meta_ads_options": { "ref": "pipelines.MetaMarketingOptions", @@ -38806,6 +39511,10 @@ "ref": "pipelines.OutlookOptions", "launch_stage": "PRIVATE_PREVIEW" }, + "reddit_ads_options": { + "ref": "pipelines.RedditAdsOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, "sharepoint_options": { "ref": "pipelines.SharepointOptions", "launch_stage": "PRIVATE_PREVIEW" @@ -39358,11 +40067,11 @@ "NONE" ], "enum_launch_stages": { - "ADD_NEW_COLUMNS": "PRIVATE_PREVIEW", - "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING": "PRIVATE_PREVIEW", - "FAIL_ON_NEW_COLUMNS": "PRIVATE_PREVIEW", - "NONE": "PRIVATE_PREVIEW", - "RESCUE": "PRIVATE_PREVIEW" + "ADD_NEW_COLUMNS": "PUBLIC_BETA", + "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING": "PUBLIC_BETA", + "FAIL_ON_NEW_COLUMNS": "PUBLIC_BETA", + "NONE": "PUBLIC_BETA", + "RESCUE": "PUBLIC_BETA" } }, "pipelines.FileLibrary": { @@ -39494,9 +40203,35 @@ } } }, + "pipelines.GoogleAdsCustomReportOptions": { + "description": "User-defined custom report for the Google Ads connector. Mirrors the\nresource + fields + segments + metrics model that Google Ads GAQL exposes.\nThe customer account this report runs against is supplied by the source\nschema (namespace), not by this message. The whole message is gated by\nthe parent GoogleAdsOptions.custom_report_options stage; per-field stage\nannotations are intentionally omitted.\nOnly supported on table-type objects: a custom report requires a\ndestination table, so it cannot be specified at the schema/source level.", + "fields": { + "metrics": { + "description": "(Optional) Metric fields to select (e.g. \"metrics.clicks\",\n\"metrics.cost_micros\"). Multiple values are joined into the GAQL\nSELECT clause.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "resource": { + "description": "(Required) Google Ads resource to query (e.g. \"ad_group_ad\",\n\"keyword_view\", \"search_term_view\"). Must be a resource that has\nmetrics. Values are validated against Google Ads' field-service\ncatalog at pipeline plan time.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "resource_fields": { + "description": "(Optional) Resource fields to select, in fully-qualified GAQL form\n(e.g. \"ad_group_ad.ad.id\", \"ad_group_ad.status\"). Multiple values are\njoined into the GAQL SELECT clause.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "segments": { + "description": "(Optional) Segment fields to select (e.g. \"segments.date\",\n\"segments.device\"). Must include at least one of segments.date,\nsegments.week, or segments.month — that segment is used as the\nincremental cursor for the table.", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "pipelines.GoogleAdsOptions": { "description": "Google Ads specific options for ingestion (object-level).\nWhen set, these values override the corresponding fields in GoogleAdsConfig\n(source_configurations).", "fields": { + "custom_report_options": { + "description": "(Optional) Custom report definition. When set, the table is treated as a\nuser-defined Google Ads custom report: the connector synthesizes a GAQL\nquery from the resource, fields, segments, and metrics specified here.\nWhen unset, the table must match one of the connector's prebuilt sources.", + "ref": "pipelines.GoogleAdsCustomReportOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, "lookback_window_days": { "description": "(Optional) Number of days to look back for report tables to capture late-arriving data.\nIf not specified, defaults to 30 days.", "launch_stage": "PRIVATE_PREVIEW" @@ -39644,6 +40379,19 @@ } } }, + "pipelines.IngestionPipelineDefinitionFanoutOptions": { + "description": "Fanout configuration for multi-table routing from streaming sources.\nRoutes each input record to a destination table based on a routing\nkey derived from the record. The key value becomes the table name\nsuffix: {destination_catalog}.{destination_schema}.{key_value}.", + "fields": { + "fanout_by": { + "description": "Column path or SQL expression whose value determines the destination table.\nSupports dotted paths (e.g. \"value.event_name\") and expressions\n(e.g. \"value:event_name::string\").", + "launch_stage": "PRIVATE_PREVIEW" + }, + "transforms": { + "description": "Optional transforms applied to each route's DataFrame before writing\nto the destination table.", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "pipelines.IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig": { "description": "Configurations that are only applicable for query-based ingestion connectors.", "fields": { @@ -39750,24 +40498,24 @@ "fields": { "as_variant": { "description": "Parse the entire value as a single Variant column.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "schema": { "description": "Inline schema string for JSON parsing (Spark DDL format).", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "schema_evolution_mode": { "description": "(Optional) Schema evolution mode for schema inference.", "ref": "pipelines.FileIngestionOptionsSchemaEvolutionMode", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "schema_file_path": { "description": "Path to a schema file (.ddl).", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "schema_hints": { "description": "(Optional) Schema hints as a comma-separated string of \"column_name type\" pairs.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" } } }, @@ -39780,7 +40528,7 @@ "key_transformer": { "description": "(Optional) Transformer for the message key.\nIf not specified, the key is left as raw bytes.", "ref": "pipelines.Transformer", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "max_offsets_per_trigger": { "description": "Internal option to control the maximum number of offsets to process per trigger.", @@ -39788,20 +40536,20 @@ }, "starting_offset": { "description": "(Optional) Where to begin reading when no checkpoint exists.\nValid values: \"latest\" and \"earliest\". Defaults to \"latest\".", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "topic_pattern": { "description": "Java regex pattern to subscribe to matching topics.\nOnly one of topics or topic_pattern must be specified.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "topics": { "description": "Topics to subscribe to.\nOnly one of topics or topic_pattern must be specified.", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "value_transformer": { "description": "(Optional) Transformer for the message value.\nIf not specified, the value is left as raw bytes.", "ref": "pipelines.Transformer", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" } } }, @@ -39945,6 +40693,11 @@ "description": "(Optional) Window in days to revisit data during sync to capture\nupdated conversion data from the API, shared by prebuilt and custom reports.", "launch_stage": "PUBLIC_BETA" }, + "custom_report_options": { + "description": "(Optional) Per-table custom report definition. When set, defines the shape of the insights\ncall for this table (level/fields/breakdowns/action_breakdowns/etc.). Supersedes the deprecated\nflat report-shape fields above.", + "ref": "pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, "level": { "description": "(Optional, DEPRECATED — use custom_report_options.level) Granularity of data to pull\n(account, ad, adset, campaign)", "deprecated": true, @@ -39961,6 +40714,35 @@ } } }, + "pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions": { + "description": "Defines the shape of a single Meta Ads custom report (one /insights call shape).\nstart_date, custom_insights_lookback_window live on MetaMarketingOptions, not here.\nMetrics are not customer-selectable; the connector returns a fixed standard metric set.", + "fields": { + "action_attribution_windows": { + "description": "(Optional) Action attribution windows for insights reporting (e.g. \"28d_click\", \"1d_view\")", + "launch_stage": "PRIVATE_PREVIEW" + }, + "action_breakdowns": { + "description": "(Optional) Action breakdowns to configure for data aggregation", + "launch_stage": "PRIVATE_PREVIEW" + }, + "action_report_time": { + "description": "(Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime)", + "launch_stage": "PRIVATE_PREVIEW" + }, + "breakdowns": { + "description": "(Optional) Breakdowns to configure for data aggregation", + "launch_stage": "PRIVATE_PREVIEW" + }, + "level": { + "description": "(Optional) Granularity of data to pull (account, ad, adset, campaign)", + "launch_stage": "PRIVATE_PREVIEW" + }, + "time_increment": { + "description": "(Optional) Value in string by which to aggregate statistics (all_days, monthly or number of days)", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "pipelines.NotebookLibrary": { "fields": { "path": { @@ -40752,6 +41534,37 @@ "LEGACY_PUBLISHING_MODE": "GA" } }, + "pipelines.RedditAdsOptions": { + "description": "Reddit Ads specific options for ingestion", + "fields": { + "custom_report_options": { + "description": "(Optional) Custom report definition. When set, the table is treated as a\nuser-defined Reddit Ads custom report. When unset, the table must match\none of the connector's prebuilt sources.", + "ref": "pipelines.RedditAdsOptionsRedditAdsCustomReportOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, + "lookback_window_days": { + "description": "(Optional) Number of days to look back for report tables during incremental sync\nto capture late-arriving conversions and attribution data.\nIf not specified, defaults to 30 days.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "sync_start_date": { + "description": "(Optional) Start date for the initial sync of report tables in YYYY-MM-DD format.\nThis determines the earliest date from which to sync historical data.\nIf not specified, defaults to 2 years ago.", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, + "pipelines.RedditAdsOptionsRedditAdsCustomReportOptions": { + "description": "User-defined custom report for the Reddit Ads connector.\nApplies only to the custom_report table — prebuilt tables ignore this.", + "fields": { + "breakdowns": { + "description": "(Optional) Breakdown dimensions to group report data by.\nExamples: CAMPAIGN_ID, DATE, COUNTRY, REGION, AD_ID.\nMust include at least one time dimension (DATE or HOUR).", + "launch_stage": "PRIVATE_PREVIEW" + }, + "fields": { + "description": "(Optional) Fields to include in the report (maps to the Reddit Ads API `fields` parameter).\nExamples: IMPRESSIONS, CLICKS, SPEND, CPC, CTR.", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "pipelines.ReplaceWhereOverride": { "description": "Specifies a replace_where predicate override for a replace where flow.", "fields": { @@ -40868,6 +41681,11 @@ "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.", "launch_stage": "PUBLIC_PREVIEW" }, + "fanout_options": { + "description": "Fanout options for multi-table routing from streaming sources.\nWhen set, records are routed to destination tables based on a\nper-record routing key. The key value becomes the table name:\n{destination_catalog}.{destination_schema}.{key_value}.", + "ref": "pipelines.IngestionPipelineDefinitionFanoutOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, "source_catalog": { "description": "The source catalog name. Might be optional depending on the type of source.", "launch_stage": "PUBLIC_PREVIEW" @@ -41156,6 +41974,10 @@ "description": "The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order.", "launch_stage": "PUBLIC_PREVIEW" }, + "source_metadata_column": { + "description": "(Optional) Name of the struct column added to each ingested record to hold per row source\nmetadata.", + "launch_stage": "PRIVATE_PREVIEW" + }, "table_properties": { "description": "Table properties to set on the destination table.\nThese are key-value pairs that configure various Delta table behaviors or any user defined properties.\nExample: {\"delta.feature.variantType\": \"supported\", \"delta.enableTypeWidening\": \"true\"}\nNote: table_properties in table specific configuration will override the table_properties of the pipeline definition.", "launch_stage": "PUBLIC_BETA" @@ -41183,6 +42005,11 @@ "pipelines.TikTokAdsOptions": { "description": "TikTok Ads specific options for ingestion", "fields": { + "custom_report_options": { + "description": "(Optional) Custom report definition. When set, the table is treated as a\nuser-defined TikTok Ads custom report: the connector synthesizes a report\nrequest from the dimensions, metrics, report type, and data level specified\nhere. Supersedes the deprecated top-level dimensions/metrics/report_type/\ndata_level/query_lifetime fields above.", + "ref": "pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions", + "launch_stage": "PRIVATE_PREVIEW" + }, "data_level": { "description": "Deprecated. Use custom_report_options.data_level instead.", "ref": "pipelines.TikTokAdsOptionsTikTokDataLevel", @@ -41220,6 +42047,33 @@ } } }, + "pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions": { + "description": "User-defined custom report for the TikTok Ads connector. Groups the\ndimensions + metrics + report type + data level that define a TikTok Ads\ncustom report request.", + "fields": { + "data_level": { + "description": "(Optional) Data level for the report.\nIf not specified, defaults to AUCTION_CAMPAIGN.", + "ref": "pipelines.TikTokAdsOptionsTikTokDataLevel", + "launch_stage": "PRIVATE_PREVIEW" + }, + "dimensions": { + "description": "(Optional) Dimensions to include in the report (e.g. \"campaign_id\",\n\"adgroup_id\", \"ad_id\", \"stat_time_day\", \"stat_time_hour\").", + "launch_stage": "PRIVATE_PREVIEW" + }, + "metrics": { + "description": "(Optional) Metrics to include in the report (e.g. \"spend\", \"impressions\",\n\"clicks\", \"conversion\", \"cpc\").", + "launch_stage": "PRIVATE_PREVIEW" + }, + "query_lifetime": { + "description": "(Optional) Whether to request lifetime metrics (all-time aggregated data).\nWhen true, the report returns all-time data.\nIf not specified, defaults to false.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "report_type": { + "description": "(Optional) Report type for the TikTok Ads API.\nIf not specified, defaults to BASIC.", + "ref": "pipelines.TikTokAdsOptionsTikTokReportType", + "launch_stage": "PRIVATE_PREVIEW" + } + } + }, "pipelines.TikTokAdsOptionsTikTokDataLevel": { "description": "Data level for TikTok Ads report aggregation.", "enum": [ @@ -41260,11 +42114,11 @@ "format": { "description": "Required: the wire format of the data.", "ref": "pipelines.TransformerFormat", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" }, "json_options": { "ref": "pipelines.JsonTransformerOptions", - "launch_stage": "PRIVATE_PREVIEW" + "launch_stage": "PUBLIC_BETA" } } }, @@ -41274,8 +42128,8 @@ "JSON" ], "enum_launch_stages": { - "JSON": "PRIVATE_PREVIEW", - "STRING": "PRIVATE_PREVIEW" + "JSON": "PUBLIC_BETA", + "STRING": "PUBLIC_BETA" } }, "pipelines.Truncation": { @@ -41792,6 +42646,127 @@ } }, "postgres.CatalogOperationMetadata": {}, + "postgres.CdfConfig": { + "description": "A Lakebase CDF configuration (CdfConfig): one per Postgres schema per\ndatabase, replicating that schema's tables into a Unity Catalog schema.\nImmutable once created.", + "fields": { + "catalog": { + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "IMMUTABLE" + ] + }, + "cdf_config_id": { + "description": "The user-specified id; equals the final segment of `name`. Defaults to the\nPostgres schema name for configs without an explicit id.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "create_time": { + "description": "When the CdfConfig was created.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "name": { + "description": "Output only. The full resource name of the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "launch_stage": "PUBLIC_BETA" + }, + "postgres_schema": { + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "IMMUTABLE" + ] + }, + "schema": { + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "IMMUTABLE" + ] + } + } + }, + "postgres.CdfConfigOperationMetadata": { + "description": "Metadata for CdfConfig long-running operations. Intentionally empty today;\nfields (e.g. progress) may be added as the operation contract grows." + }, + "postgres.CdfState": { + "description": "The replication state of a single replicated table (CdfStatus).", + "enum": [ + "CDF_STATE_SNAPSHOTTING", + "CDF_STATE_STREAMING", + "CDF_STATE_TERMINATED", + "CDF_STATE_SKIPPED" + ], + "enum_launch_stages": { + "CDF_STATE_SKIPPED": "PUBLIC_BETA", + "CDF_STATE_SNAPSHOTTING": "PUBLIC_BETA", + "CDF_STATE_STREAMING": "PUBLIC_BETA", + "CDF_STATE_TERMINATED": "PUBLIC_BETA" + } + }, + "postgres.CdfStatus": { + "description": "The read-only replication status of a single Postgres table replicated\nunder a CdfConfig. One status exists per replicated table.\nIt is created automatically and cannot be modified.", + "fields": { + "committed_lsn": { + "description": "The high-watermark Log Sequence Number (LSN) committed to Delta Lake.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "create_time": { + "description": "When replication for this table was first established.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "last_sync_time": { + "description": "The last time changes for this table were written to Delta Lake.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "name": { + "description": "Output only. The full resource name of the CdfStatus.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}\nThe {cdf_status} segment is the Postgres table name.", + "launch_stage": "PUBLIC_BETA" + }, + "postgres_table": { + "description": "The Postgres table being replicated.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "state": { + "description": "The current replication state of this table.", + "ref": "postgres.CdfState", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "status_detail": { + "description": "Human-readable detail for the current state (e.g. the skip/error reason).\nEmpty for healthy states.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + }, + "uc_table": { + "description": "The Unity Catalog table receiving replicated data.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OUTPUT_ONLY" + ] + } + } + }, "postgres.CreateBranchRequest": { "fields": { "branch": { @@ -41825,6 +42800,22 @@ } } }, + "postgres.CreateCdfConfigRequest": { + "fields": { + "cdf_config": { + "description": "The CdfConfig to create. The catalog, schema, and postgres_schema fields are\nrequired; all other fields are output only and ignored on input.", + "ref": "postgres.CdfConfig", + "launch_stage": "PUBLIC_BETA" + }, + "cdf_config_id": { + "description": "The user-specified id for the CdfConfig, forming the final segment of its\nresource name. Must match the pattern `[a-z][a-z0-9_]{0,62}`. Defaults to\nthe Postgres schema name when omitted.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "postgres.CreateDataApiRequest": { "fields": { "data_api": { @@ -42205,10 +43196,7 @@ }, "role": { "description": "The name of the role that owns the database.\nFormat: projects/{project_id}/branches/{branch_id}/roles/{role_id}\n\nTo change the owner, pass valid existing Role name when updating the Database\n\nA database always has an owner.", - "launch_stage": "PUBLIC_BETA", - "behaviors": [ - "OPTIONAL" - ] + "launch_stage": "PUBLIC_BETA" } } }, @@ -42268,6 +43256,17 @@ } }, "postgres.DeleteCatalogRequest": {}, + "postgres.DeleteCdfConfigRequest": { + "fields": { + "force": { + "description": "When true, also drops the replicated Delta tables in Unity Catalog. When\nfalse (the default), the replicated tables are preserved at their last\nsynced state.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "postgres.DeleteDataApiRequest": {}, "postgres.DeleteDatabaseRequest": {}, "postgres.DeleteEndpointRequest": {}, @@ -42832,6 +43831,8 @@ }, "postgres.GetBranchRequest": {}, "postgres.GetCatalogRequest": {}, + "postgres.GetCdfConfigRequest": {}, + "postgres.GetCdfStatusRequest": {}, "postgres.GetDataApiRequest": {}, "postgres.GetDatabaseRequest": {}, "postgres.GetEndpointRequest": {}, @@ -42844,7 +43845,7 @@ "fields": { "is_protected": { "description": "Whether the initial default branch should be protected from deletion.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" ] @@ -42856,14 +43857,14 @@ "fields": { "autoscaling_limit_max_cu": { "description": "The maximum number of Compute Units for the initial endpoint.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" ] }, "autoscaling_limit_min_cu": { "description": "The minimum number of Compute Units for the initial endpoint.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" ] @@ -42878,14 +43879,14 @@ }, "no_suspension": { "description": "When set to true, explicitly disables automatic suspension (never suspend).\nShould be set to true when provided.\nMutually exclusive with `suspend_timeout_duration`.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" ] }, "suspend_timeout_duration": { "description": "Duration of inactivity after which the initial endpoint is automatically suspended.\nIf specified, should be between 60s and 604800s (1 minute to 1 week).\nMutually exclusive with `no_suspension`.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" ] @@ -42929,6 +43930,68 @@ } } }, + "postgres.ListCdfConfigsRequest": { + "fields": { + "page_size": { + "description": "Maximum number of CdfConfigs to return.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + }, + "page_token": { + "description": "Pagination token returned by a previous ListCdfConfigs call. Empty on the\nfirst page.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, + "postgres.ListCdfConfigsResponse": { + "description": "Response to a ListCdfConfigs request, containing a page of CdfConfigs and a\ntoken for fetching the next page.", + "fields": { + "cdf_configs": { + "description": "The CdfConfigs under the parent database.", + "launch_stage": "PUBLIC_BETA" + }, + "next_page_token": { + "description": "Token to retrieve the next page of results; empty when there are no more.", + "launch_stage": "PUBLIC_BETA" + } + } + }, + "postgres.ListCdfStatusesRequest": { + "fields": { + "page_size": { + "description": "Maximum number of CdfStatuses to return.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + }, + "page_token": { + "description": "Pagination token returned by a previous ListCdfStatuses call. Empty on the\nfirst page.", + "launch_stage": "PUBLIC_BETA", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, + "postgres.ListCdfStatusesResponse": { + "description": "Response to a ListCdfStatuses request, containing a page of replicated table\nstatuses and a token for fetching the next page.", + "fields": { + "cdf_statuses": { + "description": "The replicated tables under the parent CdfConfig.", + "launch_stage": "PUBLIC_BETA" + }, + "next_page_token": { + "description": "Token to retrieve the next page of results; empty when there are no more.", + "launch_stage": "PUBLIC_BETA" + } + } + }, "postgres.ListDatabasesRequest": { "fields": { "page_size": { @@ -43143,7 +44206,7 @@ "initial_branch_spec": { "description": "Configuration for the initial default branch created as part of project creation.\nAllows overriding branch protection. These settings only apply at creation time\nand do not affect resources created after project creation.", "ref": "postgres.InitialBranchSpec", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "INPUT_ONLY", "OPTIONAL" @@ -43341,7 +44404,7 @@ }, "compute_last_active_time": { "description": "The most recent time when any endpoint of this project was active.", - "launch_stage": "PRIVATE_PREVIEW", + "launch_stage": "PUBLIC_BETA", "behaviors": [ "OUTPUT_ONLY" ] @@ -43850,6 +44913,13 @@ "OPTIONAL" ] }, + "extra_columns": { + "description": "Extra PostgreSQL-only columns to add to the synced table.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, "new_pipeline_spec": { "description": "Specification for creating a new pipeline.\nAt most one of existing_pipeline_id and new_pipeline_spec should be defined.\n\nThe pipeline used for the synced table is returned via the top level pipeline_id attribute.", "ref": "postgres.NewPipelineSpec", @@ -43903,6 +44973,42 @@ } } }, + "postgres.SyncedTableSyncedTableSpecExtraColumn": { + "description": "An extra PostgreSQL column to add to the synced table.", + "fields": { + "column_name": { + "description": "Name of the column.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "column_type": { + "description": "PostgreSQL type of the column, for example \"tsvector\" or \"vector(1024)\".", + "launch_stage": "PRIVATE_PREVIEW" + }, + "compute": { + "description": "SQL expression used to compute the column's value, for example\n\"to_tsvector('english', content)\".", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "maintenance": { + "ref": "postgres.SyncedTableSyncedTableSpecExtraColumnMaintenance", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, + "postgres.SyncedTableSyncedTableSpecExtraColumnMaintenance": { + "description": "How the column's value is populated and kept up to date.", + "enum": [ + "STORED_GENERATED" + ], + "enum_launch_stages": { + "STORED_GENERATED": "PRIVATE_PREVIEW" + } + }, "postgres.SyncedTableSyncedTableSpecPgSpecificType": { "description": "PostgreSQL-specific target types that can override the default Delta-to-PG mapping.", "enum": [ @@ -43938,7 +45044,7 @@ "launch_stage": "PUBLIC_BETA" }, "size": { - "description": "Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR\nor PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).", + "description": "Size parameter for the target type, for types that take one (e.g. vector\ndimension, varchar length). Required when the chosen pg_type needs a size.", "launch_stage": "PUBLIC_BETA", "behaviors": [ "OPTIONAL" @@ -45405,7 +46511,10 @@ }, "ai21labs_api_key_plaintext": { "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -45579,7 +46688,10 @@ }, "aws_access_key_id_plaintext": { "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "aws_region": { "description": "The AWS region to use. Bedrock has to be enabled there.", @@ -45591,7 +46703,10 @@ }, "aws_secret_access_key_plaintext": { "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", @@ -45626,7 +46741,10 @@ }, "anthropic_api_key_plaintext": { "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -45642,7 +46760,10 @@ }, "value_plaintext": { "description": "The API Key provided as a plaintext string. If you prefer to reference your\ntoken using Databricks Secrets, see `value`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -45708,7 +46829,10 @@ }, "token_plaintext": { "description": "The token provided as a plaintext string. If you prefer to reference your\ntoken using Databricks Secrets, see `token`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -45759,7 +46883,10 @@ }, "cohere_api_key_plaintext": { "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -45882,7 +47009,10 @@ }, "databricks_api_token_plaintext": { "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "databricks_workspace_url": { "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.", @@ -46307,7 +47437,10 @@ }, "private_key_plaintext": { "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "project_id": { "description": "This is the Google Cloud project id that the service account is\nassociated with.", @@ -46359,7 +47492,10 @@ }, "microsoft_entra_client_secret_plaintext": { "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "microsoft_entra_tenant_id": { "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.", @@ -46375,7 +47511,10 @@ }, "openai_api_key_plaintext": { "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] }, "openai_api_type": { "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.", @@ -46403,7 +47542,10 @@ }, "palm_api_key_plaintext": { "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.", - "launch_stage": "GA" + "launch_stage": "GA", + "behaviors": [ + "INPUT_ONLY" + ] } } }, @@ -46419,6 +47561,16 @@ } } }, + "serving.PatchTelemetryConfigRequest": { + "description": "Updates the telemetry configuration of a serving endpoint.", + "fields": { + "telemetry_config": { + "description": "The telemetry configuration to be applied to the serving endpoint.\nCan specify either a telemetry_profile_id to use an existing profile,\nor table_names to create a new profile with the specified Unity Catalog tables.\nIf not provided, the telemetry configuration will be removed from the endpoint.", + "ref": "serving.TelemetryConfig", + "launch_stage": "GA" + } + } + }, "serving.PayloadTable": { "fields": { "name": { @@ -46714,8 +47866,85 @@ "fields": { "burst_scaling_enabled": { "description": "Whether burst scaling is enabled. When enabled (default), the endpoint can automatically\nscale up beyond provisioned capacity to handle traffic spikes. When disabled, the endpoint\nmaintains fixed capacity at provisioned_model_units.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "INPUT_ONLY" + ] + }, + "entity_name": { + "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.", + "launch_stage": "GA" + }, + "entity_version": { + "launch_stage": "GA" + }, + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "launch_stage": "GA" + }, + "external_model": { + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.", + "ref": "serving.ExternalModel", + "launch_stage": "GA" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", "launch_stage": "PUBLIC_PREVIEW" }, + "max_provisioned_concurrency": { + "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", + "launch_stage": "GA" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "launch_stage": "GA" + }, + "min_provisioned_concurrency": { + "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", + "launch_stage": "GA" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "launch_stage": "GA" + }, + "name": { + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", + "launch_stage": "GA" + }, + "provisioned_model_units": { + "description": "The number of model units provisioned.", + "launch_stage": "PUBLIC_PREVIEW" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served entity should scale down to zero.", + "launch_stage": "GA" + }, + "workload_size": { + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", + "launch_stage": "GA" + }, + "workload_type": { + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", + "ref": "serving.ServingModelWorkloadType", + "launch_stage": "GA" + } + } + }, + "serving.ServedEntityOutput": { + "fields": { + "burst_scaling_enabled": { + "description": "Whether burst scaling is enabled. When enabled (default), the endpoint can automatically\nscale up beyond provisioned capacity to handle traffic spikes. When disabled, the endpoint\nmaintains fixed capacity at provisioned_model_units.", + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "INPUT_ONLY" + ] + }, + "creation_timestamp": { + "launch_stage": "GA" + }, + "creator": { + "launch_stage": "GA" + }, "entity_name": { "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.", "launch_stage": "GA" @@ -46731,81 +47960,10 @@ "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.", "ref": "serving.ExternalModel", "launch_stage": "GA" - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served entity uses to access AWS resources.", - "launch_stage": "PUBLIC_PREVIEW" - }, - "max_provisioned_concurrency": { - "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", - "launch_stage": "GA" - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to.", - "launch_stage": "GA" - }, - "min_provisioned_concurrency": { - "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", - "launch_stage": "GA" - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to.", - "launch_stage": "GA" - }, - "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", - "launch_stage": "GA" - }, - "provisioned_model_units": { - "description": "The number of model units provisioned.", - "launch_stage": "PUBLIC_PREVIEW" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served entity should scale down to zero.", - "launch_stage": "GA" - }, - "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", - "launch_stage": "GA" - }, - "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", - "ref": "serving.ServingModelWorkloadType", - "launch_stage": "GA" - } - } - }, - "serving.ServedEntityOutput": { - "fields": { - "burst_scaling_enabled": { - "description": "Whether burst scaling is enabled. When enabled (default), the endpoint can automatically\nscale up beyond provisioned capacity to handle traffic spikes. When disabled, the endpoint\nmaintains fixed capacity at provisioned_model_units.", - "launch_stage": "PUBLIC_PREVIEW" - }, - "creation_timestamp": { - "launch_stage": "GA" - }, - "creator": { - "launch_stage": "GA" - }, - "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.", - "launch_stage": "GA" - }, - "entity_version": { - "launch_stage": "GA" - }, - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "launch_stage": "GA" - }, - "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.", - "ref": "serving.ExternalModel", - "launch_stage": "GA" - }, - "foundation_model": { - "ref": "serving.FoundationModel", - "launch_stage": "PUBLIC_PREVIEW" + }, + "foundation_model": { + "ref": "serving.FoundationModel", + "launch_stage": "PUBLIC_PREVIEW" }, "instance_profile_arn": { "description": "ARN of the instance profile that the served entity uses to access AWS resources.", @@ -46879,7 +48037,10 @@ "fields": { "burst_scaling_enabled": { "description": "Whether burst scaling is enabled. When enabled (default), the endpoint can automatically\nscale up beyond provisioned capacity to handle traffic spikes. When disabled, the endpoint\nmaintains fixed capacity at provisioned_model_units.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "INPUT_ONLY" + ] }, "environment_vars": { "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", @@ -46961,7 +48122,10 @@ "fields": { "burst_scaling_enabled": { "description": "Whether burst scaling is enabled. When enabled (default), the endpoint can automatically\nscale up beyond provisioned capacity to handle traffic spikes. When disabled, the endpoint\nmaintains fixed capacity at provisioned_model_units.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "PUBLIC_PREVIEW", + "behaviors": [ + "INPUT_ONLY" + ] }, "creation_timestamp": { "launch_stage": "GA" @@ -47358,6 +48522,15 @@ "description": "Configuration for inference table payload logging, including sampling.", "ref": "serving.TelemetryInferenceTableConfig", "launch_stage": "PUBLIC_PREVIEW" + }, + "table_names": { + "description": "The Unity Catalog tables to which endpoint telemetry (logs, traces, and metrics) is exported.\nProvide this to create a new telemetry profile for the endpoint from the given tables.", + "ref": "serving.UnityCatalogTableNames", + "launch_stage": "PUBLIC_PREVIEW" + }, + "telemetry_profile_id": { + "description": "The ID of an existing telemetry profile to apply to this endpoint. Provide this to reuse a\ntelemetry profile that has already been created, instead of specifying table_names.", + "launch_stage": "PUBLIC_PREVIEW" } } }, @@ -47385,6 +48558,26 @@ } } }, + "serving.UnityCatalogTableNames": { + "fields": { + "annotations_table": { + "description": "The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported annotations.", + "launch_stage": "PUBLIC_PREVIEW" + }, + "logs_table": { + "description": "The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported logs.", + "launch_stage": "PUBLIC_PREVIEW" + }, + "metrics_table": { + "description": "The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported metrics.", + "launch_stage": "PUBLIC_PREVIEW" + }, + "traces_table": { + "description": "The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported traces (spans).", + "launch_stage": "PUBLIC_PREVIEW" + } + } + }, "serving.UpdateInferenceEndpointNotifications": { "fields": { "email_notifications": { @@ -47474,7 +48667,7 @@ "ingress": { "description": "The network policies applying for ingress traffic.", "ref": "settings.CustomerFacingIngressNetworkPolicy", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -47482,7 +48675,7 @@ "ingress_dry_run": { "description": "The ingress policy for dry run mode. Dry run will always run even if the request\nis allowed by the ingress policy. When this field is set, the policy will be evaluated\nand emit logs only without blocking requests.", "ref": "settings.CustomerFacingIngressNetworkPolicy", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -47787,7 +48980,8 @@ "HITRUST", "K_FSI", "GERMANY_C5", - "GERMANY_TISAX" + "GERMANY_TISAX", + "KSA_ECC_CCC_DCC" ], "enum_launch_stages": { "CANADA_PROTECTED_B": "GA", @@ -47802,6 +48996,7 @@ "IRAP_PROTECTED": "GA", "ISMAP": "GA", "ITAR_EAR": "GA", + "KSA_ECC_CCC_DCC": "GA", "K_FSI": "GA", "NONE": "GA", "PCI_DSS": "GA" @@ -48059,7 +49254,7 @@ "public_access": { "description": "The network policy restrictions for public access to the workspace.\nConfigures how public internet traffic is allowed or denied access.", "ref": "settings.CustomerFacingIngressNetworkPolicyPublicAccess", - "launch_stage": "PUBLIC_BETA", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -48104,15 +49299,15 @@ "API_SCOPE_QUALIFIER_ALL" ], "enum_launch_stages": { - "API_SCOPE_QUALIFIER_ALL": "PUBLIC_BETA", - "API_SCOPE_QUALIFIER_READ": "PUBLIC_BETA" + "API_SCOPE_QUALIFIER_ALL": "GA", + "API_SCOPE_QUALIFIER_READ": "GA" } }, "settings.CustomerFacingIngressNetworkPolicyAppsRuntimeDestination": { "fields": { "all_destinations": { "description": "Must be set to true.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48120,22 +49315,22 @@ "fields": { "identities": { "description": "Valid only when IdentityType is IDENTITY_TYPE_SELECTED_IDENTITIES.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "identity_type": { "ref": "settings.CustomerFacingIngressNetworkPolicyAuthenticationIdentityType", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, "settings.CustomerFacingIngressNetworkPolicyAuthenticationIdentity": { "fields": { "principal_id": { - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "principal_type": { "ref": "settings.CustomerFacingIngressNetworkPolicyAuthenticationIdentityPrincipalType", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48145,8 +49340,8 @@ "PRINCIPAL_TYPE_SERVICE_PRINCIPAL" ], "enum_launch_stages": { - "PRINCIPAL_TYPE_SERVICE_PRINCIPAL": "PUBLIC_BETA", - "PRINCIPAL_TYPE_USER": "PUBLIC_BETA" + "PRINCIPAL_TYPE_SERVICE_PRINCIPAL": "GA", + "PRINCIPAL_TYPE_USER": "GA" } }, "settings.CustomerFacingIngressNetworkPolicyAuthenticationIdentityType": { @@ -48156,9 +49351,9 @@ "IDENTITY_TYPE_SELECTED_IDENTITIES" ], "enum_launch_stages": { - "IDENTITY_TYPE_ALL_SERVICE_PRINCIPALS": "PUBLIC_BETA", - "IDENTITY_TYPE_ALL_USERS": "PUBLIC_BETA", - "IDENTITY_TYPE_SELECTED_IDENTITIES": "PUBLIC_BETA" + "IDENTITY_TYPE_ALL_SERVICE_PRINCIPALS": "GA", + "IDENTITY_TYPE_ALL_USERS": "GA", + "IDENTITY_TYPE_SELECTED_IDENTITIES": "GA" } }, "settings.CustomerFacingIngressNetworkPolicyCrossWorkspaceAccess": { @@ -48237,7 +49432,7 @@ "fields": { "ip_ranges": { "description": "We only support IPv4 and IPv4 CIDR notation for now.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48245,7 +49440,7 @@ "fields": { "all_destinations": { "description": "Must be set to true.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48333,20 +49528,20 @@ "settings.CustomerFacingIngressNetworkPolicyPublicAccess": { "fields": { "allow_rules": { - "launch_stage": "PUBLIC_BETA", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "deny_rules": { - "launch_stage": "PUBLIC_BETA", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "restriction_mode": { "ref": "settings.CustomerFacingIngressNetworkPolicyPublicAccessRestrictionMode", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48356,8 +49551,8 @@ "RESTRICTED_ACCESS" ], "enum_launch_stages": { - "FULL_ACCESS": "PUBLIC_BETA", - "RESTRICTED_ACCESS": "PUBLIC_BETA" + "FULL_ACCESS": "GA", + "RESTRICTED_ACCESS": "GA" } }, "settings.CustomerFacingIngressNetworkPolicyPublicIngressRule": { @@ -48365,19 +49560,19 @@ "fields": { "authentication": { "ref": "settings.CustomerFacingIngressNetworkPolicyAuthentication", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "destination": { "ref": "settings.CustomerFacingIngressNetworkPolicyRequestDestination", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "label": { "description": "The label for this ingress rule.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "origin": { "ref": "settings.CustomerFacingIngressNetworkPolicyPublicRequestOrigin", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48385,17 +49580,17 @@ "fields": { "all_ip_ranges": { "description": "Matches all IPv4 and IPv6 ranges (both public and private).", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "excluded_ip_ranges": { "description": "Excluded means: all public IP ranges except this one.", "ref": "settings.CustomerFacingIngressNetworkPolicyIpRanges", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "included_ip_ranges": { "description": "Will not allow IP ranges with private IPs.", "ref": "settings.CustomerFacingIngressNetworkPolicyIpRanges", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48418,23 +49613,23 @@ }, "all_destinations": { "description": "When true, match all destinations, no other destination fields can be set.\nWhen not set or false, at least one specific destination must be provided.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "apps_runtime": { "ref": "settings.CustomerFacingIngressNetworkPolicyAppsRuntimeDestination", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "lakebase_runtime": { "ref": "settings.CustomerFacingIngressNetworkPolicyLakebaseRuntimeDestination", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "workspace_api": { "ref": "settings.CustomerFacingIngressNetworkPolicyWorkspaceApiDestination", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "workspace_ui": { "ref": "settings.CustomerFacingIngressNetworkPolicyWorkspaceUiDestination", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48444,10 +49639,10 @@ "scope_qualifier": { "description": "Qualifies the breadth of API access for the listed scopes. See ApiScopeQualifier.", "ref": "settings.CustomerFacingIngressNetworkPolicyApiScopeQualifier", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" }, "scopes": { - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -48462,7 +49657,7 @@ "fields": { "all_destinations": { "description": "Must be set to true.", - "launch_stage": "PUBLIC_BETA" + "launch_stage": "GA" } } }, @@ -53466,14 +54661,14 @@ "STDDEV" ], "enum_launch_stages": { - "AVG": "PUBLIC_PREVIEW", - "COUNT": "PUBLIC_PREVIEW", - "COUNT_DISTINCT": "PUBLIC_PREVIEW", - "MAX": "PUBLIC_PREVIEW", - "MEDIAN": "PUBLIC_PREVIEW", - "MIN": "PUBLIC_PREVIEW", - "STDDEV": "PUBLIC_PREVIEW", - "SUM": "PUBLIC_PREVIEW" + "AVG": "GA", + "COUNT": "GA", + "COUNT_DISTINCT": "GA", + "MAX": "GA", + "MEDIAN": "GA", + "MIN": "GA", + "STDDEV": "GA", + "SUM": "GA" } }, "sql.Alert": { @@ -53592,10 +54787,10 @@ "ERROR" ], "enum_launch_stages": { - "ERROR": "PUBLIC_PREVIEW", - "OK": "PUBLIC_PREVIEW", - "TRIGGERED": "PUBLIC_PREVIEW", - "UNKNOWN": "PUBLIC_PREVIEW" + "ERROR": "GA", + "OK": "GA", + "TRIGGERED": "GA", + "UNKNOWN": "GA" } }, "sql.AlertLifecycleState": { @@ -53604,8 +54799,8 @@ "DELETED" ], "enum_launch_stages": { - "ACTIVE": "PUBLIC_PREVIEW", - "DELETED": "PUBLIC_PREVIEW" + "ACTIVE": "GA", + "DELETED": "GA" } }, "sql.AlertOperandColumn": { @@ -53762,48 +54957,71 @@ "UNKNOWN": "GA" } }, + "sql.AlertStatementParameter": { + "description": "Redash-owned copy of the internal StatementParameter for the external AlertV2 API.\nThe internal `ordinal` and `args` fields are intentionally omitted: the public API\nsupports only flat, named scalar parameters; complex types (ARRAY, MAP, STRUCT) are\nnot supported. This mirrors SEA's public StatementParameter schema, see:\ncmdexec/sql-exec-api/proto/sql_exec_api_service.proto:763-779", + "fields": { + "name": { + "description": "The name of the parameter, referenced in the query as `:name`.", + "launch_stage": "PRIVATE_PREVIEW" + }, + "type": { + "description": "The SQL data type of the parameter, e.g. STRING, INT, or DATE. Defaults to STRING. This is a\nstring rather than an enum because scalar subtypes such as DECIMAL(10, 4) cannot be enumerated.\nComplex types such as ARRAY, MAP, and STRUCT are not supported.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + }, + "value": { + "description": "The bound value for the parameter, given as a string. If omitted, the value is interpreted as NULL.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL" + ] + } + } + }, "sql.AlertV2": { "fields": { "create_time": { "description": "The timestamp indicating when the alert was created.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "custom_description": { "description": "Custom description for the alert. support mustache template.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "custom_summary": { "description": "Custom summary for the alert. support mustache template.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "display_name": { "description": "The display name of the alert.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "effective_run_as": { "description": "The actual identity that will be used to execute the alert.\nThis is an output-only field that shows the resolved run-as identity after applying\npermissions and defaults.", "ref": "sql.AlertV2RunAs", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "evaluation": { "ref": "sql.AlertV2Evaluation", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "id": { "description": "UUID identifying the alert.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] @@ -53811,33 +55029,41 @@ "lifecycle_state": { "description": "Indicates whether the query is trashed.", "ref": "sql.AlertLifecycleState", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "owner_user_name": { "description": "The owner's username. This field is set to \"Unavailable\" if the user has been deleted.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, + "parameters": { + "description": "Query parameters bound when executing the alert query, referenced in the\nquery text with `:name` syntax. Static values only.", + "launch_stage": "PRIVATE_PREVIEW", + "behaviors": [ + "OPTIONAL", + "UNORDERED_LIST" + ] + }, "parent_path": { "description": "The workspace path of the folder containing the alert. Can only be set on create, and cannot be updated.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "query_text": { "description": "Text of the query to be run.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "run_as": { "description": "Specifies the identity that will be used to run the alert.\nThis field allows you to configure alerts to run as a specific user or service principal.\n- For user identity: Set `user_name` to the email of an active workspace user. Users can only set this to their own email.\n- For service principal: Set `service_principal_name` to the application ID. Requires the `servicePrincipal/user` role.\nIf not specified, the alert will run as the request user.", "ref": "sql.AlertV2RunAs", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -53845,25 +55071,25 @@ "run_as_user_name": { "description": "The run as username or application ID of service principal.\nOn Create and Update, this field can be set to application ID of an active service principal. Setting this field requires the servicePrincipal/user role.\nDeprecated: Use `run_as` field instead. This field will be removed in a future release.", "deprecated": true, - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "schedule": { "ref": "sql.CronSchedule", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "update_time": { "description": "The timestamp indicating when the alert was updated.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] }, "warehouse_id": { "description": "ID of the SQL warehouse attached to the alert.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -53872,19 +55098,19 @@ "comparison_operator": { "description": "Operator used for comparison in alert evaluation.", "ref": "sql.ComparisonOperator", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "empty_result_state": { "description": "Alert state if result is empty. Please avoid setting this field to be `UNKNOWN` because `UNKNOWN` state is planned to be deprecated.", "ref": "sql.AlertEvaluationState", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "last_evaluated_at": { "description": "Timestamp of the last evaluation.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] @@ -53892,7 +55118,7 @@ "notification": { "description": "User or Notification Destination to notify when alert is triggered.", "ref": "sql.AlertV2Notification", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -53900,12 +55126,12 @@ "source": { "description": "Source column from result to use to evaluate alert", "ref": "sql.AlertV2OperandColumn", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "state": { "description": "Latest state of alert evaluation.", "ref": "sql.AlertEvaluationState", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OUTPUT_ONLY" ] @@ -53913,7 +55139,7 @@ "threshold": { "description": "Threshold to user for alert evaluation, can be a column or a value.", "ref": "sql.AlertV2Operand", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -53924,20 +55150,20 @@ "fields": { "notify_on_ok": { "description": "Whether to notify alert subscribers when alert returns back to normal.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "retrigger_seconds": { "description": "Number of seconds an alert waits after being triggered before it is allowed to send another notification.\nIf set to 0 or omitted, the alert will not send any further notifications after the first trigger\nSetting this value to 1 allows the alert to send a notification on every evaluation where the condition is met, effectively making it always retrigger for notification purposes.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "subscriptions": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL", "UNORDERED_LIST" @@ -53949,14 +55175,14 @@ "fields": { "column": { "ref": "sql.AlertV2OperandColumn", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "value": { "ref": "sql.AlertV2OperandValue", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -53968,38 +55194,38 @@ "aggregation": { "description": "If not set, the behavior is equivalent to using `First row` in the UI.", "ref": "sql.Aggregation", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "display": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "name": { - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, "sql.AlertV2OperandValue": { "fields": { "bool_value": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "double_value": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "string_value": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -54010,14 +55236,14 @@ "fields": { "service_principal_name": { "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "user_name": { "description": "The email of an active workspace user. Can only set this field to their own email.", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -54027,10 +55253,10 @@ "sql.AlertV2Subscription": { "fields": { "destination_id": { - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "user_email": { - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -54221,14 +55447,14 @@ "IS_NOT_NULL" ], "enum_launch_stages": { - "EQUAL": "PUBLIC_PREVIEW", - "GREATER_THAN": "PUBLIC_PREVIEW", - "GREATER_THAN_OR_EQUAL": "PUBLIC_PREVIEW", - "IS_NOT_NULL": "PUBLIC_PREVIEW", - "IS_NULL": "PUBLIC_PREVIEW", - "LESS_THAN": "PUBLIC_PREVIEW", - "LESS_THAN_OR_EQUAL": "PUBLIC_PREVIEW", - "NOT_EQUAL": "PUBLIC_PREVIEW" + "EQUAL": "GA", + "GREATER_THAN": "GA", + "GREATER_THAN_OR_EQUAL": "GA", + "IS_NOT_NULL": "GA", + "IS_NULL": "GA", + "LESS_THAN": "GA", + "LESS_THAN_OR_EQUAL": "GA", + "NOT_EQUAL": "GA" } }, "sql.CreateAlert": { @@ -54309,7 +55535,7 @@ "fields": { "alert": { "ref": "sql.AlertV2", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -54456,7 +55682,7 @@ "launch_stage": "GA" }, "cluster_size": { - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "launch_stage": "GA" }, "creator_name": { @@ -54554,18 +55780,18 @@ "pause_status": { "description": "Indicate whether this schedule is paused or not.", "ref": "sql.SchedulePauseStatus", - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "quartz_cron_schedule": { "description": "A cron expression using quartz syntax that specifies the schedule for this pipeline.\nShould use the quartz format described here: http://www.quartz-scheduler.org/documentation/quartz-2.1.7/tutorials/tutorial-lesson-06.html", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "timezone_id": { "description": "A Java timezone id. The schedule will be resolved using this timezone.\nThis will be combined with the quartz_cron_schedule to determine the schedule.\nSee https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -54911,7 +56137,7 @@ "launch_stage": "GA" }, "cluster_size": { - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "launch_stage": "GA" }, "creator_name": { @@ -55028,7 +56254,7 @@ "launch_stage": "GA" }, "cluster_size": { - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "launch_stage": "GA" }, "creator_name": { @@ -55377,7 +56603,7 @@ "launch_stage": "GA" }, "cluster_size": { - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "launch_stage": "GA" }, "creator_name": { @@ -55836,13 +57062,13 @@ "sql.ListAlertsV2Request": { "fields": { "page_size": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] }, "page_token": { - "launch_stage": "PUBLIC_PREVIEW", + "launch_stage": "GA", "behaviors": [ "OPTIONAL" ] @@ -55852,10 +57078,10 @@ "sql.ListAlertsV2Response": { "fields": { "alerts": { - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "next_page_token": { - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -57019,8 +58245,8 @@ "PAUSED" ], "enum_launch_stages": { - "PAUSED": "PUBLIC_PREVIEW", - "UNPAUSED": "PUBLIC_PREVIEW" + "PAUSED": "GA", + "UNPAUSED": "GA" } }, "sql.ServiceError": { @@ -57174,7 +58400,9 @@ "RELIABILITY_OPTIMIZED": "GA" } }, - "sql.StartRequest": {}, + "sql.StartRequest": { + "description": "Starts a SQL warehouse.\nThis API is idempotent." + }, "sql.StartWarehouseResponse": {}, "sql.State": { "description": "*\nState of a warehouse.", @@ -57279,7 +58507,9 @@ "HEALTHY": "GA" } }, - "sql.StopRequest": {}, + "sql.StopRequest": { + "description": "Stops a SQL warehouse.\nThis API is idempotent." + }, "sql.StopWarehouseResponse": {}, "sql.Success": { "fields": { @@ -57835,11 +59065,11 @@ "fields": { "alert": { "ref": "sql.AlertV2", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "update_mask": { "description": "The field mask must be a single string, with multiple fields separated by commas (no spaces). The field path is relative to the resource object, using a dot (`.`) to navigate sub-fields (e.g., `author.given_name`). Specification of elements in sequence or map fields is not allowed, as only the entire collection field can be specified. Field names must exactly match the resource field names.\n\nA field mask of `*` indicates full replacement. It’s recommended to always explicitly list the fields being updated and avoid using `*` wildcards, as it can lead to unintended results if the API changes in the future.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -58639,7 +59869,7 @@ ] }, "tool_type": { - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "launch_stage": "PUBLIC_BETA" }, "uc_connection": { @@ -58939,7 +60169,7 @@ }, "target_qps": { "description": "Target QPS for the endpoint. Mutually exclusive with num_replicas.\nThe actual replica count is calculated at index creation/sync time based on this value.\nBest-effort target; the system does not guarantee this QPS will be achieved.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "usage_policy_id": { "description": "The usage policy id to be applied once we've migrated to usage policies", @@ -59213,7 +60443,7 @@ "scaling_info": { "description": "Scaling information for the endpoint", "ref": "vectorsearch.EndpointScalingInfo", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" } } }, @@ -59221,7 +60451,7 @@ "fields": { "requested_target_qps": { "description": "The requested QPS target for the endpoint. Best-effort; the system does not\nguarantee this QPS will be achieved.", - "launch_stage": "PUBLIC_PREVIEW" + "launch_stage": "GA" }, "state": { "description": "The current state of the scaling change request.", @@ -60390,6 +61620,10 @@ "description": "Branch that the local version of the repo is checked out to.", "launch_stage": "GA" }, + "git_cli_enabled": { + "description": "Whether the Git CLI is enabled for this Git folder (repo). When true, Git commands\ncan be run directly against this Git folder using the Git CLI.", + "launch_stage": "PUBLIC_BETA" + }, "head_commit_id": { "description": "SHA-1 hash representing the commit ID of the current HEAD of the repo.", "launch_stage": "GA" @@ -62676,10 +63910,10 @@ "name": "aisearch" }, "docs_group": "aisearch", - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "list": { "request": { "pascal_name": "ListEndpointsRequest" @@ -62694,10 +63928,10 @@ "can_use_json": true, "is_crud_create": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "CreateEndpointRequest", "is_object": true, @@ -63111,10 +64345,10 @@ "can_use_json": true, "is_crud_create": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "CreateIndexRequest", "is_object": true, @@ -63524,10 +64758,10 @@ "summary": "Delete an AI Search endpoint.", "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*}", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "DeleteEndpointRequest", "is_object": true, @@ -63586,10 +64820,10 @@ "summary": "Delete an AI Search index.", "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*/indexes/*}", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "DeleteIndexRequest", "is_object": true, @@ -63649,10 +64883,10 @@ "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*}", "is_crud_read": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "GetEndpointRequest", "is_object": true, @@ -63752,10 +64986,10 @@ "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*/indexes/*}", "is_crud_read": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "GetIndexRequest", "is_object": true, @@ -63872,10 +65106,10 @@ "summary": "List AI Search endpoints.", "path": "/api/2.0/ai-search/{parent=workspaces/*}/endpoints", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "ListEndpointsRequest", "is_object": true, @@ -63963,10 +65197,10 @@ "summary": "List AI Search indexes.", "path": "/api/2.0/ai-search/{parent=workspaces/*/endpoints/*}/indexes", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "ListIndexesRequest", "is_object": true, @@ -64056,10 +65290,10 @@ "can_use_json": true, "must_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "QueryIndexRequest", "is_object": true, @@ -64265,10 +65499,10 @@ "can_use_json": true, "must_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "RemoveDataRequest", "is_object": true, @@ -64368,10 +65602,10 @@ "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*/indexes/*}:scan", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "ScanIndexRequest", "is_object": true, @@ -64447,10 +65681,10 @@ "summary": "Synchronize an AI Search index.", "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*/indexes/*}:sync", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "SyncIndexRequest", "is_object": true, @@ -64511,10 +65745,10 @@ "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*}", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "UpdateEndpointRequest", "is_object": true, @@ -64956,10 +66190,10 @@ "path": "/api/2.0/ai-search/{name=workspaces/*/endpoints/*/indexes/*}:upsertData", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_BETA", - "cli_launch_stage_label": "*Beta*", - "cli_launch_stage_banner": "This command is in Beta and may change without notice.", - "cli_launch_stage_display": "Beta", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "UpsertDataRequest", "is_object": true, @@ -66341,10 +67575,8 @@ "name": "sql" }, "docs_group": "sql", - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "list": { "named_id_map": { "pascal_name": "AlertV2DisplayNameToIdMap" @@ -66362,10 +67594,8 @@ "can_use_json": true, "is_crud_create": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "CreateAlertV2Request", "is_object": true, @@ -66790,6 +68020,18 @@ "is_string": true } }, + { + "name": "parameters", + "description": "Query parameters bound when executing the alert query, referenced in the\nquery text with `:name` syntax. Static values only.", + "is_request_body_field": true, + "entity": { + "pascal_name": "AlertStatementParameterList", + "array_value": { + "pascal_name": "AlertStatementParameter", + "is_object": true + } + } + }, { "name": "parent_path", "description": "The workspace path of the folder containing the alert. Can only be set on create, and cannot be updated.", @@ -66908,10 +68150,8 @@ "path": "/api/2.0/alerts/{id}", "is_crud_read": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "GetAlertV2Request", "is_object": true, @@ -67062,10 +68302,8 @@ "summary": "List alerts.", "path": "/api/2.0/alerts", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "ListAlertsV2Request", "is_object": true @@ -67108,10 +68346,8 @@ "summary": "Delete an alert (legacy TrashAlert).", "path": "/api/2.0/alerts/{id}", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "TrashAlertV2Request", "is_object": true, @@ -67177,10 +68413,8 @@ "path": "/api/2.0/alerts/{id}", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "UpdateAlertV2Request", "is_object": true, @@ -67664,6 +68898,18 @@ "is_string": true } }, + { + "name": "parameters", + "description": "Query parameters bound when executing the alert query, referenced in the\nquery text with `:name` syntax. Static values only.", + "is_request_body_field": true, + "entity": { + "pascal_name": "AlertStatementParameterList", + "array_value": { + "pascal_name": "AlertStatementParameter", + "is_object": true + } + } + }, { "name": "parent_path", "description": "The workspace path of the folder containing the alert. Can only be set on create, and cannot be updated.", @@ -74167,7 +75413,7 @@ }, { "name": "CreateDeployment", - "description": "Creates a new deployment in the workspace.\n\nThe caller must provide a `deployment_id` which becomes the final\ncomponent of the deployment's resource name. If a deployment with the\nsame ID already exists, the server returns `ALREADY_EXISTS`.", + "description": "Creates a new deployment in the workspace.", "summary": "Create a deployment.", "path": "/api/2.0/bundle/deployments", "can_use_json": true, @@ -74183,39 +75429,19 @@ "required_fields": [ { "name": "deployment", - "description": "The deployment to create.", + "description": "The deployment to create. The caller must set `initial_parent_path`. Other fields are ignored\non input and populated by the service.", "required": true, "is_request_body_field": true, "entity": { "pascal_name": "Deployment", "is_object": true } - }, - { - "name": "deployment_id", - "description": "The ID to use for the deployment, which will become the final\ncomponent of the deployment's resource name\n(i.e. `deployments/{deployment_id}`).", - "required": true, - "is_query": true, - "entity": { - "is_string": true - } - } - ], - "required_in_url_fields": [ - { - "name": "deployment_id", - "description": "The ID to use for the deployment, which will become the final\ncomponent of the deployment's resource name\n(i.e. `deployments/{deployment_id}`).", - "required": true, - "is_query": true, - "entity": { - "is_string": true - } } ], "required_request_body_fields": [ { "name": "deployment", - "description": "The deployment to create.", + "description": "The deployment to create. The caller must set `initial_parent_path`. Other fields are ignored\non input and populated by the service.", "required": true, "is_request_body_field": true, "entity": { @@ -74227,7 +75453,7 @@ }, "request_body_field": { "name": "deployment", - "description": "The deployment to create.", + "description": "The deployment to create. The caller must set `initial_parent_path`. Other fields are ignored\non input and populated by the service.", "required": true, "is_request_body_field": true, "entity": { @@ -74242,7 +75468,7 @@ "all_fields": [ { "name": "deployment", - "description": "The deployment to create.", + "description": "The deployment to create. The caller must set `initial_parent_path`. Other fields are ignored\non input and populated by the service.", "required": true, "is_request_body_field": true, "entity": { @@ -74250,15 +75476,6 @@ "is_object": true } }, - { - "name": "deployment_id", - "description": "The ID to use for the deployment, which will become the final\ncomponent of the deployment's resource name\n(i.e. `deployments/{deployment_id}`).", - "required": true, - "is_query": true, - "entity": { - "is_string": true - } - }, { "name": "create_time", "description": "When the deployment was created.", @@ -74297,7 +75514,7 @@ }, { "name": "destroy_time", - "description": "When the deployment was destroyed (i.e. `bundle destroy` completed).\nUnset if the deployment has not been destroyed.\nNamed destroy_time (not delete_time) because this tracks the\n`databricks bundle destroy` command, not the API-level deletion.", + "description": "When deletion was recorded. Unset if deletion has not been recorded.\nThis response metadata does not determine the deployment's lifecycle status.", "is_output_only": true, "is_request_body_field": true, "is_optional_object": true, @@ -74316,7 +75533,7 @@ }, { "name": "display_name", - "description": "Human-readable name for the deployment. Output only: it is denormalized from\nthe latest version, not set directly on the deployment.", + "description": "Human-readable name for the deployment, up to 256 characters. Output only: clients update it\nby setting `display_name` when creating a version.", "is_output_only": true, "is_request_body_field": true, "entity": { @@ -74334,6 +75551,14 @@ "is_object": true } }, + { + "name": "initial_parent_path", + "description": "The workspace path of the existing folder where the deployment is initially created. Must be\nabsolute and canonical, with single separators, no `.` or `..` segments, and no trailing slash\nunless the path is `/`. It may contain at most 24 path segments, excluding an optional leading\n`/Workspace` segment. The complete path may contain up to 1,024 characters, and each segment\nmay contain up to 511 characters. This field is input only and is not returned in create, get,\nor list responses.", + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, { "name": "last_version_id", "description": "The version_id of the most recent deployment version.", @@ -74405,17 +75630,6 @@ "is_object": true } } - ], - "required_positional_arguments": [ - { - "name": "deployment_id", - "description": "The ID to use for the deployment, which will become the final\ncomponent of the deployment's resource name\n(i.e. `deployments/{deployment_id}`).", - "required": true, - "is_query": true, - "entity": { - "is_string": true - } - } ] }, { @@ -75464,7 +76678,7 @@ }, { "name": "display_name", - "description": "Display name for the deployment, captured at the time of this version.", + "description": "Display name for the deployment, captured at the time of this version. Up to 256 characters.\nWhen present, creating the version updates the deployment display name. An empty value clears\nit; an absent value leaves the current deployment display name unchanged.", "is_request_body_field": true, "entity": { "is_string": true @@ -75600,7 +76814,7 @@ }, { "name": "DeleteDeployment", - "description": "Deletes a deployment.\n\nThe deployment is marked as deleted. It and all its children (versions\nand their operations) will be permanently deleted after the retention\npolicy expires. If the deployment has an in-progress version, the\nserver returns `RESOURCE_CONFLICT`.", + "description": "Deletes a deployment.", "summary": "Delete a deployment.", "path": "/api/2.0/bundle/{name=deployments/*}", "is_hidden_cli": true, @@ -76330,7 +77544,7 @@ "all_fields": [ { "name": "page_size", - "description": "The maximum number of deployments to return. The service may return\nfewer than this value.\nIf unspecified, at most 50 deployments will be returned.\nThe maximum value is 1000; values above 1000 will be coerced to 1000.", + "description": "The maximum number of deployments to return. The service may return\nfewer than this value.\nIf unspecified, at most 20 deployments will be returned.\nThe maximum value is 100; values above 100 will be coerced to 100.", "is_query": true, "entity": { "is_int": true @@ -76580,7 +77794,7 @@ "all_fields": [ { "name": "page_size", - "description": "The maximum number of versions to return. The service may return\nfewer than this value.\nIf unspecified, at most 50 versions will be returned.\nThe maximum value is 1000; values above 1000 will be coerced to 1000.", + "description": "The maximum number of versions to return. The service may return\nfewer than this value.\nIf unspecified, at most 20 versions will be returned.\nThe maximum value is 100; values above 100 will be coerced to 100.", "is_query": true, "entity": { "is_int": true @@ -77213,6 +78427,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77268,6 +78485,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77328,6 +78548,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77365,6 +78588,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77394,6 +78620,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77458,6 +78687,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77527,6 +78759,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77573,6 +78808,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77615,6 +78853,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77686,6 +78927,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77828,6 +79072,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77865,6 +79112,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77908,6 +79158,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77945,6 +79198,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -77992,6 +79248,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78029,6 +79288,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78076,6 +79338,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78119,6 +79384,16 @@ "is_object": true } }, + { + "name": "jar_analysis", + "description": "Jar analysis details available to all collaborators of the clean room.\nPresent if and only if **asset_type** is **JAR_ANALYSIS**", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "CleanRoomAssetJarAnalysis", + "is_object": true + } + }, { "name": "name", "description": "A fully qualified name that uniquely identifies the asset within the clean room.\nThis is also the name displayed in the clean room UI.\n\nFor UC securable assets (tables, volumes, etc.), the format is *shared_catalog*.*shared_schema*.*asset_name*\n\nFor notebooks, the name is the notebook file name.\nFor jar analyses, the name is the jar analysis name.", @@ -78248,6 +79523,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78300,6 +79578,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78346,6 +79627,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78388,6 +79672,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78412,6 +79699,15 @@ "is_string": true } }, + { + "name": "jar_analysis_review", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "JarAnalysisVersionReview", + "is_object": true + } + }, { "name": "name", "description": "Name of the asset", @@ -78452,6 +79748,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78510,6 +79809,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78556,6 +79858,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78599,6 +79904,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78654,6 +79962,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78713,6 +80024,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78759,6 +80073,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78810,6 +80127,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78847,6 +80167,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78876,6 +80199,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -78931,6 +80257,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79072,6 +80401,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79128,6 +80460,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79196,6 +80531,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79233,6 +80571,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79276,6 +80617,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79313,6 +80657,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79361,6 +80708,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79398,6 +80748,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79427,6 +80780,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79480,6 +80836,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -79523,6 +80882,16 @@ "is_object": true } }, + { + "name": "jar_analysis", + "description": "Jar analysis details available to all collaborators of the clean room.\nPresent if and only if **asset_type** is **JAR_ANALYSIS**", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "CleanRoomAssetJarAnalysis", + "is_object": true + } + }, { "name": "name", "description": "A fully qualified name that uniquely identifies the asset within the clean room.\nThis is also the name displayed in the clean room UI.\n\nFor UC securable assets (tables, volumes, etc.), the format is *shared_catalog*.*shared_schema*.*asset_name*\n\nFor notebooks, the name is the notebook file name.\nFor jar analyses, the name is the jar analysis name.", @@ -79643,6 +81012,9 @@ { "content": "FOREIGN_TABLE" }, + { + "content": "JAR_ANALYSIS" + }, { "content": "NOTEBOOK_FILE" }, @@ -80270,7 +81642,7 @@ { "id": "cleanrooms.CleanRoomTaskRuns", "name": "CleanRoomTaskRuns", - "description": "Clean room task runs are the executions of notebooks in a clean room.", + "description": "Clean room task runs are the executions of notebooks and JAR analyses in a clean room.", "package": { "name": "cleanrooms" }, @@ -80379,6 +81751,119 @@ } } } + }, + { + "name": "ListCleanRoomTaskRunsHandler", + "description": "List all the historical task runs in a clean room.", + "summary": "List task runs.", + "path": "/api/2.0/clean-rooms/{clean_room_name}/task-runs", + "has_required_positional_arguments": true, + "launch_stage": "GA", + "cli_launch_stage_display": "GA", + "request": { + "pascal_name": "ListCleanRoomTaskRunsRequest", + "is_object": true, + "required_fields": [ + { + "name": "clean_room_name", + "description": "Name of the clean room.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "clean_room_name", + "description": "Name of the clean room.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "ListCleanRoomTaskRunsResponse", + "is_object": true + }, + "all_fields": [ + { + "name": "clean_room_name", + "description": "Name of the clean room.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + }, + { + "name": "name", + "description": "Executable name.", + "is_query": true, + "entity": { + "is_string": true + } + }, + { + "name": "page_size", + "description": "The maximum number of task runs to return. Maximum value of 100.", + "is_query": true, + "entity": { + "is_int": true + } + }, + { + "name": "page_token", + "description": "Opaque pagination token to go to next page based on previous query.", + "is_query": true, + "entity": { + "is_string": true + } + }, + { + "name": "task_type", + "description": "Filter by the type of Clean Room task.", + "is_query": true, + "entity": { + "pascal_name": "CleanRoomTaskType", + "enum": [ + { + "content": "JAR" + }, + { + "content": "NOTEBOOK" + } + ] + } + } + ], + "required_positional_arguments": [ + { + "name": "clean_room_name", + "description": "Name of the clean room.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "pagination": { + "token": { + "poll_field": { + "name": "page_token", + "description": "Opaque pagination token to go to next page based on previous query.", + "is_query": true, + "entity": { + "is_string": true + } + } + } + } } ] }, @@ -80491,6 +81976,14 @@ "is_int64": true } }, + { + "name": "enable_shared_output", + "description": "Whether allow task to write to shared output schema.\nWhen enabled, clean room task runs triggered by the current collaborator\ncan write to the run-scoped shared output schema which is accessible by all collaborators.", + "is_request_body_field": true, + "entity": { + "is_bool": true + } + }, { "name": "local_collaborator_alias", "description": "The alias of the collaborator tied to the local clean room.", @@ -81931,6 +83424,25 @@ ] } }, + { + "name": "dependency_mode", + "description": "Controls dependency configuration for the cluster.", + "is_request_body_field": true, + "entity": { + "pascal_name": "DependencyMode", + "enum": [ + { + "content": "DEPENDENCY_MODE_AUTO" + }, + { + "content": "DEPENDENCY_MODE_CLUSTER_LIBRARIES" + }, + { + "content": "DEPENDENCY_MODE_ENVIRONMENTS" + } + ] + } + }, { "name": "docker_image", "description": "Custom docker image BYOC", @@ -82478,6 +83990,25 @@ ] } }, + { + "name": "dependency_mode", + "description": "Controls dependency configuration for the cluster.", + "is_request_body_field": true, + "entity": { + "pascal_name": "DependencyMode", + "enum": [ + { + "content": "DEPENDENCY_MODE_AUTO" + }, + { + "content": "DEPENDENCY_MODE_CLUSTER_LIBRARIES" + }, + { + "content": "DEPENDENCY_MODE_ENVIRONMENTS" + } + ] + } + }, { "name": "docker_image", "description": "Custom docker image BYOC", @@ -84906,6 +86437,9 @@ { "content": "HUBSPOT" }, + { + "content": "JDBC" + }, { "content": "META_MARKETING" }, @@ -85023,6 +86557,9 @@ { "content": "HUBSPOT" }, + { + "content": "JDBC" + }, { "content": "META_MARKETING" }, @@ -85144,6 +86681,9 @@ { "content": "HUBSPOT" }, + { + "content": "JDBC" + }, { "content": "META_MARKETING" }, @@ -85231,6 +86771,14 @@ } } }, + { + "name": "parent", + "description": "Parent schema for schema-level connections, in format \"schemas/{catalog}.{schema}\".\nAbsent for metastore-level (L1) connections.", + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, { "name": "properties", "description": "A map of key-value properties attached to the securable.", @@ -85407,6 +86955,14 @@ "entity": { "is_string": true } + }, + { + "name": "parent", + "description": "Optional. Parent schema filter for listing schema-level connections, in format \"schemas/{catalog}.{schema}\".", + "is_query": true, + "entity": { + "is_string": true + } } ], "pagination": { @@ -99140,10 +100696,8 @@ }, "is_accounts": true, "docs_group": "disasterrecovery", - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "list": { "request": { "pascal_name": "ListFailoverGroupsRequest" @@ -99158,10 +100712,8 @@ "can_use_json": true, "is_crud_create": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "CreateFailoverGroupRequest", "is_object": true, @@ -99698,10 +101250,8 @@ "can_use_json": true, "is_crud_create": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "CreateStableUrlRequest", "is_object": true, @@ -99889,6 +101439,15 @@ "is_bool": true } }, + { + "name": "effective_workspace_id", + "description": "The workspace this stable URL currently routes to. Set to\n`initial_workspace_id` at creation, advanced to the failover group's primary\nwhile attached (including across a failover), and preserved when the stable\nURL is detached from its failover group. Read this to see where an unattached\nstable URL points: after a failover followed by a detach it reflects the\npost-failover primary, not `initial_workspace_id`.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, { "name": "failover_group_name", "description": "Fully qualified resource name of the FailoverGroup this stable URL is\ncurrently linked to, in the format\n`accounts/{account_id}/failover-groups/{failover_group_id}`. Empty when\nthe stable URL is not attached to any failover group.", @@ -99915,6 +101474,15 @@ "is_string": true } }, + { + "name": "stable_workspace_id", + "description": "The stable workspace ID for this stable URL. Generated on creation and\nimmutable thereafter; identifies the URL across failovers and is the same\nvalue embedded in the `url` (as the `w=` query parameter for SPOG URLs,\nor in the `conn-\u003cid\u003e` hostname for Private-Link URLs).", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, { "name": "url", "description": "The stable URL endpoint. Generated on creation and\nimmutable thereafter. For non-Private-Link workspaces this is\n`https://\u003cspog_host\u003e/?w=\u003cconnection_id\u003e`. For Private-Link workspaces\nthis is the per-connection hostname.", @@ -99961,10 +101529,8 @@ "summary": "Delete a Failover Group.", "path": "/api/disaster-recovery/v1/{name=accounts/*/failover-groups/*}", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "DeleteFailoverGroupRequest", "is_object": true, @@ -100031,10 +101597,8 @@ "summary": "Delete a Stable URL.", "path": "/api/disaster-recovery/v1/{name=accounts/*/stable-urls/*}", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "DeleteStableUrlRequest", "is_object": true, @@ -100094,10 +101658,8 @@ "path": "/api/disaster-recovery/v1/{name=accounts/*/failover-groups/*}/failover", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "FailoverFailoverGroupRequest", "is_object": true, @@ -100334,10 +101896,8 @@ "path": "/api/disaster-recovery/v1/{name=accounts/*/failover-groups/*}", "is_crud_read": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "GetFailoverGroupRequest", "is_object": true, @@ -100471,10 +102031,8 @@ "path": "/api/disaster-recovery/v1/{name=accounts/*/stable-urls/*}", "is_crud_read": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "GetStableUrlRequest", "is_object": true, @@ -100557,10 +102115,8 @@ "summary": "List Failover Groups.", "path": "/api/disaster-recovery/v1/{parent=accounts/*}/failover-groups", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "ListFailoverGroupsRequest", "is_object": true, @@ -100648,10 +102204,8 @@ "summary": "List Stable URLs.", "path": "/api/disaster-recovery/v1/{parent=accounts/*}/stable-urls", "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "ListStableUrlsRequest", "is_object": true, @@ -100740,10 +102294,8 @@ "path": "/api/disaster-recovery/v1/{name=accounts/*/failover-groups/*}", "can_use_json": true, "has_required_positional_arguments": true, - "launch_stage": "PUBLIC_PREVIEW", - "cli_launch_stage_label": "*Public Preview*", - "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", - "cli_launch_stage_display": "Public Preview", + "launch_stage": "GA", + "cli_launch_stage_display": "GA", "request": { "pascal_name": "UpdateFailoverGroupRequest", "is_object": true, @@ -102469,6 +104021,16 @@ "is_string": true } }, + { + "name": "aws_vpc_endpoint_info", + "description": "Info for an AWS VPC endpoint.", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "AwsVpcEndpointInfo", + "is_object": true + } + }, { "name": "azure_private_endpoint_info", "description": "Info for an Azure private endpoint.", @@ -102507,6 +104069,16 @@ "is_string": true } }, + { + "name": "gcp_psc_endpoint_info", + "description": "Info for a GCP Private Service Connect endpoint.", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "GcpPscEndpointInfo", + "is_object": true + } + }, { "name": "name", "description": "The resource name of the endpoint, which uniquely identifies the endpoint.", @@ -105743,6 +107315,16 @@ "is_object": true } } + }, + { + "name": "trace_location", + "description": "The location where the experiment's traces are stored. When set, the\nunderlying storage is provisioned and the experiment's traces are routed\nto it. When unset, traces are stored in the default MLflow backend. This\nfield cannot be updated after the experiment is created.", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "ExperimentTraceLocation", + "is_object": true + } } ], "required_positional_arguments": [ @@ -113990,7 +115572,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114041,7 +115623,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114098,7 +115680,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114149,7 +115731,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114211,7 +115793,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114262,7 +115844,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114350,7 +115932,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -114420,7 +116002,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -115112,7 +116694,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -115163,7 +116745,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -116981,7 +118563,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117032,7 +118614,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117089,7 +118671,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117140,7 +118722,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117211,7 +118793,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117262,7 +118844,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117359,7 +118941,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -117438,7 +119020,7 @@ }, { "name": "schema_config", - "description": "Schema definitions for the stream. Currently only direct schemas are supported.\nIn a future milestone, we will support schema registries through a UC Connection.", + "description": "Schema definitions for the stream, provided either directly on the Stream or\nresolved from an external schema registry through a UC Connection.", "required": true, "is_request_body_field": true, "entity": { @@ -121338,7 +122920,7 @@ }, { "name": "DownloadMessageAttachmentVisualization", - "description": "Download a rendered image of a message visualization attachment.\nThe response body is the raw PNG image, not a JSON payload.\nThis is only available if the attachment is a visualization and the message status is `COMPLETED`.", + "description": "Download a rendered image of a message visualization attachment.\nThe response body is the raw PNG image, not a JSON payload.\nThis is only available if the attachment is a visualization and the message status is `COMPLETED`.\nThis endpoint is not supported for Private Link workspaces.", "summary": "Download message attachment visualization.", "path": "/api/2.0/genie/{name=spaces/*/conversations/*/messages/*/attachments/*}/download-visualization", "is_crud_read": true, @@ -125843,10 +127425,11 @@ "description": "Lists the privilege assignments for a securable. Does not include inherited privileges.\nPaginated version of Get Permissions API.", "summary": "List permissions.", "path": "/api/2.1/unity-catalog/privilege-assignments/{securable_type}/{full_name}", - "is_hidden_cli": true, "has_required_positional_arguments": true, - "launch_stage": "PRIVATE_PREVIEW", - "cli_launch_stage_display": "Private Preview", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "ListPrivilegeAssignmentsRequest", "is_object": true, @@ -125977,10 +127560,11 @@ "description": "Lists the effective privilege assignments for a securable. Includes inherited privileges.\nPaginated version of Get Effective Permissions API.", "summary": "List effective permissions.", "path": "/api/2.1/unity-catalog/effective-privilege-assignments/{securable_type}/{full_name}", - "is_hidden_cli": true, "has_required_positional_arguments": true, - "launch_stage": "PRIVATE_PREVIEW", - "cli_launch_stage_display": "Private Preview", + "launch_stage": "PUBLIC_PREVIEW", + "cli_launch_stage_label": "*Public Preview*", + "cli_launch_stage_banner": "This command is in Public Preview and may change without notice.", + "cli_launch_stage_display": "Public Preview", "request": { "pascal_name": "ListEffectivePrivilegeAssignmentsRequest", "is_object": true, @@ -127367,7 +128951,7 @@ "name": "CreateWorkspaceAssignmentDetail", "description": "Creates a workspace assignment detail for a principal. Entitlement grants are applied\nindividually and non-atomically — if a failure occurs partway through, the principal will be\nassigned to the workspace but with only a subset of the requested entitlements. Use\nGetWorkspaceAssignmentDetail to confirm which entitlements were successfully granted.", "summary": "Create a workspace assignment detail.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAssignmentDetails", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-assignment-details", "can_use_json": true, "is_hidden_cli": true, "has_required_positional_arguments": true, @@ -127534,6 +129118,38 @@ "is_string": true } }, + { + "name": "effective_entitlements", + "description": "The principal's full effective entitlements granted in this workspace: every entitlement it holds\nwhether granted directly or via group membership. Populated on Get; empty on List.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "EntitlementList", + "array_value": { + "pascal_name": "Entitlement", + "enum": [ + { + "content": "ALLOW_CLUSTER_CREATE" + }, + { + "content": "ALLOW_INSTANCE_POOL_CREATE" + }, + { + "content": "DATABRICKS_SQL_ACCESS" + }, + { + "content": "WORKSPACE_ACCESS" + }, + { + "content": "WORKSPACE_ADMIN" + }, + { + "content": "WORKSPACE_CONSUME" + } + ] + } + } + }, { "name": "entitlements", "description": "Entitlements granted directly to the principal on this workspace. The only client-settable\nfield: create and update manage exactly this set (including entitlements the principal also\nholds via a group).\nNot populated by ListWorkspaceAssignmentDetails (omitted for scalability); call\nGetWorkspaceAssignmentDetail to read the entitlements for a single principal.", @@ -127628,7 +129244,7 @@ "name": "DeleteWorkspaceAssignmentDetail", "description": "Deletes a workspace assignment detail for a principal, revoking all associated entitlements.\nEntitlement revocations are applied individually and non-atomically — if a failure occurs\npartway through, the principal remains assigned with a subset of its original entitlements,\nand the operation is safe to retry.", "summary": "Delete a workspace assignment detail.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-assignment-details/{principal_id}", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -127725,7 +129341,7 @@ "name": "GetWorkspaceAccessDetail", "description": "Returns the access details for a principal in a workspace. Allows for checking access details for any\nprovisioned principal (user, service principal, or group) in a workspace.\n* Provisioned principal here refers to one that has been synced into Databricks from the customer's IdP or\nadded explicitly to Databricks via SCIM/UI.\nAllows for passing in a \"view\" parameter to control what fields are returned (BASIC by default or FULL).", "summary": "Get workspace access details for a principal.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAccessDetails/{principal_id}", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-access-details/{principal_id}", "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", "cli_launch_stage_label": "*Beta*", @@ -127840,7 +129456,7 @@ "name": "GetWorkspaceAssignmentDetail", "description": "Returns the assignment details for a principal in a workspace.", "summary": "Get workspace assignment details for a principal.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-assignment-details/{principal_id}", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -127961,7 +129577,7 @@ "name": "ListWorkspaceAssignmentDetails", "description": "Lists workspace assignment details for a workspace.\nFor scalability, the response omits the per-principal entitlement fields (`entitlements` and\n`effective_entitlements`); call GetWorkspaceAssignmentDetail to read entitlements for a single\nprincipal.", "summary": "List workspace assignment details for a workspace.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAssignmentDetails", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-assignment-details", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -128039,7 +129655,7 @@ "name": "ResolveGroup", "description": "Resolves a group with the given external ID from the customer's IdP. If the group does not exist, it will be created in the account.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external group in the Databricks account.", - "path": "/api/2.0/identity/accounts/{account_id}/groups/resolveByExternalId", + "path": "/api/2.0/identity/accounts/{account_id}/groups/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -128104,7 +129720,7 @@ "name": "ResolveServicePrincipal", "description": "Resolves an SP with the given external ID from the customer's IdP. If the SP does not exist, it will be created.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external service principal in the Databricks account.", - "path": "/api/2.0/identity/accounts/{account_id}/servicePrincipals/resolveByExternalId", + "path": "/api/2.0/identity/accounts/{account_id}/service-principals/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -128169,7 +129785,7 @@ "name": "ResolveUser", "description": "Resolves a user with the given external ID from the customer's IdP. If the user does not exist, it will be created.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external user in the Databricks account.", - "path": "/api/2.0/identity/accounts/{account_id}/users/resolveByExternalId", + "path": "/api/2.0/identity/accounts/{account_id}/users/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -128234,7 +129850,7 @@ "name": "UpdateWorkspaceAssignmentDetail", "description": "Updates the entitlements of a directly assigned principal in a workspace. Entitlement changes\nare applied individually and non-atomically — if a failure occurs partway through, only a\nsubset of the requested changes may have been applied. Use GetWorkspaceAssignmentDetail to\nconfirm the final state.", "summary": "Update a workspace assignment detail.", - "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/accounts/{account_id}/workspaces/{workspace_id}/workspace-assignment-details/{principal_id}", "can_use_json": true, "is_hidden_cli": true, "has_required_positional_arguments": true, @@ -128456,6 +130072,38 @@ "is_string": true } }, + { + "name": "effective_entitlements", + "description": "The principal's full effective entitlements granted in this workspace: every entitlement it holds\nwhether granted directly or via group membership. Populated on Get; empty on List.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "EntitlementList", + "array_value": { + "pascal_name": "Entitlement", + "enum": [ + { + "content": "ALLOW_CLUSTER_CREATE" + }, + { + "content": "ALLOW_INSTANCE_POOL_CREATE" + }, + { + "content": "DATABRICKS_SQL_ACCESS" + }, + { + "content": "WORKSPACE_ACCESS" + }, + { + "content": "WORKSPACE_ADMIN" + }, + { + "content": "WORKSPACE_CONSUME" + } + ] + } + } + }, { "name": "entitlements", "description": "Entitlements granted directly to the principal on this workspace. The only client-settable\nfield: create and update manage exactly this set (including entitlements the principal also\nholds via a group).\nNot populated by ListWorkspaceAssignmentDetails (omitted for scalability); call\nGetWorkspaceAssignmentDetail to read the entitlements for a single principal.", @@ -132866,6 +134514,22 @@ "is_object": true } }, + { + "name": "performance_target", + "description": "The performance mode on a serverless one-time run. This field determines the level of compute performance or cost-efficiency for the run.\nThe performance target does not apply to tasks that run on Serverless GPU compute.\n\n* `STANDARD`: Enables cost-efficient execution of serverless workloads.\n* `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance.", + "is_request_body_field": true, + "entity": { + "pascal_name": "PerformanceTarget", + "enum": [ + { + "content": "PERFORMANCE_OPTIMIZED" + }, + { + "content": "STANDARD" + } + ] + } + }, { "name": "queue", "description": "The queue settings of the one-time run.", @@ -156528,6 +158192,331 @@ "response_type": {} } }, + { + "name": "CreateCdfConfig", + "description": "Create a CDF configuration that materializes the change data feed for all\ntables in a Postgres schema as open-format Delta tables in Unity Catalog.\nOnce created, each table's change history is continuously written to its\ncorresponding Lakehouse table.", + "summary": "Create a change data feed configuration.", + "path": "/api/2.0/postgres/{parent=projects/*/branches/*/databases/*}/cdf-configs", + "can_use_json": true, + "is_crud_create": true, + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "CreateCdfConfigRequest", + "is_object": true, + "has_required_request_body_fields": true, + "required_fields": [ + { + "name": "parent", + "description": "The parent database under which to create the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + }, + { + "name": "cdf_config", + "description": "The CdfConfig to create. The catalog, schema, and postgres_schema fields are\nrequired; all other fields are output only and ignored on input.", + "required": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "CdfConfig", + "is_object": true + } + } + ], + "required_in_url_fields": [ + { + "name": "parent", + "description": "The parent database under which to create the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_request_body_fields": [ + { + "name": "cdf_config", + "description": "The CdfConfig to create. The catalog, schema, and postgres_schema fields are\nrequired; all other fields are output only and ignored on input.", + "required": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "CdfConfig", + "is_object": true + } + } + ] + }, + "request_body_field": { + "name": "cdf_config", + "description": "The CdfConfig to create. The catalog, schema, and postgres_schema fields are\nrequired; all other fields are output only and ignored on input.", + "required": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "CdfConfig", + "is_object": true, + "has_required_request_body_fields": true, + "required_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ], + "required_request_body_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ] + } + }, + "response": { + "pascal_name": "Operation", + "is_object": true + }, + "all_fields": [ + { + "name": "cdf_config", + "description": "The CdfConfig to create. The catalog, schema, and postgres_schema fields are\nrequired; all other fields are output only and ignored on input.", + "required": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "CdfConfig", + "is_object": true, + "has_required_request_body_fields": true, + "required_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ], + "required_request_body_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ] + } + }, + { + "name": "cdf_config_id", + "description": "The user-specified id for the CdfConfig, forming the final segment of its\nresource name. Must match the pattern `[a-z][a-z0-9_]{0,62}`. Defaults to\nthe Postgres schema name when omitted.", + "is_query": true, + "entity": { + "is_string": true + } + }, + { + "name": "parent", + "description": "The parent database under which to create the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + }, + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "cdf_config_id", + "description": "The user-specified id; equals the final segment of `name`. Defaults to the\nPostgres schema name for configs without an explicit id.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "create_time", + "description": "When the CdfConfig was created.", + "is_output_only": true, + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "is_timestamp": true + } + }, + { + "name": "name", + "description": "Output only. The full resource name of the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "parent", + "description": "The parent database under which to create the CdfConfig.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + }, + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ], + "long_running_operation": { + "get_operation": { + "kebab_name": "get-operation", + "pascal_name": "GetOperation", + "request": { + "pascal_name": "GetOperationRequest" + } + }, + "response_type": {} + } + }, { "name": "CreateDataApi", "description": "Enable Data API for a database.", @@ -157894,6 +159883,89 @@ } } }, + { + "name": "DeleteCdfConfig", + "description": "Delete a CDF configuration and stop materializing the change data feed. When\nforce=true, also drops the Delta tables in Unity Catalog. When force=false\n(default), the existing tables are preserved at their last state.", + "summary": "Delete a change data feed configuration.", + "path": "/api/2.0/postgres/{name=projects/*/branches/*/databases/*/cdf-configs/*}", + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "DeleteCdfConfigRequest", + "is_object": true, + "required_fields": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to delete.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to delete.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "Operation", + "is_object": true + }, + "all_fields": [ + { + "name": "force", + "description": "When true, also drops the replicated Delta tables in Unity Catalog. When\nfalse (the default), the replicated tables are preserved at their last\nsynced state.", + "is_query": true, + "entity": { + "is_bool": true + } + }, + { + "name": "name", + "description": "The resource name of the CdfConfig to delete.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to delete.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "long_running_operation": { + "get_operation": { + "kebab_name": "get-operation", + "pascal_name": "GetOperation", + "request": { + "pascal_name": "GetOperationRequest" + } + }, + "response_type": { + "is_empty_response": true + } + } + }, { "name": "DeleteDataApi", "description": "Disable Data API for a database.", @@ -158583,6 +160655,193 @@ } ] }, + { + "name": "GetCdfConfig", + "description": "Get a single Lakebase CDF configuration, including the source Postgres\nschema, target Unity Catalog schema, and the identity under which writes are\nauthorized.", + "summary": "Get a change data feed configuration.", + "path": "/api/2.0/postgres/{name=projects/*/branches/*/databases/*/cdf-configs/*}", + "is_crud_read": true, + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "GetCdfConfigRequest", + "is_object": true, + "required_fields": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "CdfConfig", + "is_object": true, + "has_required_request_body_fields": true, + "required_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ], + "required_request_body_fields": [ + { + "name": "catalog", + "description": "The Unity Catalog catalog that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "schema", + "description": "The Unity Catalog schema that replicated tables are written into.\nSet at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + }, + { + "name": "postgres_schema", + "description": "The Postgres schema this CdfConfig replicates from. Unique within the\nparent database. Set at creation; the CdfConfig is immutable.", + "required": true, + "is_request_body_field": true, + "entity": { + "is_string": true + } + } + ] + }, + "all_fields": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "name", + "description": "The resource name of the CdfConfig to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + { + "name": "GetCdfStatus", + "description": "Get the CDF status of a single table within a Lakebase CDF configuration,\nincluding its current state and the last committed position in the feed.", + "summary": "Get a change data feed status.", + "path": "/api/2.0/postgres/{name=projects/*/branches/*/databases/*/cdf-configs/*/cdf-statuses/*}", + "is_crud_read": true, + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "GetCdfStatusRequest", + "is_object": true, + "required_fields": [ + { + "name": "name", + "description": "The resource name of the CdfStatus to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "name", + "description": "The resource name of the CdfStatus to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "CdfStatus", + "is_object": true + }, + "all_fields": [ + { + "name": "name", + "description": "The resource name of the CdfStatus to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "name", + "description": "The resource name of the CdfStatus to retrieve.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, { "name": "GetDataApi", "description": "Get Data API configuration for a database.", @@ -159129,6 +161388,188 @@ } } }, + { + "name": "ListCdfConfigs", + "description": "List all CDF configurations for a Lakebase database. Each configuration maps\na Postgres schema to a Unity Catalog schema where the change data feed is\nmaterialized.", + "summary": "List change data feed configurations.", + "path": "/api/2.0/postgres/{parent=projects/*/branches/*/databases/*}/cdf-configs", + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "ListCdfConfigsRequest", + "is_object": true, + "required_fields": [ + { + "name": "parent", + "description": "The parent database to list CdfConfigs for.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "parent", + "description": "The parent database to list CdfConfigs for.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "ListCdfConfigsResponse", + "is_object": true + }, + "all_fields": [ + { + "name": "page_size", + "description": "Maximum number of CdfConfigs to return.", + "is_query": true, + "entity": { + "is_int": true + } + }, + { + "name": "page_token", + "description": "Pagination token returned by a previous ListCdfConfigs call. Empty on the\nfirst page.", + "is_query": true, + "entity": { + "is_string": true + } + }, + { + "name": "parent", + "description": "The parent database to list CdfConfigs for.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "parent", + "description": "The parent database to list CdfConfigs for.\nFormat: projects/{project}/branches/{branch}/databases/{database}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "pagination": { + "token": { + "poll_field": { + "name": "page_token", + "description": "Pagination token returned by a previous ListCdfConfigs call. Empty on the\nfirst page.", + "is_query": true, + "entity": { + "is_string": true + } + } + } + } + }, + { + "name": "ListCdfStatuses", + "description": "List the per-table CDF statuses within a Lakebase CDF configuration. Each\nstatus shows whether a table's change data feed is snapshotting, streaming,\nor skipped.", + "summary": "List change data feed statuses.", + "path": "/api/2.0/postgres/{parent=projects/*/branches/*/databases/*/cdf-configs/*}/cdf-statuses", + "has_required_positional_arguments": true, + "launch_stage": "PUBLIC_BETA", + "cli_launch_stage_label": "*Beta*", + "cli_launch_stage_banner": "This command is in Beta and may change without notice.", + "cli_launch_stage_display": "Beta", + "request": { + "pascal_name": "ListCdfStatusesRequest", + "is_object": true, + "required_fields": [ + { + "name": "parent", + "description": "The parent CdfConfig to list CdfStatuses for.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "parent", + "description": "The parent CdfConfig to list CdfStatuses for.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "ListCdfStatusesResponse", + "is_object": true + }, + "all_fields": [ + { + "name": "page_size", + "description": "Maximum number of CdfStatuses to return.", + "is_query": true, + "entity": { + "is_int": true + } + }, + { + "name": "page_token", + "description": "Pagination token returned by a previous ListCdfStatuses call. Empty on the\nfirst page.", + "is_query": true, + "entity": { + "is_string": true + } + }, + { + "name": "parent", + "description": "The parent CdfConfig to list CdfStatuses for.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_positional_arguments": [ + { + "name": "parent", + "description": "The parent CdfConfig to list CdfStatuses for.\nFormat: projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "pagination": { + "token": { + "poll_field": { + "name": "page_token", + "description": "Pagination token returned by a previous ListCdfStatuses call. Empty on the\nfirst page.", + "is_query": true, + "entity": { + "is_string": true + } + } + } + } + }, { "name": "ListDatabases", "description": "List Databases.", @@ -170241,6 +172682,7 @@ { "name": "browse_only", "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", + "is_output_only": true, "is_request_body_field": true, "entity": { "is_bool": true @@ -170841,6 +173283,7 @@ { "name": "browse_only", "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", + "is_output_only": true, "is_request_body_field": true, "entity": { "is_bool": true @@ -171359,7 +173802,7 @@ }, { "name": "List", - "description": "Returns repos that the calling user has Manage permissions on.\nUse `next_page_token` to iterate through additional pages.", + "description": "Returns repos that the calling user has Manage permissions on.\nUse `next_page_token` to iterate through additional pages.\n\nDeprecated: This operation does not return a complete list of the repos in the workspace, because\nrepos with the Git CLI enabled are not included in its results. Instead, use the Repos and\nWorkspace APIs to find repos and their associated metadata in the workspace.", "summary": "Get repos.", "path": "/api/2.0/repos", "has_required_positional_arguments": true, @@ -172284,7 +174727,7 @@ }, { "name": "UpdateAccessRequestDestinations", - "description": "Updates the access request destinations for the given securable.\nThe caller must be a metastore admin, the owner of the securable, or a user that has the **MANAGE** privilege on the securable in order to assign destinations.\nDestinations cannot be updated for securables underneath schemas (tables, volumes, functions, and models). For these securable types, destinations are inherited from the parent securable.\nA maximum of 5 emails and 5 external notification destinations (Slack, Microsoft Teams, and Generic Webhook destinations) can be assigned to a securable.\nIf a URL destination is assigned, no other destinations can be set.\n\nThe supported securable types are: \"metastore\", \"catalog\", \"schema\", \"table\",\n\"external_location\", \"connection\", \"credential\", \"function\", \"registered_model\", and \"volume\".", + "description": "Updates the access request destinations for the given securable.\nThe caller must be a metastore admin, the owner of the securable, or a user that has the **MANAGE** privilege on the securable in order to assign destinations.\nA maximum of 5 emails and 5 external notification destinations (Slack, Microsoft Teams, and Generic Webhook destinations) can be assigned to a securable.\nIf a URL destination is assigned, no other destinations can be set.\n\nThe supported securable types are: \"metastore\", \"catalog\", \"schema\", \"table\",\n\"external_location\", \"connection\", \"credential\", \"function\", \"registered_model\", and \"volume\".", "summary": "Update Access Request Destinations.", "path": "/api/3.0/rfa/destinations", "can_use_json": true, @@ -179073,6 +181516,78 @@ } ] }, + { + "name": "PatchTelemetryConfig", + "description": "Updates the telemetry configuration of a serving endpoint.", + "summary": "Update the telemetry config of a serving endpoint.", + "path": "/api/2.0/serving-endpoints/{name}/telemetry-config", + "can_use_json": true, + "has_required_positional_arguments": true, + "launch_stage": "GA", + "cli_launch_stage_display": "GA", + "request": { + "pascal_name": "PatchTelemetryConfigRequest", + "is_object": true, + "required_fields": [ + { + "name": "name", + "description": "The name of the serving endpoint whose telemetry configuration is being updated. This field is required.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ], + "required_in_url_fields": [ + { + "name": "name", + "description": "The name of the serving endpoint whose telemetry configuration is being updated. This field is required.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, + "response": { + "pascal_name": "ServingEndpointDetailed", + "is_object": true + }, + "all_fields": [ + { + "name": "name", + "description": "The name of the serving endpoint whose telemetry configuration is being updated. This field is required.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + }, + { + "name": "telemetry_config", + "description": "The telemetry configuration to be applied to the serving endpoint.\nCan specify either a telemetry_profile_id to use an existing profile,\nor table_names to create a new profile with the specified Unity Catalog tables.\nIf not provided, the telemetry configuration will be removed from the endpoint.", + "is_request_body_field": true, + "is_optional_object": true, + "entity": { + "pascal_name": "TelemetryConfig", + "is_object": true + } + } + ], + "required_positional_arguments": [ + { + "name": "name", + "description": "The name of the serving endpoint whose telemetry configuration is being updated. This field is required.", + "required": true, + "is_path": true, + "entity": { + "is_string": true + } + } + ] + }, { "name": "Put", "description": "Deprecated: Please use AI Gateway to manage rate limits instead.", @@ -181840,7 +184355,7 @@ { "id": "sql.StatementExecution", "name": "StatementExecution", - "description": "The Databricks SQL Statement Execution API can be used to execute SQL statements on a SQL warehouse and fetch the result.\n\n**Getting started**\n\nWe suggest beginning with the [Databricks SQL Statement Execution API tutorial](https://docs.databricks.com/sql/api/sql-execution-tutorial.html).\n\n**Overview of statement execution and result fetching**\n\nStatement execution begins by issuing a :method:statementexecution/executeStatement request with a valid\nSQL statement and warehouse ID, along with optional parameters such as the data catalog and output format.\nIf no other parameters are specified, the server will wait for up to 10s before returning a response.\nIf the statement has completed within this timespan, the response will include the result data as a JSON\narray and metadata.\nOtherwise, if no result is available after the 10s timeout expired, the response will provide the\nstatement ID that can be used to poll for results by using a :method:statementexecution/getStatement\nrequest.\n\nYou can specify whether the call should behave synchronously, asynchronously\nor start synchronously with a fallback to asynchronous execution.\nThis is controlled with the `wait_timeout` and `on_wait_timeout` settings.\nIf `wait_timeout` is set between 5-50 seconds (default:\n10s), the call waits for results up to\nthe specified timeout; when set to `0s`, the call is asynchronous and responds immediately with a\nstatement ID. The `on_wait_timeout` setting specifies what should happen when the timeout is reached while\nthe statement execution has not yet finished. This can be set to either `CONTINUE`, to fallback to\nasynchronous mode, or it can be set to `CANCEL`, which cancels the statement.\n\nIn summary:\n- **Synchronous mode** (`wait_timeout=30s` and `on_wait_timeout=CANCEL`):\nThe call waits up to 30 seconds; if the statement execution finishes within this time,\nthe result data is returned directly in the response. If the execution takes longer than 30 seconds,\nthe execution is canceled and the call returns with a `CANCELED` state.\n- **Asynchronous mode** (`wait_timeout=0s` and `on_wait_timeout` is ignored):\nThe call doesn't wait for the statement to finish but returns directly with a statement ID.\nThe status of the statement execution can be polled by issuing :method:statementexecution/getStatement with the statement ID.\nOnce the execution has succeeded, this call also returns the result and metadata in the response.\n- **[Default] Hybrid mode** (`wait_timeout=10s` and `on_wait_timeout=CONTINUE`):\nThe call waits for up to 10 seconds; if the statement execution finishes within this time,\nthe result data is returned directly in the response. If the execution takes longer than 10\nseconds, a statement ID is returned. The statement ID can be used to fetch status and results in\nthe same way as in the asynchronous mode.\n\nDepending on the size, the result can be split into multiple chunks. If the statement execution is\nsuccessful, the statement response contains a manifest and the first chunk of the result. The manifest\ncontains schema information and provides metadata for each chunk in the result. Result chunks can be\nretrieved by index with :method:statementexecution/getStatementResultChunkN which may be called in any\norder and in parallel. For sequential fetching, each chunk, apart from the last, also contains a\n`next_chunk_index` and `next_chunk_internal_link` that point to the next chunk.\n\nA statement can be canceled with :method:statementexecution/cancelExecution.\n\n**Fetching result data: format and disposition**\n\nTo specify the format of the result data, use the `format` field, which can be set to one of the following\noptions: `JSON_ARRAY` (JSON), `ARROW_STREAM`\n([Apache Arrow Columnar](https://arrow.apache.org/overview/)), or `CSV`.\n\nThere are two ways to receive statement results, controlled by the `disposition` setting,\nwhich can be either `INLINE` or `EXTERNAL_LINKS`:\n\n- `INLINE`: In this mode, the result data is directly included in the response. It's best suited for smaller results.\nThis mode can only be used with the `JSON_ARRAY` format.\n\n- `EXTERNAL_LINKS`: In this mode, the response provides links that can be used to download the result data in chunks separately.\nThis approach is ideal for larger results and offers higher throughput.\nThis mode can be used with all the formats: `JSON_ARRAY`, `ARROW_STREAM`, and `CSV`.\n\nBy default, the API uses `format=JSON_ARRAY` and `disposition=INLINE`.\n\n**Limits and limitations**\n\nNote: The byte limit for INLINE disposition is based on internal storage metrics and will not exactly match the byte count of the actual payload.\n\n- Statements with `disposition=INLINE` are limited to 25 MiB and will fail when this limit is\nexceeded.\n- Statements with `disposition=EXTERNAL_LINKS` are limited to 100 GiB. Result sets larger than\nthis limit will be truncated. Truncation is indicated by the `truncated` field in the result manifest.\n- The maximum query text size is 16 MiB.\n- Cancelation might silently fail. A successful response from a cancel request indicates that the cancel request\nwas successfully received and sent to the processing engine. However, an outstanding statement\nmight have already completed execution when the cancel request arrives. Polling\nfor status until a terminal state is reached is a reliable way to determine the final state.\n- Wait timeouts are approximate, occur server-side, and cannot account for things such as caller delays and network latency from caller to service.\n- To guarantee that the statement is kept alive, you must poll at least once every 15 minutes.\n- The results are only available for one hour after success; polling does not extend this.\n- The SQL Execution API must be used for the entire lifecycle of the statement. For example, you cannot use the Jobs API to execute the command, and then the SQL Execution API to cancel it.", + "description": "The Databricks SQL Statement Execution API can be used to execute SQL statements on a SQL warehouse and fetch the result.\n\n**Getting started**\n\nWe suggest beginning with the [Databricks SQL Statement Execution API tutorial](https://docs.databricks.com/sql/api/sql-execution-tutorial.html).\n\n**Overview of statement execution and result fetching**\n\nStatement execution begins by issuing a :method:statementexecution/executeStatement request with a valid\nSQL statement and warehouse ID, along with optional parameters such as the data catalog and output format.\nIf no other parameters are specified, the server will wait for up to 10s before returning a response.\nIf the statement has completed within this timespan, the response will include the result data as a JSON\narray and metadata.\nOtherwise, if no result is available after the 10s timeout expired, the response will provide the\nstatement ID that can be used to poll for results by using a :method:statementexecution/getStatement\nrequest.\n\nYou can specify whether the call should behave synchronously, asynchronously\nor start synchronously with a fallback to asynchronous execution.\nThis is controlled with the `wait_timeout` and `on_wait_timeout` settings.\nIf `wait_timeout` is set between 5-50 seconds (default:\n10s), the call waits for results up to\nthe specified timeout; when set to `0s`, the call is asynchronous and responds immediately with a\nstatement ID. The `on_wait_timeout` setting specifies what should happen when the timeout is reached while\nthe statement execution has not yet finished. This can be set to either `CONTINUE`, to fallback to\nasynchronous mode, or it can be set to `CANCEL`, which cancels the statement.\n\nIn summary:\n- **Synchronous mode** (`wait_timeout=30s` and `on_wait_timeout=CANCEL`):\nThe call waits up to 30 seconds; if the statement execution finishes within this time,\nthe result data is returned directly in the response. If the execution takes longer than 30 seconds,\nthe execution is canceled and the call returns with a `CANCELED` state.\n- **Asynchronous mode** (`wait_timeout=0s` and `on_wait_timeout` is ignored):\nThe call doesn't wait for the statement to finish but returns directly with a statement ID.\nThe status of the statement execution can be polled by issuing :method:statementexecution/getStatement with the statement ID.\nOnce the execution has succeeded, this call also returns the result and metadata in the response.\n- **[Default] Hybrid mode** (`wait_timeout=10s` and `on_wait_timeout=CONTINUE`):\nThe call waits for up to 10 seconds; if the statement execution finishes within this time,\nthe result data is returned directly in the response. If the execution takes longer than 10\nseconds, a statement ID is returned. The statement ID can be used to fetch status and results in\nthe same way as in the asynchronous mode.\n\nDepending on the size, the result can be split into multiple chunks. If the statement execution is\nsuccessful, the statement response contains a manifest and the first chunk of the result. The manifest\ncontains schema information and provides metadata for each chunk in the result. Result chunks can be\nretrieved by index with :method:statementexecution/getStatementResultChunkN which may be called in any\norder and in parallel. For sequential fetching, each chunk, apart from the last, also contains a\n`next_chunk_index` and `next_chunk_internal_link` that point to the next chunk.\n\nA statement can be canceled with :method:statementexecution/cancelExecution.\n\n**Fetching result data: format and disposition**\n\nTo specify the format of the result data, use the `format` field, which can be set to one of the following\noptions: `JSON_ARRAY` (JSON), `ARROW_STREAM`\n([Apache Arrow Columnar](https://arrow.apache.org/overview/)), or `CSV`.\n\nThere are two ways to receive statement results, controlled by the `disposition` setting,\nwhich can be either `INLINE` or `EXTERNAL_LINKS`:\n\n- `INLINE`: In this mode, the result data is directly included in the response. It's best suited for smaller results.\nThis mode can only be used with the `JSON_ARRAY` format.\n\n- `EXTERNAL_LINKS`: In this mode, the response provides links that can be used to download the result data in chunks separately.\nThis approach is ideal for larger results and offers higher throughput.\nThis mode can be used with all the formats: `JSON_ARRAY`, `ARROW_STREAM`, and `CSV`.\n\nBy default, the API uses `format=JSON_ARRAY` and `disposition=INLINE`.\n\n**Limits and limitations**\n\nNote: The byte limit for INLINE disposition is based on internal storage metrics and will not exactly match the byte count of the actual payload.\n\n- Statements with `disposition=INLINE` are limited to 25 MiB and will fail when this limit is\nexceeded.\n- Statements with `disposition=EXTERNAL_LINKS` are limited to 100 GiB. Result sets larger than\nthis limit will be truncated. Truncation is indicated by the `truncated` field in the result manifest.\n- The maximum query text size is 16 MiB.\n- Cancelation might silently fail. A successful response from a cancel request indicates that the cancel request\nwas successfully received and sent to the processing engine. However, an outstanding statement\nmight have already completed execution when the cancel request arrives. Polling\nfor status until a terminal state is reached is a reliable way to determine the final state.\n- Wait timeouts are approximate, occur server-side, and cannot account for things such as caller delays and network latency from caller to service.\n- To keep a statement alive, poll for its status at least once every 15 minutes. Regular polling guarantees the statement stays alive.\n- To stop a statement you no longer need, cancel it explicitly with :method:statementexecution/cancelExecution. Stopping polling does not cancel a statement. Explicit cancellation is the only way to end statement execution on demand.\n- The results are only available for one hour after success; polling does not extend this.\n- The SQL Execution API must be used for the entire lifecycle of the statement. For example, you cannot use the Jobs API to execute the command, and then the SQL Execution API to cancel it.", "package": { "name": "sql" }, @@ -184192,7 +186707,7 @@ }, { "name": "CreateTool", - "description": "Creates a Tool under a Supervisor Agent. Specify one of \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"app\", \"volume\", \"dashboard\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\" in the request body. The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Creates a Tool under a Supervisor Agent. Specify one of \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"app\", \"volume\", \"dashboard\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\" in the request body. The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "summary": "Create a Tool.", "path": "/api/2.1/{parent=supervisor-agents/*}/tools", "can_use_json": true, @@ -184278,7 +186793,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184289,7 +186804,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184306,7 +186821,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184317,7 +186832,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184347,7 +186862,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184358,7 +186873,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184440,7 +186955,7 @@ }, { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -184496,7 +187011,7 @@ }, { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -185059,7 +187574,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -185070,7 +187585,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186221,7 +188736,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186232,7 +188747,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186249,7 +188764,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186260,7 +188775,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186291,7 +188806,7 @@ "required_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186302,7 +188817,7 @@ "required_request_body_fields": [ { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186384,7 +188899,7 @@ }, { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -186440,7 +188955,7 @@ }, { "name": "tool_type", - "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"web_search\", \"skill\". The legacy values \"lakeview_dashboard\" and \"uc_table\" are also accepted and remain equivalent to \"dashboard\" and \"table\" respectively.", + "description": "Tool type. Must be one of: \"genie_space\", \"knowledge_assistant\", \"uc_function\", \"uc_connection\", \"uc_mcp\", \"app\", \"volume\", \"dashboard\", \"serving_endpoint\", \"table\", \"vector_search_index\", \"catalog\", \"schema\", \"supervisor_agent\", \"databricks_web_search\", \"skill\". The legacy values \"lakeview_dashboard\", \"uc_table\", and \"web_search\" are also accepted and remain equivalent to \"dashboard\", \"table\", and \"databricks_web_search\" respectively. The \"databricks_web_search\" tool_type maps to the `web_search` spec field.", "required": true, "is_request_body_field": true, "entity": { @@ -194560,7 +197075,7 @@ }, { "name": "cluster_size", - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "is_request_body_field": true, "entity": { "is_string": true @@ -195171,7 +197686,7 @@ }, { "name": "cluster_size", - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "is_request_body_field": true, "entity": { "is_string": true @@ -199545,7 +202060,7 @@ "name": "CreateWorkspaceAssignmentDetailProxy", "description": "Creates a workspace assignment detail for a principal (workspace-level proxy). Entitlement\ngrants are applied individually and non-atomically — if a failure occurs partway through, the\nprincipal will be assigned to the workspace but with only a subset of the requested\nentitlements. Use GetWorkspaceAssignmentDetail to confirm which entitlements were successfully\ngranted.", "summary": "Create a workspace assignment detail for a workspace.", - "path": "/api/2.0/identity/workspaceAssignmentDetails", + "path": "/api/2.0/identity/workspace-assignment-details", "can_use_json": true, "is_hidden_cli": true, "has_required_positional_arguments": true, @@ -199683,6 +202198,38 @@ "is_string": true } }, + { + "name": "effective_entitlements", + "description": "The principal's full effective entitlements granted in this workspace: every entitlement it holds\nwhether granted directly or via group membership. Populated on Get; empty on List.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "EntitlementList", + "array_value": { + "pascal_name": "Entitlement", + "enum": [ + { + "content": "ALLOW_CLUSTER_CREATE" + }, + { + "content": "ALLOW_INSTANCE_POOL_CREATE" + }, + { + "content": "DATABRICKS_SQL_ACCESS" + }, + { + "content": "WORKSPACE_ACCESS" + }, + { + "content": "WORKSPACE_ADMIN" + }, + { + "content": "WORKSPACE_CONSUME" + } + ] + } + } + }, { "name": "entitlements", "description": "Entitlements granted directly to the principal on this workspace. The only client-settable\nfield: create and update manage exactly this set (including entitlements the principal also\nholds via a group).\nNot populated by ListWorkspaceAssignmentDetails (omitted for scalability); call\nGetWorkspaceAssignmentDetail to read the entitlements for a single principal.", @@ -199768,7 +202315,7 @@ "name": "DeleteWorkspaceAssignmentDetailProxy", "description": "Deletes a workspace assignment detail for a principal (workspace-level proxy), revoking all\nassociated entitlements. Entitlement revocations are applied individually and non-atomically\n— if a failure occurs partway through, the principal remains assigned with a subset of its\noriginal entitlements, and the operation is safe to retry.", "summary": "Delete a workspace assignment detail for a workspace.", - "path": "/api/2.0/identity/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/workspace-assignment-details/{principal_id}", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -199829,7 +202376,7 @@ "name": "GetWorkspaceAccessDetailLocal", "description": "Returns the access details for a principal in the current workspace. Allows for checking access details for any\nprovisioned principal (user, service principal, or group) in the current workspace.\n* Provisioned principal here refers to one that has been synced into Databricks from the customer's IdP or\nadded explicitly to Databricks via SCIM/UI.\nAllows for passing in a \"view\" parameter to control what fields are returned (BASIC by default or FULL).", "summary": "Get workspace access details for a principal.", - "path": "/api/2.0/identity/workspaceAccessDetails/{principal_id}", + "path": "/api/2.0/identity/workspace-access-details/{principal_id}", "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", "cli_launch_stage_label": "*Beta*", @@ -199908,7 +202455,7 @@ "name": "GetWorkspaceAssignmentDetailProxy", "description": "Returns the assignment details for a principal in a workspace (workspace-level proxy).", "summary": "Get workspace assignment details for a principal.", - "path": "/api/2.0/identity/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/workspace-assignment-details/{principal_id}", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -199993,7 +202540,7 @@ "name": "ListWorkspaceAssignmentDetailsProxy", "description": "Lists workspace assignment details for a workspace (workspace-level proxy).\nFor scalability, the response omits the per-principal entitlement fields (`entitlements` and\n`effective_entitlements`); call GetWorkspaceAssignmentDetailProxy to read entitlements for a\nsingle principal.", "summary": "List workspace assignment details for a workspace.", - "path": "/api/2.0/identity/workspaceAssignmentDetails", + "path": "/api/2.0/identity/workspace-assignment-details", "is_hidden_cli": true, "has_required_positional_arguments": true, "launch_stage": "PRIVATE_PREVIEW", @@ -200029,7 +202576,7 @@ "name": "ResolveGroupProxy", "description": "Resolves a group with the given external ID from the customer's IdP. If the group does not exist, it will be created in the account.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external group in the Databricks account.", - "path": "/api/2.0/identity/groups/resolveByExternalId", + "path": "/api/2.0/identity/groups/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -200094,7 +202641,7 @@ "name": "ResolveServicePrincipalProxy", "description": "Resolves an SP with the given external ID from the customer's IdP. If the SP does not exist, it will be created.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external service principal in the Databricks account.", - "path": "/api/2.0/identity/servicePrincipals/resolveByExternalId", + "path": "/api/2.0/identity/service-principals/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -200159,7 +202706,7 @@ "name": "ResolveUserProxy", "description": "Resolves a user with the given external ID from the customer's IdP. If the user does not exist, it will be created.\nIf the customer is not onboarded onto Automatic Identity Management (AIM), this will return an error.", "summary": "Resolve an external user in the Databricks account.", - "path": "/api/2.0/identity/users/resolveByExternalId", + "path": "/api/2.0/identity/users/resolve-by-external-id", "can_use_json": true, "has_required_positional_arguments": true, "launch_stage": "PUBLIC_BETA", @@ -200224,7 +202771,7 @@ "name": "UpdateWorkspaceAssignmentDetailProxy", "description": "Updates the entitlements of a directly assigned principal in a workspace (workspace-level\nproxy). Entitlement changes are applied individually and non-atomically — if a failure occurs\npartway through, only a subset of the requested changes may have been applied. Use\nGetWorkspaceAssignmentDetail to confirm the final state.", "summary": "Update a workspace assignment detail for a workspace.", - "path": "/api/2.0/identity/workspaceAssignmentDetails/{principal_id}", + "path": "/api/2.0/identity/workspace-assignment-details/{principal_id}", "can_use_json": true, "is_hidden_cli": true, "has_required_positional_arguments": true, @@ -200419,6 +202966,38 @@ "is_string": true } }, + { + "name": "effective_entitlements", + "description": "The principal's full effective entitlements granted in this workspace: every entitlement it holds\nwhether granted directly or via group membership. Populated on Get; empty on List.", + "is_output_only": true, + "is_request_body_field": true, + "entity": { + "pascal_name": "EntitlementList", + "array_value": { + "pascal_name": "Entitlement", + "enum": [ + { + "content": "ALLOW_CLUSTER_CREATE" + }, + { + "content": "ALLOW_INSTANCE_POOL_CREATE" + }, + { + "content": "DATABRICKS_SQL_ACCESS" + }, + { + "content": "WORKSPACE_ACCESS" + }, + { + "content": "WORKSPACE_ADMIN" + }, + { + "content": "WORKSPACE_CONSUME" + } + ] + } + } + }, { "name": "entitlements", "description": "Entitlements granted directly to the principal on this workspace. The only client-settable\nfield: create and update manage exactly this set (including entitlements the principal also\nholds via a group).\nNot populated by ListWorkspaceAssignmentDetails (omitted for scalability); call\nGetWorkspaceAssignmentDetail to read the entitlements for a single principal.", diff --git a/.github/OWNERS b/.github/OWNERS index dab66d8c9c7..76bc081f0eb 100644 --- a/.github/OWNERS +++ b/.github/OWNERS @@ -14,6 +14,11 @@ # Labs /cmd/labs/ @alexott @asnare +# Local environments / DB Connect +/libs/localenv/ @rugpanov @rclarey @anton-107 @misha-db +/cmd/environments/ @rugpanov @rclarey @anton-107 @misha-db +/acceptance/localenv/ @rugpanov @rclarey @anton-107 @misha-db + # Apps /cmd/apps/ team:eng-apps-devex /cmd/workspace/apps/ team:eng-apps-devex diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index dfc66648f3f..4f39093beb3 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -137,9 +137,33 @@ jobs: with: cache-key: test-${{ matrix.deployment }} + # Make origin/main available with enough history for `git diff --merge-base + # origin/main`, which the acceptance subset selector uses to always run the + # tests this PR adds/changes on the windows/macOS cells. actions/checkout does + # a shallow single-ref clone by default, so origin/main is otherwise absent and + # the diff fails, silently disabling change detection. This must run AFTER + # setup-build-environment, whose own checkout would re-shallow the repo. Only + # PRs subset tests, so the fetch is unnecessary elsewhere. + - name: Fetch origin/main for change detection + if: ${{ github.event_name == 'pull_request' }} + run: | + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + git fetch --no-tags --unshallow origin + fi + - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} + # On pull requests, run only a subset of acceptance subtests on the slower + # windows/macOS cells so neither lands on the critical path to merge. An + # empty percentage disables subsetting, so Linux PRs and push-to-main run + # the full suite (full coverage on every OS). The percentages are fixed and + # tuned to keep windows/macOS under the Linux wall-clock time. The seed is + # the head commit, so a re-run of the same commit repeats the subset while a + # new commit reshuffles it. + TESTS_SELECT_SUBSET_SEED: ${{ github.event.pull_request.head.sha }} + TESTS_SELECT_SUBSET_PCT: ${{ github.event_name == 'pull_request' && (matrix.os.name == 'windows' && '10' || matrix.os.name == 'macos' && '60') || '' }} run: go tool -modfile=tools/task/go.mod task test - name: Upload gotestsum JSON output @@ -152,6 +176,13 @@ jobs: if-no-files-found: warn retention-days: 7 + # `task test` prints a lot of output that GitHub is slow to render, so + # surface just the failed test names on the job summary page (and in this + # step's log) for a quick read of what broke. + - name: Summarize failed tests + if: ${{ failure() }} + run: uv run tools/summarize_failed_tests.py test-output.json | tee -a "$GITHUB_STEP_SUMMARY" + - name: Check no files changed or appeared after running tests run: | # Register untracked files with intent-to-add so `git diff` reports them diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml index 735ef5c7ebc..eb20f7f3328 100644 --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -34,6 +34,15 @@ jobs: github.repository == 'databricks/databricks-sdk-go' || github.repository == 'databricks/databricks-sdk-py' || github.repository == 'databricks/databricks-sdk-java' + env: + # Ref the release is cut from. Releasing from a branch is currently + # supported in the terraform repository only: for + # terraform-provider-databricks this follows the dispatched branch + # (``github.ref_name``), so a release can be cut from a branch other + # than main; every other synced repo (SDK go/py/java, …) stays pinned + # to main, preserving the pre-existing hardcoded-main behaviour and + # blocking branch releases even via a manual non-main dispatch. + RELEASE_REF: ${{ github.repository == 'databricks/terraform-provider-databricks' && github.ref_name || 'main' }} environment: "release-is" runs-on: group: databricks-protected-runner-group @@ -49,12 +58,19 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - # Force re-resolution of ``main`` at step time. Without - # ``ref:``, checkout pins to ``github.sha`` — the SHA frozen - # at workflow_dispatch time — which means re-running a stale - # dispatch checks out an older main even when newer commits - # exist. - ref: main + # Check out the release ref. ``RELEASE_REF`` is the dispatched + # branch for terraform-provider-databricks and ``main`` for every + # other repo (see the job-level ``env``). For scheduled runs and + # manual dispatches left at defaults this is the default branch + # (main); dispatching terraform with ``--ref `` cuts the + # release from that branch instead. + # + # Naming the ref explicitly (rather than omitting ``ref:``) + # forces re-resolution of the branch head at step time. Without + # it, checkout pins to ``github.sha`` — the SHA frozen at + # workflow_dispatch time — so re-running a stale dispatch would + # check out an older commit even when newer ones exist. + ref: ${{ env.RELEASE_REF }} fetch-depth: 0 token: ${{ steps.generate-token.outputs.token }} @@ -72,6 +88,12 @@ jobs: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} PACKAGES: ${{ inputs.packages }} + # Branch the release is cut from. internal/genkit/release_tagging.py commits the + # changelog bump and creates the tag on this branch, and its + # concurrent-advance guard checks this branch. Matches the ref + # checked out above (``RELEASE_REF``): the dispatched branch for + # terraform, main everywhere else. + DECO_TAGGING_REF: ${{ env.RELEASE_REF }} run: | if [ -n "$PACKAGES" ]; then uv run --locked internal/genkit/release_tagging.py --package "$PACKAGES" diff --git a/.gitignore b/.gitignore index 543b638a537..4b82c6d1521 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ go.work.sum tools/gofumpt .claude/worktrees/ .worktrees/ +.isaac/ diff --git a/.nextchanges/bundles/deprecate-terraform-engine.md b/.nextchanges/bundles/deprecate-terraform-engine.md new file mode 100644 index 00000000000..955c523bed1 --- /dev/null +++ b/.nextchanges/bundles/deprecate-terraform-engine.md @@ -0,0 +1 @@ +The terraform deployment engine is deprecated and will stop working in a future version of the CLI. Setting `bundle.engine: terraform` now emits a deprecation warning. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct for how to migrate to the direct deployment engine. diff --git a/.nextchanges/version b/.nextchanges/version index f8e233b2733..1cac385c6cb 100644 --- a/.nextchanges/version +++ b/.nextchanges/version @@ -1 +1 @@ -1.9.0 +1.11.0 diff --git a/.release_metadata.json b/.release_metadata.json index a2e04f53296..b8a9f0b3d6b 100644 --- a/.release_metadata.json +++ b/.release_metadata.json @@ -1,3 +1,3 @@ { - "timestamp": "2026-07-15 14:31:34+0000" + "timestamp": "2026-07-29 15:56:37+0000" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a1964a288bd..571767c6fce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Version changelog +## Release v1.10.0 (2026-07-29) + +### CLI + + * `ssh connect` now supports specifying a serverless usage policy with `--usage-policy-id` + +### Bundles + + * Fixed `bundle deploy`/`bundle destroy` failing when an app enters the transient DELETING state between plan and apply (e.g. with a saved plan); the delete is now treated as complete instead of erroring (direct engine only). + * Fixed `bundle deploy`/`bundle destroy` failing when an app is still in the transient DELETING state; the delete is now treated as complete instead of erroring (direct engine only). + * `bundle destroy --force-lock` now proceeds without a deployment lock when the workspace directory is at its child-node limit and cannot accept the lock file, so a deployment can still be torn down when the workspace is full. + * Empty-string values on optional (omitempty) resource fields are now dropped before deployment instead of being sent to the backend. This fixes deploys failing with errors like `'' is not a valid cluster policy ID` when a field such as `policy_id` was set to `""` (often via a variable that resolved to an empty string). The behavior now matches between the terraform and direct engines and is reflected in `bundle validate -o json`. + * `bundle validate` and `bundle deploy` now reject a grant that is missing a `principal` with an error instead of a warning. Previously the deploy would start and, on the direct engine, create the securable before the grants PATCH failed (`400 INVALID_PARAMETER_VALUE`), leaving a partially-applied deployment. + * `bundle validate` and `bundle deploy` now reject a grant with an empty `privileges` list with an error. Previously, on the direct engine, such a grant never converged: the backend drops principals with no privileges, so every subsequent `bundle plan` reported the grant as a perpetual update. + * Fixes [#6030](https://github.com/databricks/cli/issues/6030): spurious `update` on catalog/schema/volume grants (direct engine); a principal granted `ALL_PRIVILEGES` no longer drifts when the backend also reports the concrete privileges it implies ([#6064](https://github.com/databricks/cli/pull/6064)). + * Use vector search endpoint permission types that are supported by the backend ([#6022](https://github.com/databricks/cli/pull/6022)). + +### Dependency Updates + + * Bump `github.com/databricks/databricks-sdk-go` from v0.160.0 to v0.165.0. + * Upgrade Terraform provider to 1.123.0 + + +## Release v1.9.0 (2026-07-22) + +### CLI + + * `databricks auth profiles` no longer stalls on an unreachable workspace and instead fails validation after 5 seconds per host ([#5928](https://github.com/databricks/cli/pull/5928)). + * Fixed `databricks fs rm -r` failing on UC Volumes backed by GCS when a directory becomes empty during recursive deletion ([#5958](https://github.com/databricks/cli/pull/5958)). + * You can now ask questions about your data directly from the CLI with `databricks genie ask "..."`. Genie answers natural-language questions ("what were total sales last month?", "which tables are in the sales catalog?"), runs the query inside Databricks, and renders the answer in the terminal. This promotes the former `databricks experimental genie ask` command; the experimental alias still works but is deprecated and will be removed in a future release ([#6010](https://github.com/databricks/cli/pull/6010)). + +### Bundles + + * `bundle validate` now reports a clear error when a `sql_warehouse` is missing a `name` (including whitespace-only names), and a warning when a grant is missing a `principal` ([#5818](https://github.com/databricks/cli/pull/5818)). + * Bundle templates now scaffold an `AGENTS.md` that points coding agents at Databricks AI Tools, alongside a minimal `CLAUDE.md` that includes it via `@AGENTS.md` ([#5996](https://github.com/databricks/cli/pull/5996)). + * `bundle generate job` can now download workspace files referenced by `spark_python_task`, rewriting them to a relative path like it already does for notebooks. This is opt-in via the `--download-spark-python-files` flag ([#5799](https://github.com/databricks/cli/pull/5799)). + * Simplified the `default-minimal` bundle template and added an alias `databricks bundle init empty` ([#5899](https://github.com/databricks/cli/pull/5899)). + * Add support for the `instance_pools` resource type in Declarative Automation Bundles. Instance pools are only supported in direct deployment mode. + * Do not emit "unknown field" warnings for YAML anchors grouped in a list or map, matching the existing suppression for standalone anchors ([#5975](https://github.com/databricks/cli/pull/5975)). + * Provide an actionable error message if databricks.yml is missing or DATABRICKS_BUNDLE_ROOT is invalid ([#5953](https://github.com/databricks/cli/pull/5953)). + +### Dependency Updates + + * Bump `github.com/databricks/databricks-sdk-go` from v0.154.0 to v0.160.0 ([#5982](https://github.com/databricks/cli/pull/5982)). + * Bump Terraform provider from v1.121.0 to v1.122.0 ([#5977](https://github.com/databricks/cli/pull/5977)). + + ## Release v1.8.0 (2026-07-15) ### Notable Changes diff --git a/Taskfile.yml b/Taskfile.yml index 4384de278c4..1eb41d6ac53 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -309,15 +309,16 @@ tasks: cmds: - ./tools/check_deadcode.py - check-uv-lock: - desc: Fail if a Databricks PyPI proxy URL leaked into a uv.lock - # pydabs-codegen reverts this proxy-URL churn after regenerating; this is a + check-lockfiles: + desc: Fail if a Databricks PyPI proxy URL leaked into a uv.lock or .py.lock + # pydabs-codegen reverts this proxy-URL churn after regenerating, and + # generate-clijson scrubs it from the genkit tagging locks; this is a # backstop against a proxy URL reaching a committed lock any other way. cmds: - - "! git grep -lF databricks.com -- '*uv.lock'" + - "! git grep -lF databricks.com -- '*uv.lock' '*.py.lock'" checks: - desc: Run quick checks (tidy, whitespace, links, deadcode, changelog, uv.lock) + desc: Run quick checks (tidy, whitespace, links, deadcode, changelog, lockfiles) # Sequential: `tidy` rewrites go.mod/go.sum and any future tidy work # touching more paths should not race with whitespace/link scanners. cmds: @@ -326,7 +327,7 @@ tasks: - task: links - task: deadcode - task: check-changelog - - task: check-uv-lock + - task: check-lockfiles install-pythons: desc: Install Python 3.9-3.13 via uv @@ -904,6 +905,20 @@ tasks: sed -i 's|tagging.py|internal/genkit/release_tagging.py|g' .github/workflows/tagging.yml fi - cp internal/genkit/tagging.py.lock internal/genkit/release_tagging.py.lock + # In an environment configured with the internal PyPI proxy, uv bakes the + # proxy URL into the resolved locks. Unlike pydabs (which blind-reverts its + # locks because codegen never wants to change them), these locks legitimately + # track the synced producer's deps, so rewrite only the proxy URL back to the + # public registry and preserve any real dependency changes. Glob the lock + # files so a future genkit lock is scrubbed without editing this task. The + # trailing slash is optional so both proxy URL forms are matched. + # check-lockfiles is the backstop if this ever misses. + - | + if [ "$(uname)" = "Darwin" ]; then + sed -i '' 's|https://pypi-proxy.cloud.databricks.com/simple/\{0,1\}|https://pypi.org/simple|g' internal/genkit/*.py.lock + else + sed -i 's|https://pypi-proxy.cloud.databricks.com/simple/\{0,1\}|https://pypi.org/simple|g' internal/genkit/*.py.lock + fi - "{{.GO_TOOL}} yamlfmt .github/workflows/tagging.yml" - task: ws diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e5781f88f43..2c9c2b31a1c 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -111,12 +111,6 @@ const ( var ApplyCITimeoutMultipler = os.Getenv("GITHUB_WORKFLOW") != "" -// IsPullRequest is true when the test run was triggered by a GitHub pull_request -// event. GitHub Actions sets GITHUB_EVENT_NAME automatically. On pull requests we -// additionally apply each test's GOOSOnPR filter, so OS-independent tests can be -// skipped on windows/macOS for PRs while still running on every OS on push to main. -var IsPullRequest = os.Getenv("GITHUB_EVENT_NAME") == "pull_request" - var exeSuffix = func() string { if runtime.GOOS == "windows" { return ".exe" @@ -241,10 +235,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { // signal because os.LookupEnv reports them as present. // Keep this list in sync with listKnownAgents() in // github.com/databricks/databricks-sdk-go/useragent/agent.go - // plus the AGENT and AI_AGENT generic fallbacks. + // plus the AGENT and AI_AGENT generic fallbacks and the CLI's own + // AIDEVKIT_HOME detection override. for _, v := range []string{ "AGENT", "AI_AGENT", + "AIDEVKIT_HOME", "AMP_CURRENT_THREAD_ID", "ANTIGRAVITY_AGENT", "AUGMENT_AGENT", @@ -434,21 +430,29 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { testDirs := getTests(t) require.NotEmpty(t, testDirs) + testDirsSet := make(map[string]bool, len(testDirs)) + for _, d := range testDirs { + testDirsSet[d] = true + } + skipLocalMode := os.Getenv(SkipLocalEnvVar) - // changedTests maps test dir to extra env filters to apply for that dir. - // nil value means all variants run; a non-nil slice restricts to matching variants. - var changedTests map[string][]string + subset := newSubsetSelector(t, testdiff.OverwriteMode, Forcerun) + switch skipLocalMode { - case "", SkipLocalAll: - case SkipLocalWithChanged: - testDirsSet := make(map[string]bool, len(testDirs)) - for _, d := range testDirs { - testDirsSet[d] = true - } - changedTests = selectChangedLocalTests(testDirsSet) + case "", SkipLocalAll, SkipLocalWithChanged: default: t.Fatalf("Unsupported %s=%q, expected %q or %q", SkipLocalEnvVar, skipLocalMode, SkipLocalAll, SkipLocalWithChanged) } + skipLocalWithChanged := skipLocalMode == SkipLocalWithChanged + + // changedTests maps test dir to extra env filters for added/modified tests; nil + // filters means all variants of that dir changed. Both SkipLocalWithChanged and the + // subset selector keep these tests, so detect them at most once here. + var changedTests map[string][]string + if skipLocalWithChanged || subset.enabled { + changedTests = selectChangedLocalTests(t, testDirsSet) + } + subset.changed = changedTests if singleTest != "" { testDirs = slices.DeleteFunc(testDirs, func(n string) bool { @@ -547,6 +551,9 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { // If the matrix expands to a single empty envset, run the test directly // without creating a subtest (avoids the "#00" dummy subtest name). if len(expanded) == 1 && len(expanded[0]) == 0 { + if reason := subset.skipReason(dir, nil); reason != "" { + t.Skip(reason) + } runTest(t, dir, 0, coverDir, repls.Clone(), config, nil, envFilters, sandboxProxyURL) } else { for ind, envset := range expanded { @@ -555,10 +562,15 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { if runParallel { t.Parallel() } - // For invariant dirs re-enabled by a specific config change, - // skip variants not matching that config. - if variantFilters := changedTests[dir]; variantFilters != nil { - checkEnvFilters(t, envset, variantFilters) + // Under SkipLocalWithChanged, an invariant dir re-enabled by a + // specific config change runs only its matching variants. + if skipLocalWithChanged { + if variantFilters := changedTests[dir]; variantFilters != nil { + checkEnvFilters(t, envset, variantFilters) + } + } + if reason := subset.skipReason(dir, envset); reason != "" { + t.Skip(reason) } runTest(t, dir, ind, coverDir, repls.Clone(), config, envset, envFilters, sandboxProxyURL) }) @@ -661,13 +673,6 @@ func getSkipReason(config *internal.TestConfig, configPath, dir, skipLocalMode s return fmt.Sprintf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath) } - if IsPullRequest { - isEnabled, isPresent := config.GOOSOnPR[runtime.GOOS] - if isPresent && !isEnabled { - return fmt.Sprintf("Disabled via GOOSOnPR.%s setting in %s", runtime.GOOS, configPath) - } - } - cloudEnv := os.Getenv("CLOUD_ENV") isRunningOnCloud := cloudEnv != "" diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 30405fbc5a0..7d1ba207434 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES +Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] PATTERN HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*" STATUS_CODE HTTP status code to return, e.g. 504 OFFSET number of requests to let through before fault starts TIMES number of times to return the fault response + ERROR_CODE optional error_code for the response body, e.g. + MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) The rule is scoped to the current DATABRICKS_TOKEN so it only affects the test that registers it, even when tests share a server. @@ -25,12 +27,13 @@ print("DATABRICKS_HOST not set", file=sys.stderr) sys.exit(1) -if len(sys.argv) != 5: - print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES", file=sys.stderr) +if len(sys.argv) not in (5, 6): + print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) sys.exit(1) pattern, status_code, offset, times = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) -body = '{"error_code":"INJECTED","message":"Fault injected by test."}' +error_code = sys.argv[5] if len(sys.argv) == 6 else "INJECTED" +body = json.dumps({"error_code": error_code, "message": "Fault injected by test."}) data = json.dumps( { diff --git a/acceptance/bin/normalize_uc_payload.py b/acceptance/bin/normalize_uc_payload.py new file mode 100755 index 00000000000..943e73e1ed2 --- /dev/null +++ b/acceptance/bin/normalize_uc_payload.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Normalize a UC JSON payload read from stdin, writing the result to stdout. The payload +is either a bundle plan or a `catalogs get` / `schemas get` response; the same pruning +applies to both (the plan-only `changes` handling simply does not fire on a GET response). + +UC managed-default properties (unity.catalog.managed..defaults.*) are ALWAYS +pruned. UC auto-populates these on every catalog/schema, but only on some clouds (AWS/GCP +yes, Azure no), so a committed golden cannot show them. The CLI already classifies them as +backend defaults, so they never affect the plan; dropping them here is output-only. + +Managed properties surface in three shapes, all handled by pruning matching keys: + - a `properties` map (remote_state.properties, new_state.value.properties, ...); + an emptied `properties` object is then dropped so {} vs absent doesn't diverge. + - a `changes` entry keyed `properties` or `properties['unity.catalog.managed...']`; + an emptied `changes` object is likewise dropped. + +Any field names passed as arguments are additionally deleted wherever they appear. These +are the volatile server-set fields (created_at, metastore_id, schema_id, ...) each test +previously stripped with an inline `jq 'walk(del(...))'`; the set is test-specific, so it +stays at the call site rather than being centralized here. +""" + +import json +import re +import sys + +# Matches a managed-default property key, whether bare (as a map key) or wrapped in a +# plan change key like properties['unity.catalog.managed.delta.defaults.delta....']. +managed_re = re.compile(r"unity\.catalog\.managed\..*\.defaults\..*") + + +def is_managed_change(key, value): + """Report whether a plan `changes` entry is purely a managed-default change. + + UC emits the managed-property drift either per key (change key + `properties['unity.catalog.managed...']`) or as the whole map at the parent path + (bare `properties` key, managed map under `remote`). Both are backend defaults the + CLI already skips, so they are output noise. + + >>> is_managed_change("properties['unity.catalog.managed.a.defaults.b']", {"action": "skip"}) + True + >>> is_managed_change("properties", {"action": "skip", "reason": "backend_default", "remote": {"unity.catalog.managed.a.defaults.b": "1"}}) + True + >>> is_managed_change("properties", {"action": "update", "remote": {"custom": "v"}}) + False + >>> is_managed_change("name", {"action": "update"}) + False + """ + if managed_re.search(key): + return True + if key != "properties" or not isinstance(value, dict): + return False + remote = value.get("remote") + return isinstance(remote, dict) and bool(remote) and all(managed_re.search(k) for k in remote) + + +def prune(node, drop_fields): + """Recursively drop managed-default properties and the given volatile fields. + + >>> prune({"created_at": 1, "name": "c"}, {"created_at"}) + {'name': 'c'} + >>> prune({"properties": {"unity.catalog.managed.delta.defaults.x": "true"}}, set()) + {} + >>> prune({"properties": {"unity.catalog.managed.delta.defaults.x": "1", "k": "v"}}, set()) + {'properties': {'k': 'v'}} + >>> prune({"changes": {"properties": {"action": "skip", "remote": {"unity.catalog.managed.a.defaults.b": "1"}}, "name": {"action": "update"}}}, set()) + {'changes': {'name': {'action': 'update'}}} + >>> prune([{"metastore_id": "m", "full_name": "c.s"}], {"metastore_id"}) + [{'full_name': 'c.s'}] + """ + if isinstance(node, list): + return [prune(v, drop_fields) for v in node] + if not isinstance(node, dict): + return node + + result = {} + for key, value in node.items(): + if key in drop_fields or managed_re.search(key): + continue + if key == "changes" and isinstance(value, dict): + value = {k: v for k, v in value.items() if not is_managed_change(k, v)} + pruned = prune(value, drop_fields) + # Drop a properties/changes object emptied by managed-key removal so goldens + # don't diverge on {} (cloud that injects) vs absent (cloud that doesn't). + if key in ("properties", "changes") and pruned == {}: + continue + result[key] = pruned + return result + + +def main(): + drop_fields = set(sys.argv[1:]) + data = json.load(sys.stdin) + json.dump(prune(data, drop_fields), sys.stdout, indent=2) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/apps/delete_deleting/app/app.py b/acceptance/bundle/apps/delete_deleting/app/app.py new file mode 100644 index 00000000000..271596cec04 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/app/app.py @@ -0,0 +1 @@ +print("Simple test app") diff --git a/acceptance/bundle/apps/delete_deleting/app/app.yml b/acceptance/bundle/apps/delete_deleting/app/app.yml new file mode 100644 index 00000000000..45b242d4064 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/app/app.yml @@ -0,0 +1 @@ +command: ["python", "app.py"] diff --git a/acceptance/bundle/apps/delete_deleting/databricks_header.yml.tmpl b/acceptance/bundle/apps/delete_deleting/databricks_header.yml.tmpl new file mode 100644 index 00000000000..0198b2c4eb2 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/databricks_header.yml.tmpl @@ -0,0 +1,5 @@ +bundle: + name: app-delete-deleting-$UNIQUE_NAME + +workspace: + root_path: "~/.bundle/app-delete-deleting-$UNIQUE_NAME" diff --git a/acceptance/bundle/apps/delete_deleting/databricks_resources.yml.tmpl b/acceptance/bundle/apps/delete_deleting/databricks_resources.yml.tmpl new file mode 100644 index 00000000000..1fa1d2192a0 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/databricks_resources.yml.tmpl @@ -0,0 +1,5 @@ +resources: + apps: + my_app: + name: app-$UNIQUE_NAME + source_code_path: ./app diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/apps/delete_deleting/output.txt b/acceptance/bundle/apps/delete_deleting/output.txt new file mode 100644 index 00000000000..3b2474e01ad --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/output.txt @@ -0,0 +1,5 @@ + +=== Deploy the app + +=== Apply the stale delete plan against the DELETING app +DELETE_OK diff --git a/acceptance/bundle/apps/delete_deleting/script b/acceptance/bundle/apps/delete_deleting/script new file mode 100644 index 00000000000..14267a32af4 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/script @@ -0,0 +1,38 @@ +#!/bin/bash + +# Regression: a saved delete-plan computed while the app is ACTIVE has Gone=false. +# If the app enters DELETING before the plan is applied (concurrent delete, or an +# interrupted destroy), apply must still succeed instead of failing with the +# backend's "Cannot delete app ... not terminal with state DELETING" 400. + +# databricks.yml is assembled from a shared header plus a resources fragment, so +# the delete config below reuses the exact same bundle/workspace stanza. +envsubst < databricks_header.yml.tmpl > databricks.yml +envsubst < databricks_resources.yml.tmpl >> databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve &> LOG.destroy +} +trap cleanup EXIT + +title "Deploy the app\n" +trace $CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# Save a delete plan while the app is still ACTIVE, so the plan records Gone=false. +# Same header, empty resources -> the app is removed from config. +envsubst < databricks_header.yml.tmpl > databricks.yml +echo "resources: {}" >> databricks.yml + +trace $CLI bundle plan -o json > plan_delete.json 2>LOG.plan_delete.err +cat LOG.plan_delete.err | contains.py '!panic' '!internal error' > /dev/null + +# Out-of-band delete flips the app into DELETING: a second delete now returns 400. +trace $CLI apps delete "app-$UNIQUE_NAME" &> LOG.apps_delete +cat LOG.apps_delete | contains.py '!panic' '!internal error' > /dev/null + +title "Apply the stale delete plan against the DELETING app\n" +trace $CLI bundle deploy --auto-approve --plan plan_delete.json &> LOG.deploy_delete +cat LOG.deploy_delete | contains.py '!panic' '!internal error' > /dev/null + +echo DELETE_OK diff --git a/acceptance/bundle/apps/delete_deleting/test.toml b/acceptance/bundle/apps/delete_deleting/test.toml new file mode 100644 index 00000000000..67577a6d822 --- /dev/null +++ b/acceptance/bundle/apps/delete_deleting/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +# Saved-plan deploy (`--plan`) and the IsGone gone-detection are direct-engine only. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks", "databricks.yml", "plan_delete.json"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml index f784a183258..e90b6d5d1ba 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/test.toml index 1f4bcd29fa8..012e71b09e7 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/test.toml @@ -3,3 +3,7 @@ Ignore = [ 'code.tgz', '.databricks', ] + +# The Terraform provider dropped `code_source_path` from `ai_runtime_task` in +# v1.122.0, so this feature is only exercisable via the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/bundle_tag/id/out.test.toml b/acceptance/bundle/bundle_tag/id/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/bundle_tag/id/out.test.toml +++ b/acceptance/bundle/bundle_tag/id/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/test.toml b/acceptance/bundle/bundle_tag/test.toml index 055fd322c44..8540f9500e6 100644 --- a/acceptance/bundle/bundle_tag/test.toml +++ b/acceptance/bundle/bundle_tag/test.toml @@ -1,7 +1 @@ Badness = "configs with id and url should be rejected" - -# These tests exercise platform-independent bundle logic (tag validation) and behave -# identically on every OS, so on pull requests they run on Linux only to keep the -# windows/macOS CI jobs fast. They still run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/bundle_tag/url/out.test.toml b/acceptance/bundle/bundle_tag/url/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/bundle_tag/url/out.test.toml +++ b/acceptance/bundle/bundle_tag/url/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url_ref/out.test.toml b/acceptance/bundle/bundle_tag/url_ref/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.test.toml +++ b/acceptance/bundle/bundle_tag/url_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/output.txt b/acceptance/bundle/config-remote-sync/select_basic/output.txt index 14987921858..2909fc4e403 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/output.txt +++ b/acceptance/bundle/config-remote-sync/select_basic/output.txt @@ -37,24 +37,45 @@ Resource: resources.jobs.job_two -=== An unknown resource id is rejected +=== A stale selector is skipped, not fatal: job_two still syncs alongside it +Detected changes in 1 resource(s): + +Resource: resources.jobs.job_two + max_concurrent_runs: replace + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -16,5 +16,5 @@ + + job_two: +- max_concurrent_runs: 2 ++ max_concurrent_runs: 10 + tasks: + - task_key: main + +=== An unknown resource id alone is rejected (no match at all must not silently succeed) + >>> [CLI] bundle config-remote-sync --select-ids jobs:no-such-id-123 Error: no deployed jobs resource with id no-such-id-123 Exit code: 1 -=== A selector without a type is rejected ->>> [CLI] bundle config-remote-sync --select-ids no-such-id-123 -Error: invalid --select-ids value "no-such-id-123", expected : (e.g. jobs:[NUMID]) - -Exit code: 1 +=== An id that exists under a different type matches nothing and is rejected too -=== An id that exists under a different type is rejected (no cross-type collision) >>> [CLI] bundle config-remote-sync --select-ids pipelines:[JOB_ONE_ID] Error: no deployed pipelines resource with id [JOB_ONE_ID] Exit code: 1 +=== A selector without a type is still rejected +>>> [CLI] bundle config-remote-sync --select-ids no-such-id-123 +Error: invalid --select-ids value "no-such-id-123", expected : (e.g. jobs:[NUMID]) + +Exit code: 1 + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.job_one diff --git a/acceptance/bundle/config-remote-sync/select_basic/script b/acceptance/bundle/config-remote-sync/select_basic/script index a33b9f9e57c..a85bb0b46a6 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/script +++ b/acceptance/bundle/config-remote-sync/select_basic/script @@ -38,11 +38,20 @@ title "Unfiltered sync still detects the job_two drift (no lost updates)" echo $CLI bundle config-remote-sync -title "An unknown resource id is rejected" -errcode trace $CLI bundle config-remote-sync --select-ids jobs:no-such-id-123 +title "A stale selector is skipped, not fatal: job_two still syncs alongside it" +echo +cp databricks.yml databricks.yml.backup +$CLI bundle config-remote-sync --select-ids "jobs:no-such-id-123,jobs:$job_two_id" --save +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup -title "A selector without a type is rejected" -errcode trace $CLI bundle config-remote-sync --select-ids no-such-id-123 +title "An unknown resource id alone is rejected (no match at all must not silently succeed)" +echo +errcode trace $CLI bundle config-remote-sync --select-ids jobs:no-such-id-123 -title "An id that exists under a different type is rejected (no cross-type collision)" +title "An id that exists under a different type matches nothing and is rejected too" +echo errcode trace $CLI bundle config-remote-sync --select-ids "pipelines:$job_one_id" + +title "A selector without a type is still rejected" +errcode trace $CLI bundle config-remote-sync --select-ids no-such-id-123 diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/skip_permissions/databricks.yml.tmpl new file mode 100644 index 00000000000..74055593627 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/skip_permissions/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# Bundle-level permissions are applied to every resource on deploy but have no +# per-resource YAML node. config-remote-sync must skip the resulting +# permissions sub-resource instead of failing to resolve it. +permissions: + - level: CAN_MANAGE + user_name: ${workspace.current_user.userName} + +resources: + jobs: + my_job: + max_concurrent_runs: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/main + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml new file mode 100644 index 00000000000..579b1e4a3c9 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/output.txt b/acceptance/bundle/config-remote-sync/skip_permissions/output.txt new file mode 100644 index 00000000000..e69b512fdae --- /dev/null +++ b/acceptance/bundle/config-remote-sync/skip_permissions/output.txt @@ -0,0 +1,38 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Add a permission remotely (a change on the permissions sub-resource) + +=== Modify a regular job field remotely + +=== Sync: permissions are skipped, the regular field change is written +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + max_concurrent_runs: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -12,5 +12,5 @@ + jobs: + my_job: +- max_concurrent_runs: 1 ++ max_concurrent_runs: 5 + tasks: + - task_key: main + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/script b/acceptance/bundle/config-remote-sync/skip_permissions/script new file mode 100644 index 00000000000..585db4a5535 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/skip_permissions/script @@ -0,0 +1,34 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +title "Add a permission remotely (a change on the permissions sub-resource)" +echo +# permissions set replaces the whole ACL, so keep the deploy identity as owner +# and add a grantee (the built-in "users" group) that is not in config: this +# makes the permissions sub-resource show a change on the next sync. +$CLI permissions set jobs $job_id --json '{"access_control_list":[{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},{"permission_level":"CAN_VIEW","group_name":"users"}]}' > /dev/null + +title "Modify a regular job field remotely" +echo +edit_resource.py jobs $job_id <>> [CLI] bundle deploy +Uploading immutable bundle snapshot... +Deploying resources... +Updating deployment state... +Deployment complete! +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py + +=== Add permissions and redeploy +>>> [CLI] bundle deploy +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:22:3 + +Uploading immutable bundle snapshot... +Deploying resources... +Updating deployment state... +Deployment complete! +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py + +=== Verify snapshot path changed after permissions change +OK: snapshot path changed as expected + +>>> [CLI] bundle destroy --auto-approve +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:22:3 + +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-perms-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/immutable-permissions-change/script b/acceptance/bundle/deploy/immutable-permissions-change/script new file mode 100644 index 00000000000..458bfdd2c85 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-permissions-change/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy without permissions" +trace $CLI bundle deploy + +JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') +SNAPSHOT_PATH_1=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "First snapshot path: $SNAPSHOT_PATH_1" + +title "Add permissions and redeploy" +cat >> databricks.yml <<'EOF' + +permissions: + - level: CAN_VIEW + group_name: admins +EOF + +trace $CLI bundle deploy + +SNAPSHOT_PATH_2=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "Second snapshot path: $SNAPSHOT_PATH_2" + +title "Verify snapshot path changed after permissions change" +echo "" +if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then + echo "ERROR: snapshot path did not change after permissions were added" + exit 1 +else + echo "OK: snapshot path changed as expected" +fi diff --git a/acceptance/bundle/deploy/immutable-permissions-change/src/main.py b/acceptance/bundle/deploy/immutable-permissions-change/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-permissions-change/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-permissions-change/test.toml b/acceptance/bundle/deploy/immutable-permissions-change/test.toml new file mode 100644 index 00000000000..3139210d0dc --- /dev/null +++ b/acceptance/bundle/deploy/immutable-permissions-change/test.toml @@ -0,0 +1,15 @@ +Local = true +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index f044c983846..f61486ff080 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresCluster = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index 22701b52f25..e90b6d5d1ba 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/script b/acceptance/bundle/deployment/bind/job/engine-from-config/script index ac00fc85564..62c0b6f7c97 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/script +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/script @@ -3,6 +3,8 @@ # bundle.engine is set to "direct" in databricks.yml. # Both deploy and bind should use the direct engine. +unset DATABRICKS_BUNDLE_ENGINE + trace $CLI bundle deploy echo "=== State files after deploy ===" diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/test.toml index 33e6ddaea12..501c5e1294d 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/test.toml @@ -1,6 +1,4 @@ Cloud = false # test leaves deployed job Ignore = [".databricks"] -# Omit DATABRICKS_BUNDLE_ENGINE so bind must read engine from bundle.engine config. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/databricks.yml b/acceptance/bundle/deployment/bind/postgres_database/databricks.yml index ac626e53b24..fa0d7947468 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/databricks.yml +++ b/acceptance/bundle/deployment/bind/postgres_database/databricks.yml @@ -7,3 +7,4 @@ resources: parent: projects/test-project/branches/main database_id: test-database postgres_database: app_db + role: projects/test-project/branches/main/roles/app_owner diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 0612e293703..48203e833cd 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index 7643ac460c8..c7c6f58ed6e 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1,10 +1 @@ Cloud = true - -# These tests exercise platform-independent bundle logic (bind/unbind, deploy/destroy -# sequencing). Some pass Unix-style paths and use MSYS_NO_PATHCONV or a leading "//" -# to stop Git Bash from mangling them on Windows, but that is test scaffolding to make -# the tests portable, not OS-specific product behavior under test (output is identical -# on every OS). So on pull requests they run on Linux only to keep the windows/macOS -# CI jobs fast; they still run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index 22701b52f25..e90b6d5d1ba 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/script b/acceptance/bundle/deployment/unbind/engine-from-config/script index 3b4d5d42f0f..06486dba0e0 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/script +++ b/acceptance/bundle/deployment/unbind/engine-from-config/script @@ -3,6 +3,8 @@ # bundle.engine is set to "direct" in databricks.yml. # Both deploy and unbind should use the direct engine. +unset DATABRICKS_BUNDLE_ENGINE + trace $CLI bundle deploy echo "=== State files after deploy ===" diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/test.toml index 2d349a5bb51..501c5e1294d 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/test.toml @@ -1,6 +1,4 @@ Cloud = false # test leaves deployed job Ignore = [".databricks"] -# Omit DATABRICKS_BUNDLE_ENGINE so unbind must read engine from bundle.engine config. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/force-lock-node-limit/databricks.yml b/acceptance/bundle/destroy/force-lock-node-limit/databricks.yml new file mode 100644 index 00000000000..94d2ba5369e --- /dev/null +++ b/acceptance/bundle/destroy/force-lock-node-limit/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: test-job diff --git a/acceptance/bundle/templates/default-minimal/out.test.toml b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml similarity index 100% rename from acceptance/bundle/templates/default-minimal/out.test.toml rename to acceptance/bundle/destroy/force-lock-node-limit/out.test.toml diff --git a/acceptance/bundle/destroy/force-lock-node-limit/output.txt b/acceptance/bundle/destroy/force-lock-node-limit/output.txt new file mode 100644 index 00000000000..3d44bacdad8 --- /dev/null +++ b/acceptance/bundle/destroy/force-lock-node-limit/output.txt @@ -0,0 +1,31 @@ + +=== Deploy (creates the job and the deployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== destroy without --force-lock (lock write hits the node limit: aborts) +>>> errcode [CLI] bundle destroy --auto-approve +Error: Failed to acquire deployment lock: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +Error: Failed to update, encountered possible permission error: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state +Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state as [USERNAME]. +Please make sure the current user or one of their groups is listed under the permissions of this bundle. +For assistance, contact the owners of this project. +They may need to redeploy the bundle to apply the new permissions. +Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. + + +Exit code: 1 + +=== destroy with --force-lock (tolerates the node limit: proceeds and completes) +>>> [CLI] bundle destroy --force-lock --auto-approve +Warn: Proceeding with destroy without a deployment lock: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script new file mode 100644 index 00000000000..c7fffbc16a2 --- /dev/null +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -0,0 +1,15 @@ +# When the workspace .bundle directory is at its child-node limit, writing the +# deploy.lock file fails with a 403 MAX_CHILD_NODE_SIZE_EXCEEDED. A plain destroy +# aborts on this, but `destroy --force-lock` tolerates it and proceeds lock-less +# so the deployment can still be torn down (which is what frees up capacity). + +title "Deploy (creates the job and the deployment)" +trace $CLI bundle deploy + +title "destroy without --force-lock (lock write hits the node limit: aborts)" +fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +trace errcode $CLI bundle destroy --auto-approve + +title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" +fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/destroy/force-lock-node-limit/test.toml b/acceptance/bundle/destroy/force-lock-node-limit/test.toml new file mode 100644 index 00000000000..bcc3f31c5d0 --- /dev/null +++ b/acceptance/bundle/destroy/force-lock-node-limit/test.toml @@ -0,0 +1 @@ +Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/empty_string_dropped/app/app.py b/acceptance/bundle/empty_string_dropped/app/app.py new file mode 100644 index 00000000000..ad1e64f9fb0 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/app/app.py @@ -0,0 +1,5 @@ +import http.server, os + +http.server.HTTPServer( + ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler +).serve_forever() diff --git a/acceptance/bundle/empty_string_dropped/base.yml b/acceptance/bundle/empty_string_dropped/base.yml new file mode 100644 index 00000000000..16c893adb0b --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/base.yml @@ -0,0 +1,101 @@ +bundle: + name: test-bundle + +# Deployable skeleton per resource. gen_empty_config.py fills every settable +# string leaf under the blocks present here with "" and records it. Coverage is +# scoped to the blocks below: we include one side of each mutually-exclusive +# choice (aws_attributes not gcp, s3 log conf not dbfs/volumes, git_source +# present) because including both would make terraform abort. Only resources that +# deploy on BOTH engines belong here. +# +# Fields set to a real value here are treated as required/meaningful and left +# alone; everything else settable-and-string is emptied. +resources: + apps: + my_app: + # App names are globally unique in the workspace; the two targets deploy to + # the same fake workspace, so scope the name per target to avoid a 409. + name: test-app-${bundle.target} + source_code_path: ./app + + clusters: + my_cluster: + num_workers: 1 + # spark_version is tagged omitempty in the SDK but required by terraform's + # schema and the backend, so it must have a real value or the deploy fails. + spark_version: 13.3.x-scala2.12 + aws_attributes: + availability: SPOT + cluster_log_conf: + s3: + destination: s3://test-bucket/logs + + jobs: + my_job: + max_concurrent_runs: 1 + git_source: + git_url: https://example.com/repo + git_provider: gitHub + git_branch: main + job_clusters: + - job_cluster_key: main + new_cluster: + num_workers: 1 + spark_version: 13.3.x-scala2.12 + aws_attributes: + availability: SPOT + cluster_log_conf: + s3: + destination: s3://test-bucket/logs + + pipelines: + my_pipeline: + libraries: + - file: + path: pipeline.py + clusters: + - label: default + aws_attributes: + availability: SPOT + + sql_warehouses: + my_warehouse: + name: test-warehouse + cluster_size: 2X-Small + auto_stop_mins: 10 + max_num_clusters: 1 + min_num_clusters: 1 + + models: + my_model: + name: test-model + + experiments: + my_experiment: + name: /Users/${workspace.current_user.userName}/test-experiment + + registered_models: + my_registered_model: + name: test-registered-model + catalog_name: main + schema_name: default + + schemas: + my_schema: + catalog_name: main + name: test-schema + + volumes: + my_volume: + name: test-volume + catalog_name: main + schema_name: default + + database_instances: + my_db_instance: + name: test-db-instance + capacity: CU_1 + +targets: + tf: + direct: diff --git a/acceptance/bundle/empty_string_dropped/databricks.yml b/acceptance/bundle/empty_string_dropped/databricks.yml new file mode 100644 index 00000000000..80ff85d9762 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/databricks.yml @@ -0,0 +1,138 @@ +# Generated by gen_empty_config.py. Do not edit; edit base.yml and regenerate. +bundle: + name: test-bundle +resources: + apps: + my_app: + budget_policy_id: '' + description: '' + name: test-app-${bundle.target} + source_code_path: ./app + space: '' + usage_policy_id: '' + clusters: + my_cluster: + aws_attributes: + availability: SPOT + instance_profile_arn: '' + zone_id: '' + cluster_log_conf: + s3: + canned_acl: '' + destination: s3://test-bucket/logs + encryption_type: '' + endpoint: '' + kms_key: '' + region: '' + cluster_name: '' + node_type_id: '' + num_workers: 1 + policy_id: '' + single_user_name: '' + spark_version: 13.3.x-scala2.12 + database_instances: + my_db_instance: + capacity: CU_1 + name: test-db-instance + usage_policy_id: '' + experiments: + my_experiment: + artifact_location: '' + name: /Users/${workspace.current_user.userName}/test-experiment + jobs: + my_job: + budget_policy_id: '' + description: '' + git_source: + git_branch: main + git_provider: gitHub + git_url: https://example.com/repo + job_clusters: + - job_cluster_key: main + new_cluster: + aws_attributes: + availability: SPOT + instance_profile_arn: '' + zone_id: '' + cluster_log_conf: + s3: + canned_acl: '' + destination: s3://test-bucket/logs + encryption_type: '' + endpoint: '' + kms_key: '' + region: '' + cluster_name: '' + node_type_id: '' + num_workers: 1 + policy_id: '' + single_user_name: '' + spark_version: 13.3.x-scala2.12 + max_concurrent_runs: 1 + name: '' + parent_path: '' + usage_policy_id: '' + models: + my_model: + description: '' + name: test-model + pipelines: + my_pipeline: + budget_policy_id: '' + clusters: + - aws_attributes: + availability: SPOT + instance_profile_arn: '' + zone_id: '' + driver_instance_pool_id: '' + driver_node_type_id: '' + instance_pool_id: '' + label: default + node_type_id: '' + policy_id: '' + libraries: + - file: + path: pipeline.py + jar: '' + whl: '' + name: '' + root_path: '' + serverless_compute_id: '' + usage_policy_id: '' + registered_models: + my_registered_model: + catalog_name: main + comment: '' + created_by: '' + full_name: '' + metastore_id: '' + name: test-registered-model + owner: '' + schema_name: default + storage_location: '' + updated_by: '' + schemas: + my_schema: + catalog_name: main + comment: '' + name: test-schema + storage_root: '' + sql_warehouses: + my_warehouse: + auto_stop_mins: 10 + cluster_size: 2X-Small + instance_profile_arn: '' + max_num_clusters: 1 + min_num_clusters: 1 + name: test-warehouse + volumes: + my_volume: + catalog_name: main + comment: '' + name: test-volume + schema_name: default + storage_location: '' + volume_path: '' +targets: + direct: null + tf: null diff --git a/acceptance/bundle/empty_string_dropped/empty_sent.py b/acceptance/bundle/empty_string_dropped/empty_sent.py new file mode 100755 index 00000000000..0a8f74a698b --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/empty_sent.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Classify empty-string handling by diffing terraform vs direct create requests. + +Both engines deploy the same config, so their create requests have the same +shape. From the request logs we collect, per engine, the body fields sent with an +empty-string value; comparing the two sets splits three ways: + + out.empty_sent_by_terraform_only.txt sent "" by terraform, not by direct + out.empty_sent_by_direct_only.txt sent "" by direct, not by terraform + out.empty_sent_by_both.txt sent "" by both + +The fourth bucket -- fields set to "" in config that BOTH engines dropped -- is +not visible in the request logs (they're absent from both bodies), so it is +recovered from databricks.yml: + + out.empty_dropped_by_both.txt set "" in config, sent by neither + +Terraform drops empty optional strings via omitempty (SDKv2 resources), so today +direct_only lists the gap; both/terraform_only capture the exceptions (a required +field with no omitempty, or a plugin-framework resource that serializes ""). +""" + +import json +from pathlib import Path + +# Substring uniquely identifying each resource type's create-request path, used +# to attribute a config field to the request it should appear in. +CREATE_PATHS = { + "apps": "/apps", + "clusters": "clusters/create", + "jobs": "jobs/create", + "pipelines": "/pipelines", + "sql_warehouses": "/warehouses", + "models": "/mlflow/registered-models", + "experiments": "/experiments/create", + "registered_models": "/unity-catalog/models", + "schemas": "/unity-catalog/schemas", + "volumes": "/unity-catalog/volumes", + "database_instances": "/database/instances", +} + + +def read_json_many(text): + """Decode concatenated JSON objects (the out.requests..json format).""" + decoder = json.JSONDecoder() + objects = [] + pos = 0 + while pos < len(text): + while pos < len(text) and text[pos].isspace(): + pos += 1 + if pos >= len(text): + break + obj, pos = decoder.raw_decode(text, pos) + objects.append(obj) + return objects + + +def empty_field_paths(obj, prefix): + """Yield dotted field paths whose value is an empty string.""" + if isinstance(obj, dict): + for key, value in obj.items(): + child = f"{prefix}.{key}" if prefix else key + if value == "": + yield child + else: + yield from empty_field_paths(value, child) + elif isinstance(obj, list): + for item in obj: + yield from empty_field_paths(item, prefix) + + +def empty_sent(engine): + """Return {" "} for body fields sent with value "".""" + out = set() + for req in read_json_many(Path(f"out.requests.{engine}.json").read_text()): + for field in empty_field_paths(req.get("body", {}), ""): + out.add(f"{req.get('path', '')} {field}") + return out + + +def config_empty_fields(): + """Return {" "} for every "" leaf in databricks.yml. + + Uses a minimal indentation parser (the harness python is stdlib-only, no + PyYAML). databricks.yml is generator output with a fixed 2-space style, block + mappings, and "- " list items -- not arbitrary YAML. + """ + out = set() + # stack of (indent, key) for the current mapping path within a resource + stack = [] + for raw in Path("databricks.yml").read_text().splitlines(): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip(" ")) + line = raw.strip() + # A "- " list item shares its parent's field path; treat the inline + # "key: value" after it at the item indent. + if line.startswith("- "): + line = line[2:] + indent += 2 + if ":" not in line: + continue + key, _, value = line.partition(":") + key, value = key.strip(), value.strip() + while stack and stack[-1][0] >= indent: + stack.pop() + path = [k for _, k in stack] + [key] + # resources...: the create path is fixed by rtype. + if len(path) >= 2 and path[0] == "resources" and path[1] in CREATE_PATHS: + if value in ("''", '""'): + field = ".".join(path[3:]) # drop resources.. + out.add(f"{CREATE_PATHS[path[1]]} {field}") + if not value: + stack.append((indent, key)) + return out + + +def sent_fields(*engines): + """Return {" "} for ALL body fields (any value) sent.""" + out = set() + for engine in engines: + for req in read_json_many(Path(f"out.requests.{engine}.json").read_text()): + for field in all_field_paths(req.get("body", {}), ""): + out.add(f"{req.get('path', '')} {field}") + return out + + +def all_field_paths(obj, prefix): + """Yield every dotted field path present in a body, regardless of value.""" + if isinstance(obj, dict): + for key, value in obj.items(): + child = f"{prefix}.{key}" if prefix else key + yield child + yield from all_field_paths(value, child) + elif isinstance(obj, list): + for item in obj: + yield from all_field_paths(item, prefix) + + +def sent_somewhere(entry, sent): + """True if entry (create-path substring + field) matches any sent full-path entry.""" + create_path, field = entry.split(" ", 1) + return any(create_path in path and field == f for path, f in (s.split(" ", 1) for s in sent)) + + +def main(): + tf = empty_sent("terraform") + direct = empty_sent("direct") + + _write("out.empty_sent_by_terraform_only.txt", tf - direct) + _write("out.empty_sent_by_direct_only.txt", direct - tf) + _write("out.empty_sent_by_both.txt", tf & direct) + + # Dropped by both: set to "" in config but present in neither request body + # (present with any value = not dropped, so match on field presence). + present = sent_fields("terraform", "direct") + dropped_by_both = {e for e in config_empty_fields() if not sent_somewhere(e, present)} + _write("out.empty_dropped_by_both.txt", dropped_by_both) + + +def _write(name, entries): + Path(name).write_text("".join(entry + "\n" for entry in sorted(entries))) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/empty_string_dropped/gen_empty_config.py b/acceptance/bundle/empty_string_dropped/gen_empty_config.py new file mode 100644 index 00000000000..73d360d1e8b --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/gen_empty_config.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Generate databricks.yml with every settable string field set to "". + +base.yml defines a deployable skeleton per resource, including whichever nested +blocks we want to cover (job_clusters[*].new_cluster, git_source, one cloud +attributes block, ...). This script walks every resource subtree and sets every +settable string LEAF (including nested ones) that the base doesn't already set +to "". Coverage is therefore scoped to the blocks present in base.yml, which is +how we avoid mutually-exclusive blocks (aws vs gcp attributes, git_source vs +none) that would make terraform abort. + +A field is settable/eligible when out.fields.txt types it as `string` (this +skips enums, which are named types) with flag ALL or INPUT, and it is not +output_only (resources.generated.yml), a bundle-framework field, or a known +terraform-erroring field. + +Run from the repo root; writes databricks.yml in the test directory: + acceptance/bundle/empty_string_dropped/gen_empty_config.py +""" + +import re +from pathlib import Path + +import yaml + +FIELDS = Path("acceptance/bundle/refschema/out.fields.txt") +GENERATED = Path("bundle/direct/dresources/resources.generated.yml") +TESTDIR = Path("acceptance/bundle/empty_string_dropped") +BASE = TESTDIR / "base.yml" + +# Bundle-framework fields present on every resource; not real API inputs. +FRAMEWORK_FIELDS = {"id", "url", "modified_status"} + +# Leaf field names that make terraform error at plan time (ConflictsWith, enum +# validation on string-typed fields, computed attributes), so no request is +# recorded. Applied per resource type. Matched by leaf name at any depth. +TERRAFORM_ERRORS = { + "clusters": {"instance_pool_id", "driver_instance_pool_id", "driver_node_type_id"}, + "jobs": { + "instance_pool_id", + "driver_instance_pool_id", + "driver_node_type_id", + # git_source: branch/commit/tag are mutually exclusive; base sets branch. + "git_commit", + "git_tag", + }, + "pipelines": {"channel", "edition", "catalog", "storage", "schema", "target"}, + "sql_warehouses": {"creator_name"}, +} + + +def string_leaf_parents(): + """Map resource type -> {parent_schema_path: {leaf field names}}. + + Schema paths mirror out.fields.txt: dotted segments, list elements marked + with "[*]" (e.g. "job_clusters[*].new_cluster"). Only string leaves that are + real object fields are kept: names ending in "[*]" (string-array elements) or + "*" (map values) are skipped. + """ + pat = re.compile(r"^resources\.([a-z_]+)\.\*\.(.+)$") + result = {} + for line in FIELDS.read_text().splitlines(): + parts = line.split("\t") + if len(parts) < 3: + continue + path, typ, flag = parts[0], parts[1], parts[2] + if typ != "string" or flag not in ("ALL", "INPUT"): + continue + m = pat.match(path) + if not m: + continue + rtype, rel = m.group(1), m.group(2) + if rel.endswith(("[*]", ".*")) or rel == "*": + continue + parent, _, leaf = rel.rpartition(".") + result.setdefault(rtype, {}).setdefault(parent, set()).add(leaf) + return result + + +def output_only_fields(): + """Map resource type -> {output_only field paths} (user cannot set).""" + gen = yaml.safe_load(GENERATED.read_text()) or {} + result = {} + for rtype, spec in (gen.get("resources") or {}).items(): + fields = set() + for entry in (spec or {}).get("ignore_remote_changes") or []: + if str(entry.get("reason", "")).startswith("spec:output_only"): + fields.add(entry["field"]) + result[rtype] = fields + return result + + +def fill(node, schema_path, parents, excluded): + """Recursively set settable string leaves under node to ""; recurse into blocks. + + schema_path uses out.fields.txt syntax (list elements marked "[*]") to look up + eligible leaves for the current object. + """ + if isinstance(node, dict): + for leaf in parents.get(schema_path, set()): + if leaf not in node and leaf not in excluded: + node[leaf] = "" + for key, value in list(node.items()): + child = f"{schema_path}.{key}" if schema_path else key + fill(value, child, parents, excluded) + elif isinstance(node, list): + for item in node: + fill(item, schema_path + "[*]", parents, excluded) + + +def main(): + parents = string_leaf_parents() + output_only = output_only_fields() + base = yaml.safe_load(BASE.read_text()) + + for rtype, entries in sorted(base.get("resources", {}).items()): + excluded = output_only.get(rtype, set()) | FRAMEWORK_FIELDS | TERRAFORM_ERRORS.get(rtype, set()) + for cfg in entries.values(): + fill(cfg, "", parents.get(rtype, {}), excluded) + + (TESTDIR / "databricks.yml").write_text( + "# Generated by gen_empty_config.py. Do not edit; edit base.yml and regenerate.\n" + + yaml.dump(base, sort_keys=True, default_flow_style=False) + ) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/empty_string_dropped/out.empty_dropped_by_both.txt b/acceptance/bundle/empty_string_dropped/out.empty_dropped_by_both.txt new file mode 100644 index 00000000000..1e5d5eb2a6b --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.empty_dropped_by_both.txt @@ -0,0 +1,58 @@ +/apps budget_policy_id +/apps space +/apps usage_policy_id +/database/instances usage_policy_id +/experiments/create artifact_location +/mlflow/registered-models description +/pipelines budget_policy_id +/pipelines clusters.aws_attributes.instance_profile_arn +/pipelines clusters.aws_attributes.zone_id +/pipelines clusters.driver_instance_pool_id +/pipelines clusters.driver_node_type_id +/pipelines clusters.instance_pool_id +/pipelines clusters.node_type_id +/pipelines clusters.policy_id +/pipelines libraries.jar +/pipelines libraries.whl +/pipelines name +/pipelines serverless_compute_id +/pipelines usage_policy_id +/unity-catalog/models comment +/unity-catalog/models created_by +/unity-catalog/models full_name +/unity-catalog/models metastore_id +/unity-catalog/models owner +/unity-catalog/models storage_location +/unity-catalog/models updated_by +/unity-catalog/schemas comment +/unity-catalog/schemas storage_root +/unity-catalog/volumes comment +/unity-catalog/volumes storage_location +/unity-catalog/volumes volume_path +/warehouses instance_profile_arn +clusters/create aws_attributes.instance_profile_arn +clusters/create aws_attributes.zone_id +clusters/create cluster_log_conf.s3.canned_acl +clusters/create cluster_log_conf.s3.encryption_type +clusters/create cluster_log_conf.s3.endpoint +clusters/create cluster_log_conf.s3.kms_key +clusters/create cluster_log_conf.s3.region +clusters/create cluster_name +clusters/create node_type_id +clusters/create policy_id +clusters/create single_user_name +jobs/create budget_policy_id +jobs/create description +jobs/create job_clusters.new_cluster.aws_attributes.instance_profile_arn +jobs/create job_clusters.new_cluster.aws_attributes.zone_id +jobs/create job_clusters.new_cluster.cluster_log_conf.s3.canned_acl +jobs/create job_clusters.new_cluster.cluster_log_conf.s3.encryption_type +jobs/create job_clusters.new_cluster.cluster_log_conf.s3.endpoint +jobs/create job_clusters.new_cluster.cluster_log_conf.s3.kms_key +jobs/create job_clusters.new_cluster.cluster_log_conf.s3.region +jobs/create job_clusters.new_cluster.cluster_name +jobs/create job_clusters.new_cluster.node_type_id +jobs/create job_clusters.new_cluster.policy_id +jobs/create job_clusters.new_cluster.single_user_name +jobs/create parent_path +jobs/create usage_policy_id diff --git a/acceptance/bundle/empty_string_dropped/out.empty_sent_by_both.txt b/acceptance/bundle/empty_string_dropped/out.empty_sent_by_both.txt new file mode 100644 index 00000000000..ea9330c71cd --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.empty_sent_by_both.txt @@ -0,0 +1 @@ +/api/2.0/apps description diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy1.requests.terraform.json b/acceptance/bundle/empty_string_dropped/out.empty_sent_by_direct_only.txt similarity index 100% rename from acceptance/bundle/resources/grants/schemas/empty_array/out.deploy1.requests.terraform.json rename to acceptance/bundle/empty_string_dropped/out.empty_sent_by_direct_only.txt diff --git a/acceptance/bundle/empty_string_dropped/out.empty_sent_by_terraform_only.txt b/acceptance/bundle/empty_string_dropped/out.empty_sent_by_terraform_only.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/empty_string_dropped/out.requests.direct.json b/acceptance/bundle/empty_string_dropped/out.requests.direct.json new file mode 100644 index 00000000000..c9bba29b9fa --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.requests.direct.json @@ -0,0 +1,156 @@ +{ + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + }, + "body": { + "description": "", + "name": "test-app-direct" + } +} +{ + "method": "POST", + "path": "/api/2.0/database/instances", + "body": { + "capacity": "CU_1", + "name": "test-db-instance" + } +} +{ + "method": "POST", + "path": "/api/2.0/mlflow/experiments/create", + "body": { + "name": "/Users/[USERNAME]/test-experiment" + } +} +{ + "method": "POST", + "path": "/api/2.0/mlflow/registered-models/create", + "body": { + "name": "test-model" + } +} +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "clusters": [ + { + "aws_attributes": { + "availability": "SPOT" + }, + "label": "default" + } + ], + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files/pipeline.py" + } + } + ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files" + } +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses", + "body": { + "auto_stop_mins": 10, + "cluster_size": "2X-Small", + "enable_photon": true, + "max_num_clusters": 1, + "min_num_clusters": 1, + "name": "test-warehouse", + "spot_instance_policy": "COST_OPTIMIZED" + } +} +{ + "method": "POST", + "path": "/api/2.1/clusters/create", + "body": { + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "test-registered-model", + "schema_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "test-volume", + "schema_name": "default", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "git_source": { + "git_branch": "main", + "git_provider": "gitHub", + "git_url": "https://example.com/repo" + }, + "job_clusters": [ + { + "job_cluster_key": "main", + "new_cluster": { + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/empty_string_dropped/out.requests.terraform.json b/acceptance/bundle/empty_string_dropped/out.requests.terraform.json new file mode 100644 index 00000000000..b2e9c328cd2 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.requests.terraform.json @@ -0,0 +1,157 @@ +{ + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + }, + "body": { + "description": "", + "name": "test-app-tf" + } +} +{ + "method": "POST", + "path": "/api/2.0/database/instances", + "body": { + "capacity": "CU_1", + "name": "test-db-instance" + } +} +{ + "method": "POST", + "path": "/api/2.0/mlflow/experiments/create", + "body": { + "name": "/Users/[USERNAME]/test-experiment" + } +} +{ + "method": "POST", + "path": "/api/2.0/mlflow/registered-models/create", + "body": { + "name": "test-model" + } +} +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "clusters": [ + { + "aws_attributes": { + "availability": "SPOT" + }, + "label": "default" + } + ], + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/tf/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/tf/files/pipeline.py" + } + } + ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/tf/files" + } +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses", + "body": { + "auto_stop_mins": 10, + "cluster_size": "2X-Small", + "enable_photon": true, + "max_num_clusters": 1, + "min_num_clusters": 1, + "name": "test-warehouse", + "spot_instance_policy": "COST_OPTIMIZED" + } +} +{ + "method": "POST", + "path": "/api/2.1/clusters/create", + "body": { + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "test-registered-model", + "schema_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "test-volume", + "schema_name": "default", + "volume_type": "MANAGED" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/tf/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "git_source": { + "git_branch": "main", + "git_provider": "gitHub", + "git_url": "https://example.com/repo" + }, + "job_clusters": [ + { + "job_cluster_key": "main", + "new_cluster": { + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "name": "Untitled", + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/empty_string_dropped/out.test.toml b/acceptance/bundle/empty_string_dropped/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/empty_string_dropped/out.validate.json b/acceptance/bundle/empty_string_dropped/out.validate.json new file mode 100644 index 00000000000..e627b2a428f --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/out.validate.json @@ -0,0 +1,135 @@ +{ + "apps": { + "my_app": { + "description": "", + "name": "test-app-direct", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files/app" + } + }, + "clusters": { + "my_cluster": { + "autotermination_minutes": 60, + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } + }, + "database_instances": { + "my_db_instance": { + "capacity": "CU_1", + "name": "test-db-instance" + } + }, + "experiments": { + "my_experiment": { + "name": "/Users/[USERNAME]/test-experiment" + } + }, + "jobs": { + "my_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "git_source": { + "git_branch": "main", + "git_provider": "gitHub", + "git_url": "https://example.com/repo" + }, + "job_clusters": [ + { + "job_cluster_key": "main", + "new_cluster": { + "aws_attributes": { + "availability": "SPOT" + }, + "cluster_log_conf": { + "s3": { + "destination": "s3://test-bucket/logs" + } + }, + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "queue": { + "enabled": true + } + } + }, + "models": { + "my_model": { + "name": "test-model" + } + }, + "pipelines": { + "my_pipeline": { + "channel": "CURRENT", + "clusters": [ + { + "aws_attributes": { + "availability": "SPOT" + }, + "label": "default" + } + ], + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files/pipeline.py" + } + } + ], + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files" + } + }, + "registered_models": { + "my_registered_model": { + "catalog_name": "main", + "name": "test-registered-model", + "schema_name": "default" + } + }, + "schemas": { + "my_schema": { + "catalog_name": "main", + "name": "test-schema" + } + }, + "sql_warehouses": { + "my_warehouse": { + "auto_stop_mins": 10, + "cluster_size": "2X-Small", + "enable_photon": true, + "max_num_clusters": 1, + "min_num_clusters": 1, + "name": "test-warehouse", + "spot_instance_policy": "COST_OPTIMIZED" + } + }, + "volumes": { + "my_volume": { + "catalog_name": "main", + "name": "test-volume", + "schema_name": "default", + "volume_path": "/Volumes/main/default/test-volume", + "volume_type": "MANAGED" + } + } +} diff --git a/acceptance/bundle/empty_string_dropped/output.txt b/acceptance/bundle/empty_string_dropped/output.txt new file mode 100644 index 00000000000..5de16a505a7 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/output.txt @@ -0,0 +1,12 @@ + +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy -t tf +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/tf/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -t direct +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/empty_string_dropped/pipeline.py b/acceptance/bundle/empty_string_dropped/pipeline.py new file mode 100644 index 00000000000..e1b4968c112 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/pipeline.py @@ -0,0 +1,6 @@ +import dlt + + +@dlt.table +def my_table(): + return spark.range(1) diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script new file mode 100644 index 00000000000..973e436617c --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/script @@ -0,0 +1,20 @@ +# databricks.yml is committed, generated by gen_empty_config.py with every +# settable string field set to "". Deploy through each engine (separate target => +# separate state), record create requests, and empty_sent.py classifies which +# "" fields each engine sends. +# +# Baseline: terraform drops "" via omitempty, direct sends it. As direct handling +# changes, out.empty_sent_by_direct_only.txt (the gap) shrinks. + +# Resolved config the engines see. Today every "" field is still present here; a +# fix in the initialize phase would drop them, and this golden would show that. +$CLI bundle validate -o json -t direct | jq .resources > out.validate.json + +# Exclude non-create traffic: workspace file ops and telemetry (nondeterministic). +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t tf +print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.terraform.json + +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -t direct +print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.direct.json + +$TESTDIR/empty_sent.py diff --git a/acceptance/bundle/empty_string_dropped/test.toml b/acceptance/bundle/empty_string_dropped/test.toml new file mode 100644 index 00000000000..51e7bc13e23 --- /dev/null +++ b/acceptance/bundle/empty_string_dropped/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = false +RecordRequests = true + +# This test drives both engines itself (one bundle per engine via a per-deploy +# DATABRICKS_BUNDLE_ENGINE), so pin the matrix to a single value to run once. +# The value only selects the CI runner; the script sets the engine explicitly. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/empty_string_variable/databricks.yml b/acceptance/bundle/empty_string_variable/databricks.yml new file mode 100644 index 00000000000..88e38c86cf3 --- /dev/null +++ b/acceptance/bundle/empty_string_variable/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +variables: + policy: + default: "" + +resources: + clusters: + my_cluster: + num_workers: 1 + spark_version: 13.3.x-scala2.12 + # The original bug: a variable resolves to "" and policy_id (omitempty) was + # force-sent, so the backend rejected it with "'' is not a valid cluster + # policy ID". It must now be dropped from the resolved config. + policy_id: ${var.policy} diff --git a/acceptance/bundle/empty_string_variable/out.test.toml b/acceptance/bundle/empty_string_variable/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/empty_string_variable/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/empty_string_variable/output.txt b/acceptance/bundle/empty_string_variable/output.txt new file mode 100644 index 00000000000..8b89eda8902 --- /dev/null +++ b/acceptance/bundle/empty_string_variable/output.txt @@ -0,0 +1,7 @@ + +>>> [CLI] bundle validate -o json +{ + "spark_version": "13.3.x-scala2.12", + "policy_id": null, + "has_policy_id": false +} diff --git a/acceptance/bundle/empty_string_variable/script b/acceptance/bundle/empty_string_variable/script new file mode 100644 index 00000000000..73a83e9bcce --- /dev/null +++ b/acceptance/bundle/empty_string_variable/script @@ -0,0 +1,3 @@ +# policy_id resolves to "" via the variable. It must be absent from the resolved +# config (dropped as an empty omitempty string), not present as "". +trace $CLI bundle validate -o json | jq '.resources.clusters.my_cluster | {spark_version, policy_id, has_policy_id: has("policy_id")}' diff --git a/acceptance/bundle/empty_string_variable/test.toml b/acceptance/bundle/empty_string_variable/test.toml new file mode 100644 index 00000000000..6aab6ad6906 --- /dev/null +++ b/acceptance/bundle/empty_string_variable/test.toml @@ -0,0 +1,2 @@ +# validate does not deploy, so engine choice is irrelevant: run once on direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index 41d972a0f5b..880431d5a9f 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index 41d972a0f5b..880431d5a9f 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index 44fd4e4695c..880431d5a9f 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = true -GOOSOnPR.windows = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/test.toml b/acceptance/bundle/integration_whl/interactive_cluster/test.toml deleted file mode 100644 index 2acda27ad02..00000000000 --- a/acceptance/bundle/integration_whl/interactive_cluster/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Re-enable on pull requests (the parent integration_whl/ dir opts out): kept for -# symmetry with the serverless variants so both cluster modes exercise the wheel -# deploy path (uv venv activation) on windows/macOS for PRs. -GOOSOnPR.windows = true -GOOSOnPR.darwin = true diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index ce0a18667d5..19068d43e0a 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = true -GOOSOnPR.windows = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml index 466e4c2e0a5..35f395f3c48 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml @@ -1,9 +1,3 @@ -# Re-enable on pull requests (the parent integration_whl/ dir opts out): this variant -# builds a wheel (bdist_wheel) on the interactive/classic-cluster path, exercising the -# OS-specific venv build, so it must keep running on windows/macOS for PRs. -GOOSOnPR.windows = true -GOOSOnPR.darwin = true - [EnvMatrix] DATA_SECURITY_MODE = [ "USER_ISOLATION", diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index 41d972a0f5b..880431d5a9f 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless/out.test.toml b/acceptance/bundle/integration_whl/serverless/out.test.toml index 3ced9523d4d..68b04957a4c 100644 --- a/acceptance/bundle/integration_whl/serverless/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = true -GOOSOnPR.windows = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless/test.toml b/acceptance/bundle/integration_whl/serverless/test.toml index 10a9d12a5f3..565f411a975 100644 --- a/acceptance/bundle/integration_whl/serverless/test.toml +++ b/acceptance/bundle/integration_whl/serverless/test.toml @@ -4,12 +4,6 @@ Local = true # serverless is only enabled if UC is enabled RequiresUnityCatalog = true -# Re-enable on pull requests (the parent integration_whl/ dir opts out): this variant -# builds a wheel (bdist_wheel) on the serverless path, exercising the OS-specific venv -# build, so it must keep running on windows/macOS for PRs. -GOOSOnPR.windows = true -GOOSOnPR.darwin = true - [[Repls]] Old = '\\' New = '/' diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml index 739d8364544..68b04957a4c 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml index 3ced9523d4d..68b04957a4c 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = true -GOOSOnPR.windows = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml index 136290f579c..01576446a6b 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml @@ -1,12 +1,6 @@ # serverless is only enabled if UC is enabled RequiresUnityCatalog = true -# Re-enable on pull requests (the parent integration_whl/ dir opts out): this variant -# builds a wheel (bdist_wheel) on the serverless path, exercising the OS-specific venv -# build, so it must keep running on windows/macOS for PRs. -GOOSOnPR.windows = true -GOOSOnPR.darwin = true - [[Repls]] Old = '\\' New = '/' diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index b562d1b0b1a..d97c489629e 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -1,16 +1,6 @@ Local = true CloudSlow = true -# The OS-specific surface here is the wheel build path (uv venv layout: .venv/Scripts -# vs .venv/bin, and the python3->python alias; see venv_activate in -# acceptance/script.prepare). That path is identical across every variant, so on pull -# requests we run only a couple of representative building variants on windows/macOS -# (serverless and interactive_cluster_dynamic_version re-enable themselves) and skip -# the rest, which vary only OS-independent bundle logic. All still run on every OS on -# push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false - # Workspace file system does not allow initializing python envs on it. # I suspect something about the default python env on DBR interferes with it. # We'll likely need a first class venv abstraction in acceptance tests to fix this. diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index f3f88ba26ea..44a1a2186a1 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.aws = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index f3f88ba26ea..44a1a2186a1 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.aws = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl new file mode 100644 index 00000000000..4a8d6bc5afd --- /dev/null +++ b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + instance_pools: + foo: + instance_pool_name: test-instance-pool-$UNIQUE_NAME + node_type_id: $NODE_TYPE_ID + permissions: + - level: CAN_ATTACH_TO + group_name: users diff --git a/acceptance/bundle/invariant/configs/schema_empty_grants.yml.tmpl b/acceptance/bundle/invariant/configs/schema_empty_grants.yml.tmpl new file mode 100644 index 00000000000..70dd05dd74c --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_empty_grants.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + foo: + catalog_name: main + name: test-schema-$UNIQUE_NAME + # Empty grants: Terraform records no grants resource for an empty list, so + # migrate leaves no state entry and plan must not show a spurious create. + grants: [] diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 8fd536c2b85..6e5a2799ae6 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -36,6 +34,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/script b/acceptance/bundle/invariant/continue_293/script index c1c8d758eeb..2a0e94d9719 100644 --- a/acceptance/bundle/invariant/continue_293/script +++ b/acceptance/bundle/invariant/continue_293/script @@ -1,41 +1,17 @@ # Invariant to test: current CLI can deploy on top of state produced by v0.293.0 -cp -r "$TESTDIR/../data/." . &> LOG.cp - -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init -fi - -envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml - -cleanup() { - $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT +invariant_setup # Deploy with old CLI to produce v0.293.0 state trace $CLI_293 --version -$CLI_293 bundle deploy &> LOG.deploy.293 -cat LOG.deploy.293 | contains.py '!panic' '!internal error' > /dev/null - -echo INPUT_CONFIG_OK +invariant_deploy LOG.deploy.293 $CLI_293 bundle deploy # Deploy with current CLI on top of old state $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null +cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null # Verify no drift after current CLI deploy -$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err -cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null -verify_no_drift.py LOG.planjson +invariant_verify_no_drift -$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan -cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null +$CLI bundle plan 2>LOG.plan.err | contains.py '!panic:' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan +cat LOG.plan.err | contains.py '!panic:' '!internal error' > /dev/null diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 902316bea12..c6fba9c43fb 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -13,6 +13,9 @@ EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml # genie_spaces resource is not supported on v0.293.0 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] +# instance_pools resource is not supported on v0.293.0 +EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] + # job_runs resource is not supported on v0.293.0 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml new file mode 100644 index 00000000000..a3719d00269 --- /dev/null +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -0,0 +1,59 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/delete_idempotent/output.txt b/acceptance/bundle/invariant/delete_idempotent/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/delete_idempotent/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/delete_idempotent/script b/acceptance/bundle/invariant/delete_idempotent/script new file mode 100644 index 00000000000..aba945dbb2e --- /dev/null +++ b/acceptance/bundle/invariant/delete_idempotent/script @@ -0,0 +1,61 @@ +# Invariant to test: deleting resources by removing them from config is idempotent. +# After a delete succeeds, restoring the pre-delete state and running deploy again +# must succeed even though the resources are already gone on the server. + +invariant_setup + +# All configs use `test-bundle-$UNIQUE_NAME`; workspace root/state paths follow +# the default `~/.bundle//` template with target=default. +bundle_name=test-bundle-$UNIQUE_NAME +STATE_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default/state + +# Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the +# saved-plan deploy path; otherwise `bundle deploy` plans on its own. +if [[ -n "$READPLAN" ]]; then + $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err + cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null +fi + +invariant_deploy LOG.deploy_initial $CLI bundle deploy $(readplanarg plan_initial.json) + +# Snapshot the deploy state before we delete anything. +cp -r .databricks .databricks.backup + +# Replace the config with an empty resources map so the next deploy issues deletes. +cat > databricks.yml < plan_delete.json 2>LOG.plan_delete.err + cat LOG.plan_delete.err | contains.py '!panic:' '!internal error' > /dev/null +fi + +trace $CLI bundle deploy --auto-approve $(readplanarg plan_delete.json) &> LOG.deploy_delete +cat LOG.deploy_delete | contains.py '!panic:' '!internal error' > /dev/null + +# Restore the pre-delete state; the resources are already gone from the server, +# so the next delete-deploy must handle "resource already gone" idempotently. +rm -rf .databricks +mv .databricks.backup .databricks + +# Wipe the workspace state file the delete-deploy just wrote. With it present, +# AlwaysPull=true would treat remote as newer and overwrite our restored local +# state; nuking it leaves local as the sole state source, so the next plan +# genuinely re-issues deletes for resources the server has already removed. +# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash +# workspace path into C:/Program Files/Git/...; it is scoped to this command so +# the Python helpers (which need their own paths converted) are unaffected. +MSYS_NO_PATHCONV=1 $CLI workspace delete --recursive "$STATE_PATH" &> LOG.wipe_remote_state +cat LOG.wipe_remote_state | contains.py '!panic:' '!internal error' > /dev/null + +if [[ -n "$READPLAN" ]]; then + $CLI bundle plan -o json > plan_delete2.json 2>LOG.plan_delete2.err + cat LOG.plan_delete2.err | contains.py '!panic:' '!internal error' > /dev/null +fi + +trace $CLI bundle deploy --auto-approve $(readplanarg plan_delete2.json) &> LOG.deploy_delete2 +cat LOG.deploy_delete2 | contains.py '!panic:' '!internal error' > /dev/null diff --git a/acceptance/bundle/invariant/delete_idempotent/test.toml b/acceptance/bundle/invariant/delete_idempotent/test.toml new file mode 100644 index 00000000000..3f5bb92afad --- /dev/null +++ b/acceptance/bundle/invariant/delete_idempotent/test.toml @@ -0,0 +1,5 @@ +EnvMatrix.READPLAN = ["", "1"] + +# Snapshot of pre-delete state used to re-run the delete on state that still +# references the (now-gone) resources; may linger if the test fails mid-run. +Ignore = [".databricks.backup"] diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml new file mode 100644 index 00000000000..a3719d00269 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -0,0 +1,59 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/destroy_idempotent/output.txt b/acceptance/bundle/invariant/destroy_idempotent/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_idempotent/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/destroy_idempotent/script b/acceptance/bundle/invariant/destroy_idempotent/script new file mode 100644 index 00000000000..0467b23330e --- /dev/null +++ b/acceptance/bundle/invariant/destroy_idempotent/script @@ -0,0 +1,41 @@ +# Invariant to test: bundle destroy is idempotent. +# After destroy succeeds, restoring the pre-destroy state and running destroy again +# must succeed even though the resources are already gone on the server. + +invariant_setup + +# All configs use `test-bundle-$UNIQUE_NAME`; workspace root path follows the +# default `~/.bundle//` template with target=default. +bundle_name=test-bundle-$UNIQUE_NAME +ROOT_PATH=/Workspace/Users/$CURRENT_USER_NAME/.bundle/$bundle_name/default + +# Initial deploy of the resources. Only pre-plan when READPLAN=1 exercises the +# saved-plan deploy path; otherwise `bundle deploy` plans on its own. +if [[ -n "$READPLAN" ]]; then + $CLI bundle plan -o json > plan_initial.json 2>LOG.plan_initial.err + cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null +fi + +invariant_deploy LOG.deploy_initial $CLI bundle deploy $(readplanarg plan_initial.json) + +# Snapshot the deploy state before we destroy anything. +cp -r .databricks .databricks.backup + +trace $CLI bundle destroy --auto-approve &> LOG.destroy1 +cat LOG.destroy1 | contains.py '!panic:' '!internal error' > /dev/null + +# Restore the pre-destroy state; the resources are already gone from the server, +# so the next destroy must handle "resource already gone" idempotently. +rm -rf .databricks +mv .databricks.backup .databricks + +# Recreate the workspace root path deleted by destroy1 so destroy2 proceeds +# past the "no active deployment" check and actually re-runs resource deletions. +# MSYS_NO_PATHCONV=1 stops Git Bash on Windows from rewriting the leading-slash +# workspace path into C:/Program Files/Git/...; it is scoped to this command so +# the Python helpers (which need their own paths converted) are unaffected. +MSYS_NO_PATHCONV=1 $CLI workspace mkdirs "$ROOT_PATH" &> LOG.mkdirs +cat LOG.mkdirs | contains.py '!panic:' '!internal error' > /dev/null + +trace $CLI bundle destroy --auto-approve &> LOG.destroy2 +cat LOG.destroy2 | contains.py '!panic:' '!internal error' > /dev/null diff --git a/acceptance/bundle/invariant/destroy_idempotent/test.toml b/acceptance/bundle/invariant/destroy_idempotent/test.toml new file mode 100644 index 00000000000..16cf0797a77 --- /dev/null +++ b/acceptance/bundle/invariant/destroy_idempotent/test.toml @@ -0,0 +1,5 @@ +EnvMatrix.READPLAN = ["", "1"] + +# Snapshot of pre-destroy state used to re-run destroy on state that still +# references the (now-gone) resources; may linger if the test fails mid-run. +Ignore = [".databricks.backup"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index dac8eb3e30c..0ac1b789d6f 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 78f45faa7da..4cd19571951 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -3,36 +3,9 @@ unset DATABRICKS_BUNDLE_ENGINE -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp +invariant_setup -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init -fi - -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - -cp databricks.yml LOG.config - -cleanup() { - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null - -echo INPUT_CONFIG_OK +invariant_deploy LOG.deploy DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy MIGRATE_ARGS="" # The terraform provider sorts depends_on entries alphabetically by task_key on Read @@ -48,6 +21,4 @@ trace $CLI bundle deployment migrate $MIGRATE_ARGS &> LOG.migrate cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null -$CLI bundle plan -o json > plan.json 2>plan.json.err -cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null -verify_no_drift.py plan.json +invariant_verify_no_drift diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index af25cbb62dd..37dc2557922 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -11,6 +11,8 @@ EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] # Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] +# Instance pools are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to @@ -24,6 +26,17 @@ EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.y # Grant cross-references require the EmbeddedSlice pattern not present in terraform mode. EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] +# An empty grants list plans a spurious create after migrate: Terraform records no +# databricks_grants resource for grants: [], so migrate leaves no state entry for the +# grants node, but the direct plan emits a "create" for it anyway. Found by fuzz testing. +# The plan check fails with: +# Unexpected action='create' for resources.schemas.foo.grants +# ... +# "resources.schemas.foo.grants": { "action": "create", ... } +# Exit code: 10 +# Fixed by https://github.com/databricks/cli/pull/6039; re-enable once that lands. +EnvMatrixExclude.no_empty_grants = ["INPUT_CONFIG=schema_empty_grants.yml.tmpl"] + # SQL warehouses currently failing with migration with permanent drift. TODO: fix this. EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 6c269426855..a3719d00269 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", @@ -19,6 +17,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", @@ -43,6 +42,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 95ecd7cbfd7..9f031c0f613 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -1,52 +1,16 @@ # Invariant to test: no drift after deploy # Additional checks: no internal errors / panics in validate/plan/deploy -# Copy data files to test directory -cp -r "$TESTDIR/../data/." . &> LOG.cp - -# Run init script if present -INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" -if [ -f "$INIT_SCRIPT" ]; then - source "$INIT_SCRIPT" &> LOG.init +invariant_setup + +# Only compute a plan up front when the READPLAN variant actually consumes it via --plan. +# Without READPLAN, deploy computes its own plan internally, so a separate `bundle plan` +# invocation is pure waste. +if [[ -n "$READPLAN" ]]; then + $CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err + cat LOG.plan_initial.err | contains.py '!panic:' '!internal error' > /dev/null fi -envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml - -cp databricks.yml LOG.config - -# We redirect output rather than record it because some configs that are being tested may produce warnings -trace $CLI bundle validate &> LOG.validate - -cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null - -cleanup() { - trace $CLI bundle destroy --auto-approve &> LOG.destroy - cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null - - # Run cleanup script if present - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" - if [ -f "$CLEANUP_SCRIPT" ]; then - source "$CLEANUP_SCRIPT" &> LOG.cleanup - fi -} - -trap cleanup EXIT - -$CLI bundle plan -o json > plan.json 2>LOG.plan_initial.err -cat LOG.plan_initial.err | contains.py '!panic' '!internal error' > /dev/null - -trace $CLI bundle deploy $(readplanarg plan.json) &> LOG.deploy -cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null - -# Special message to fuzzer that generated config was fine. -# Any failures after this point will be considered as "bug detected" by fuzzer. -echo INPUT_CONFIG_OK - -# Check both text and JSON plan for no changes -# Note, expect that there maybe more than one resource unchanged -$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err -cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null -verify_no_drift.py LOG.planjson +invariant_deploy LOG.deploy $CLI bundle deploy $(readplanarg plan.json) -$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan -cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null +invariant_verify_no_drift diff --git a/acceptance/bundle/invariant/script.prepare b/acceptance/bundle/invariant/script.prepare new file mode 100644 index 00000000000..54f2a3dbf25 --- /dev/null +++ b/acceptance/bundle/invariant/script.prepare @@ -0,0 +1,54 @@ +# Shared setup for the invariant targets; each script keeps only the invariant it asserts. + +invariant_cleanup() { + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null + + CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +# Separate from invariant_setup so a caller that generates its own config can override the +# render alone; child script.prepare files are concatenated after this one. +invariant_render() { + cp -r "$TESTDIR/../data/." . &> LOG.cp + + INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init + fi + + envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml + + cp databricks.yml LOG.config +} + +# Call from the target, not at prepare time: prepare runs outside the subshell wrapping the +# script, so the trap would belong to the outer shell. +invariant_setup() { + invariant_render + + trap invariant_cleanup EXIT +} + +# Goes through trace, so callers can prefix the command with VAR=val. +invariant_deploy() { + local logfile="$1" + shift + trace "$@" &> "$logfile" + cat "$logfile" | contains.py '!panic:' '!internal error' > /dev/null + + # Tells the fuzzer the generated config was valid; failures after this count as bugs. + echo INPUT_CONFIG_OK +} + +# JSON plan asserts every action is "skip" -- a strict superset of the text +# renderer's "Plan: 0 to add, 0 to change, 0 to delete" summary. +# Overridable for a caller whose config the server does not round-trip exactly. +invariant_verify_no_drift() { + $CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err + cat LOG.planjson.err | contains.py '!panic:' '!internal error' > /dev/null + verify_no_drift.py LOG.planjson +} diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index bd8f84aa6db..2723cc9294c 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -3,13 +3,6 @@ Cloud = true RequiresUnityCatalog = true Timeout = '10m' -# These tests drive platform-independent bundle logic (config normalization, -# no-drift checks, state migration) and behave identically on every OS, so on pull -# requests they run on Linux only to keep the windows/macOS CI jobs fast. They still -# run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false - Ignore = [ ".databricks", ".venv", @@ -42,6 +35,7 @@ EnvMatrix.INPUT_CONFIG = [ "experiment.yml.tmpl", "external_location.yml.tmpl", "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", @@ -66,6 +60,7 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_synced_table.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml index 3c775b93368..b37ee45aed6 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.test.toml +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/lifecycle/started/out.test.toml +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/lifecycle/test.toml b/acceptance/bundle/lifecycle/test.toml index 6100b1c60ea..7d36fb9dc18 100644 --- a/acceptance/bundle/lifecycle/test.toml +++ b/acceptance/bundle/lifecycle/test.toml @@ -1,8 +1,2 @@ Local = true Cloud = false - -# These tests exercise platform-independent bundle logic (deploy/destroy lifecycle) -# and behave identically on every OS, so on pull requests they run on Linux only to -# keep the windows/macOS CI jobs fast. They still run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 31759042f71..71970b719d4 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.original_state.json b/acceptance/bundle/migrate/basic/out.original_state.json index 9335655f398..ccdef021a0c 100644 --- a/acceptance/bundle/migrate/basic/out.original_state.json +++ b/acceptance/bundle/migrate/basic/out.original_state.json @@ -55,6 +55,7 @@ "notebook_task": [], "notification_settings": [], "parameter": [], + "parent_path": null, "performance_target": null, "pipeline_task": [], "provider_config": [ diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.state_original.json b/acceptance/bundle/migrate/default-python/out.state_original.json index 5a22c371adb..23cef2d4d93 100644 --- a/acceptance/bundle/migrate/default-python/out.state_original.json +++ b/acceptance/bundle/migrate/default-python/out.state_original.json @@ -66,6 +66,7 @@ "cluster_name": "", "custom_tags": null, "data_security_mode": "DATA_SECURITY_MODE_AUTO", + "dependency_mode": "", "docker_image": [], "driver_instance_pool_id": "", "driver_node_type_flexibility": [], @@ -95,7 +96,8 @@ "worker_node_type_flexibility": [], "workload_type": [] } - ] + ], + "serverless_compute_id": "" } ], "library": [], @@ -116,6 +118,7 @@ "name": "schema" } ], + "parent_path": null, "performance_target": null, "pipeline_task": [], "provider_config": [ diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/output.txt b/acceptance/bundle/migrate/engine-config-terraform/output.txt index 5169f045260..c971bf6e091 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/output.txt +++ b/acceptance/bundle/migrate/engine-config-terraform/output.txt @@ -1,3 +1,8 @@ >>> musterr [CLI] bundle deployment migrate +Warning: the terraform deployment engine is deprecated and will stop working in a future version of the CLI + in databricks.yml:3:11 + +See https://docs.databricks.com/aws/en/dev-tools/bundles/direct for how to migrate to the direct deployment engine + Error: bundle.engine is set to "terraform". Migration requires "engine: direct" or no engine setting. Change the setting to "engine: direct" and retry diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.original_state.json b/acceptance/bundle/migrate/permissions/out.original_state.json index a0c5a990ccd..6a2ee250266 100644 --- a/acceptance/bundle/migrate/permissions/out.original_state.json +++ b/acceptance/bundle/migrate/permissions/out.original_state.json @@ -55,6 +55,7 @@ "notebook_task": [], "notification_settings": [], "parameter": [], + "parent_path": null, "performance_target": null, "pipeline_task": [], "provider_config": [ diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 37ec2643e7c..8c4484f983a 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -7,10 +7,3 @@ Ignore = [".databricks"] # matrix to ["direct"] so CI's engine filter includes these tests without # also running the same script twice per engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# These tests exercise platform-independent bundle logic (state and config -# migration) and behave identically on every OS, so on pull requests they run on -# Linux only to keep the windows/macOS CI jobs fast. They still run on every OS on -# push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 3b66f214fb1..2d7f6d65536 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -37,6 +37,11 @@ resources.alerts.*.lifecycle.prevent_destroy bool INPUT resources.alerts.*.lifecycle_state sql.AlertLifecycleState ALL resources.alerts.*.modified_status string INPUT resources.alerts.*.owner_user_name string ALL +resources.alerts.*.parameters []sql.AlertStatementParameter ALL +resources.alerts.*.parameters[*] sql.AlertStatementParameter ALL +resources.alerts.*.parameters[*].name string ALL +resources.alerts.*.parameters[*].type string ALL +resources.alerts.*.parameters[*].value string ALL resources.alerts.*.parent_path string ALL resources.alerts.*.query_text string ALL resources.alerts.*.run_as *sql.AlertV2RunAs ALL @@ -325,6 +330,7 @@ resources.clusters.*.custom_tags.* string ALL resources.clusters.*.data_security_mode compute.DataSecurityMode ALL resources.clusters.*.default_tags map[string]string REMOTE resources.clusters.*.default_tags.* string REMOTE +resources.clusters.*.dependency_mode compute.DependencyMode ALL resources.clusters.*.docker_image *compute.DockerImage ALL resources.clusters.*.docker_image.basic_auth *compute.DockerBasicAuth ALL resources.clusters.*.docker_image.basic_auth.password string ALL @@ -454,6 +460,7 @@ resources.clusters.*.spec.cluster_name string REMOTE resources.clusters.*.spec.custom_tags map[string]string REMOTE resources.clusters.*.spec.custom_tags.* string REMOTE resources.clusters.*.spec.data_security_mode compute.DataSecurityMode REMOTE +resources.clusters.*.spec.dependency_mode compute.DependencyMode REMOTE resources.clusters.*.spec.docker_image *compute.DockerImage REMOTE resources.clusters.*.spec.docker_image.basic_auth *compute.DockerBasicAuth REMOTE resources.clusters.*.spec.docker_image.basic_auth.password string REMOTE @@ -653,6 +660,12 @@ resources.experiments.*.tags []ml.ExperimentTag ALL resources.experiments.*.tags[*] ml.ExperimentTag ALL resources.experiments.*.tags[*].key string ALL resources.experiments.*.tags[*].value string ALL +resources.experiments.*.trace_location *ml.ExperimentTraceLocation ALL +resources.experiments.*.trace_location.uc_trace_location *ml.UcTraceLocation ALL +resources.experiments.*.trace_location.uc_trace_location.catalog string ALL +resources.experiments.*.trace_location.uc_trace_location.effective_table_prefix string ALL +resources.experiments.*.trace_location.uc_trace_location.schema string ALL +resources.experiments.*.trace_location.uc_trace_location.table_prefix string ALL resources.experiments.*.url string INPUT resources.experiments.*.permissions.object_id string ALL resources.experiments.*.permissions[*] dresources.StatePermission ALL @@ -756,6 +769,73 @@ resources.genie_spaces.*.permissions[*].group_name string ALL resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL resources.genie_spaces.*.permissions[*].service_principal_name string ALL resources.genie_spaces.*.permissions[*].user_name string ALL +resources.instance_pools.*.aws_attributes *compute.InstancePoolAwsAttributes ALL +resources.instance_pools.*.aws_attributes.availability compute.InstancePoolAwsAttributesAvailability ALL +resources.instance_pools.*.aws_attributes.instance_profile_arn string ALL +resources.instance_pools.*.aws_attributes.spot_bid_price_percent int ALL +resources.instance_pools.*.aws_attributes.zone_id string ALL +resources.instance_pools.*.azure_attributes *compute.InstancePoolAzureAttributes ALL +resources.instance_pools.*.azure_attributes.availability compute.InstancePoolAzureAttributesAvailability ALL +resources.instance_pools.*.azure_attributes.capacity_reservation_group string ALL +resources.instance_pools.*.azure_attributes.spot_bid_max_price float64 ALL +resources.instance_pools.*.custom_tags map[string]string ALL +resources.instance_pools.*.custom_tags.* string ALL +resources.instance_pools.*.default_tags map[string]string REMOTE +resources.instance_pools.*.default_tags.* string REMOTE +resources.instance_pools.*.disk_spec *compute.DiskSpec ALL +resources.instance_pools.*.disk_spec.disk_count int ALL +resources.instance_pools.*.disk_spec.disk_iops int ALL +resources.instance_pools.*.disk_spec.disk_size int ALL +resources.instance_pools.*.disk_spec.disk_throughput int ALL +resources.instance_pools.*.disk_spec.disk_type *compute.DiskType ALL +resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type compute.DiskTypeAzureDiskVolumeType ALL +resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type compute.DiskTypeEbsVolumeType ALL +resources.instance_pools.*.enable_elastic_disk bool ALL +resources.instance_pools.*.gcp_attributes *compute.InstancePoolGcpAttributes ALL +resources.instance_pools.*.gcp_attributes.gcp_availability compute.GcpAvailability ALL +resources.instance_pools.*.gcp_attributes.local_ssd_count int ALL +resources.instance_pools.*.gcp_attributes.zone_id string ALL +resources.instance_pools.*.id string INPUT +resources.instance_pools.*.idle_instance_autotermination_minutes int ALL +resources.instance_pools.*.instance_pool_id string REMOTE +resources.instance_pools.*.instance_pool_name string ALL +resources.instance_pools.*.lifecycle resources.Lifecycle INPUT +resources.instance_pools.*.lifecycle.prevent_destroy bool INPUT +resources.instance_pools.*.max_capacity int ALL +resources.instance_pools.*.min_idle_instances int ALL +resources.instance_pools.*.modified_status string INPUT +resources.instance_pools.*.node_type_flexibility *compute.NodeTypeFlexibility ALL +resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids []string ALL +resources.instance_pools.*.node_type_flexibility.alternate_node_type_ids[*] string ALL +resources.instance_pools.*.node_type_id string ALL +resources.instance_pools.*.preloaded_docker_images []compute.DockerImage ALL +resources.instance_pools.*.preloaded_docker_images[*] compute.DockerImage ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth *compute.DockerBasicAuth ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth.password string ALL +resources.instance_pools.*.preloaded_docker_images[*].basic_auth.username string ALL +resources.instance_pools.*.preloaded_docker_images[*].url string ALL +resources.instance_pools.*.preloaded_spark_versions []string ALL +resources.instance_pools.*.preloaded_spark_versions[*] string ALL +resources.instance_pools.*.remote_disk_throughput int ALL +resources.instance_pools.*.state compute.InstancePoolState REMOTE +resources.instance_pools.*.stats *compute.InstancePoolStats REMOTE +resources.instance_pools.*.stats.idle_count int REMOTE +resources.instance_pools.*.stats.pending_idle_count int REMOTE +resources.instance_pools.*.stats.pending_used_count int REMOTE +resources.instance_pools.*.stats.used_count int REMOTE +resources.instance_pools.*.status *compute.InstancePoolStatus REMOTE +resources.instance_pools.*.status.pending_instance_errors []compute.PendingInstanceError REMOTE +resources.instance_pools.*.status.pending_instance_errors[*] compute.PendingInstanceError REMOTE +resources.instance_pools.*.status.pending_instance_errors[*].instance_id string REMOTE +resources.instance_pools.*.status.pending_instance_errors[*].message string REMOTE +resources.instance_pools.*.total_initial_remote_disk_size int ALL +resources.instance_pools.*.url string INPUT +resources.instance_pools.*.permissions.object_id string ALL +resources.instance_pools.*.permissions[*] dresources.StatePermission ALL +resources.instance_pools.*.permissions[*].group_name string ALL +resources.instance_pools.*.permissions[*].level iam.PermissionLevel ALL +resources.instance_pools.*.permissions[*].service_principal_name string ALL +resources.instance_pools.*.permissions[*].user_name string ALL resources.job_runs.*.dbt_commands []string ALL resources.job_runs.*.dbt_commands[*] string ALL resources.job_runs.*.id string INPUT @@ -911,6 +991,7 @@ resources.jobs.*.job_clusters[*].new_cluster.cluster_name string ALL resources.jobs.*.job_clusters[*].new_cluster.custom_tags map[string]string ALL resources.jobs.*.job_clusters[*].new_cluster.custom_tags.* string ALL resources.jobs.*.job_clusters[*].new_cluster.data_security_mode compute.DataSecurityMode ALL +resources.jobs.*.job_clusters[*].new_cluster.dependency_mode compute.DependencyMode ALL resources.jobs.*.job_clusters[*].new_cluster.docker_image *compute.DockerImage ALL resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth ALL resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth.password string ALL @@ -979,6 +1060,7 @@ resources.jobs.*.job_clusters[*].new_cluster.workload_type *compute.WorkloadType resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients compute.ClientsTypes ALL resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs bool ALL resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks bool ALL +resources.jobs.*.job_clusters[*].serverless_compute_id string ALL resources.jobs.*.job_id int64 REMOTE resources.jobs.*.lifecycle resources.Lifecycle INPUT resources.jobs.*.lifecycle.prevent_destroy bool INPUT @@ -1261,6 +1343,7 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_name string ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags map[string]string ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags.* string ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.data_security_mode compute.DataSecurityMode ALL +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.dependency_mode compute.DependencyMode ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image *compute.DockerImage ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth *compute.DockerBasicAuth ALL resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth.password string ALL @@ -1547,6 +1630,7 @@ resources.jobs.*.tasks[*].new_cluster.cluster_name string ALL resources.jobs.*.tasks[*].new_cluster.custom_tags map[string]string ALL resources.jobs.*.tasks[*].new_cluster.custom_tags.* string ALL resources.jobs.*.tasks[*].new_cluster.data_security_mode compute.DataSecurityMode ALL +resources.jobs.*.tasks[*].new_cluster.dependency_mode compute.DependencyMode ALL resources.jobs.*.tasks[*].new_cluster.docker_image *compute.DockerImage ALL resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth ALL resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth.password string ALL @@ -2266,6 +2350,12 @@ resources.model_serving_endpoints.*.endpoint_details.telemetry_config *serving.T resources.model_serving_endpoints.*.endpoint_details.telemetry_config.inference_table_config *serving.TelemetryInferenceTableConfig REMOTE resources.model_serving_endpoints.*.endpoint_details.telemetry_config.inference_table_config.name string REMOTE resources.model_serving_endpoints.*.endpoint_details.telemetry_config.inference_table_config.sampling_fraction float64 REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.table_names *serving.UnityCatalogTableNames REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.table_names.annotations_table string REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.table_names.logs_table string REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.table_names.metrics_table string REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.table_names.traces_table string REMOTE +resources.model_serving_endpoints.*.endpoint_details.telemetry_config.telemetry_profile_id string REMOTE resources.model_serving_endpoints.*.endpoint_id string REMOTE resources.model_serving_endpoints.*.id string INPUT resources.model_serving_endpoints.*.lifecycle resources.Lifecycle INPUT @@ -2286,6 +2376,12 @@ resources.model_serving_endpoints.*.telemetry_config *serving.TelemetryConfig AL resources.model_serving_endpoints.*.telemetry_config.inference_table_config *serving.TelemetryInferenceTableConfig ALL resources.model_serving_endpoints.*.telemetry_config.inference_table_config.name string ALL resources.model_serving_endpoints.*.telemetry_config.inference_table_config.sampling_fraction float64 ALL +resources.model_serving_endpoints.*.telemetry_config.table_names *serving.UnityCatalogTableNames ALL +resources.model_serving_endpoints.*.telemetry_config.table_names.annotations_table string ALL +resources.model_serving_endpoints.*.telemetry_config.table_names.logs_table string ALL +resources.model_serving_endpoints.*.telemetry_config.table_names.metrics_table string ALL +resources.model_serving_endpoints.*.telemetry_config.table_names.traces_table string ALL +resources.model_serving_endpoints.*.telemetry_config.telemetry_profile_id string ALL resources.model_serving_endpoints.*.url string INPUT resources.model_serving_endpoints.*.permissions.object_id string ALL resources.model_serving_endpoints.*.permissions[*] dresources.StatePermission ALL @@ -2507,6 +2603,7 @@ resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.scd_type pipelines.TableSpecificConfigScdType ALL resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.sequence_by []string ALL resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.sequence_by[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.source_metadata_column string ALL resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.table_properties map[string]string ALL resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.table_properties.* string ALL resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.workday_report_parameters *pipelines.IngestionPipelineDefinitionWorkdayReportParameters ALL @@ -2543,6 +2640,14 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.g resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.single_variant_column string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.url string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options *pipelines.GoogleAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options *pipelines.GoogleAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.metrics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.metrics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource_fields []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.resource_fields[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.segments []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options.segments[*] string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.lookback_window_days int ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.manager_account_id string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.sync_start_date string ALL @@ -2582,6 +2687,16 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.m resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.breakdowns []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.breakdowns[*] string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_insights_lookback_window int ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options *pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_attribution_windows []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_attribution_windows[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.action_report_time string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.level string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.custom_report_options.time_increment string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.level string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.time_increment string ALL @@ -2603,6 +2718,14 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.o resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.subject_filter []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.subject_filter[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options *pipelines.RedditAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options *pipelines.RedditAdsOptionsRedditAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.fields []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.custom_report_options.fields[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.lookback_window_days int ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.reddit_ads_options.sync_start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options *pipelines.SharepointOptions ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.entity_type pipelines.SharepointOptionsSharepointEntityType ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options *pipelines.FileIngestionOptions ALL @@ -2626,6 +2749,14 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.s resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options *pipelines.SmartsheetOptions ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.smartsheet_options.enforce_schema bool ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options *pipelines.TikTokAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options *pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.data_level pipelines.TikTokAdsOptionsTikTokDataLevel ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.dimensions []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.dimensions[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.metrics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.metrics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.query_lifetime bool ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.report_type pipelines.TikTokAdsOptionsTikTokReportType ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.data_level pipelines.TikTokAdsOptionsTikTokDataLevel ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.dimensions []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.dimensions[*] string ALL @@ -2639,6 +2770,17 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.z resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.zendesk_support_options.start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.destination_catalog string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.destination_schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options *pipelines.IngestionPipelineDefinitionFanoutOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.fanout_by string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms []pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*] pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].format pipelines.TransformerFormat ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options *pipelines.JsonTransformerOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.as_variant bool ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_evolution_mode pipelines.FileIngestionOptionsSchemaEvolutionMode ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_file_path string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_hints string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.source_catalog string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.source_schema string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration *pipelines.TableSpecificConfig ALL @@ -2664,6 +2806,7 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type pipelines.TableSpecificConfigScdType ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.sequence_by []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.sequence_by[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.source_metadata_column string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.table_properties map[string]string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.table_properties.* string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.workday_report_parameters *pipelines.IngestionPipelineDefinitionWorkdayReportParameters ALL @@ -2700,6 +2843,14 @@ resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gd resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.single_variant_column string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.url string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options *pipelines.GoogleAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options *pipelines.GoogleAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.metrics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.metrics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource_fields []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.resource_fields[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.segments []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options.segments[*] string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.lookback_window_days int ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.manager_account_id string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.sync_start_date string ALL @@ -2739,6 +2890,16 @@ resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.me resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.breakdowns []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.breakdowns[*] string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_insights_lookback_window int ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options *pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_attribution_windows []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_attribution_windows[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.action_report_time string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.level string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.custom_report_options.time_increment string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.level string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.time_increment string ALL @@ -2760,6 +2921,14 @@ resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.ou resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.subject_filter []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.subject_filter[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options *pipelines.RedditAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options *pipelines.RedditAdsOptionsRedditAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.breakdowns []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.breakdowns[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.fields []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.custom_report_options.fields[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.lookback_window_days int ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.reddit_ads_options.sync_start_date string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options *pipelines.SharepointOptions ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.entity_type pipelines.SharepointOptionsSharepointEntityType ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options *pipelines.FileIngestionOptions ALL @@ -2783,6 +2952,14 @@ resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sh resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options *pipelines.SmartsheetOptions ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.smartsheet_options.enforce_schema bool ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options *pipelines.TikTokAdsOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options *pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.data_level pipelines.TikTokAdsOptionsTikTokDataLevel ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.dimensions []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.dimensions[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.metrics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.metrics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.query_lifetime bool ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.report_type pipelines.TikTokAdsOptionsTikTokReportType ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.data_level pipelines.TikTokAdsOptionsTikTokDataLevel ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.dimensions []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.dimensions[*] string ALL @@ -2823,6 +3000,7 @@ resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration. resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type pipelines.TableSpecificConfigScdType ALL resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.sequence_by []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.sequence_by[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.source_metadata_column string ALL resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.table_properties map[string]string ALL resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.table_properties.* string ALL resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.workday_report_parameters *pipelines.IngestionPipelineDefinitionWorkdayReportParameters ALL @@ -2867,6 +3045,7 @@ resources.pipelines.*.ingestion_definition.table_configuration.salesforce_includ resources.pipelines.*.ingestion_definition.table_configuration.scd_type pipelines.TableSpecificConfigScdType ALL resources.pipelines.*.ingestion_definition.table_configuration.sequence_by []string ALL resources.pipelines.*.ingestion_definition.table_configuration.sequence_by[*] string ALL +resources.pipelines.*.ingestion_definition.table_configuration.source_metadata_column string ALL resources.pipelines.*.ingestion_definition.table_configuration.table_properties map[string]string ALL resources.pipelines.*.ingestion_definition.table_configuration.table_properties.* string ALL resources.pipelines.*.ingestion_definition.table_configuration.workday_report_parameters *pipelines.IngestionPipelineDefinitionWorkdayReportParameters ALL @@ -3162,6 +3341,12 @@ resources.postgres_synced_tables.*.branch string ALL resources.postgres_synced_tables.*.create_database_objects_if_missing bool ALL resources.postgres_synced_tables.*.create_time *time.Time REMOTE resources.postgres_synced_tables.*.existing_pipeline_id string ALL +resources.postgres_synced_tables.*.extra_columns []postgres.SyncedTableSyncedTableSpecExtraColumn ALL +resources.postgres_synced_tables.*.extra_columns[*] postgres.SyncedTableSyncedTableSpecExtraColumn ALL +resources.postgres_synced_tables.*.extra_columns[*].column_name string ALL +resources.postgres_synced_tables.*.extra_columns[*].column_type string ALL +resources.postgres_synced_tables.*.extra_columns[*].compute string ALL +resources.postgres_synced_tables.*.extra_columns[*].maintenance postgres.SyncedTableSyncedTableSpecExtraColumnMaintenance ALL resources.postgres_synced_tables.*.id string INPUT resources.postgres_synced_tables.*.lifecycle resources.Lifecycle INPUT resources.postgres_synced_tables.*.lifecycle.prevent_destroy bool INPUT diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/create_error/out.test.toml b/acceptance/bundle/resource_deps/create_error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/create_error/out.test.toml +++ b/acceptance/bundle/resource_deps/create_error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/id_chain/out.test.toml b/acceptance/bundle/resource_deps/id_chain/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.test.toml +++ b/acceptance/bundle/resource_deps/id_chain/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/id_star/out.test.toml b/acceptance/bundle/resource_deps/id_star/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/id_star/out.test.toml +++ b/acceptance/bundle/resource_deps/id_star/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id/out.test.toml b/acceptance/bundle/resource_deps/job_id/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_id/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.test.toml +++ b/acceptance/bundle/resource_deps/job_tasks/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update/out.test.toml b/acceptance/bundle/resource_deps/jobs_update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/jobs_update/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_self/out.test.toml b/acceptance/bundle/resource_deps/loop_self/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/loop_self/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_self/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml +++ b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/permission_ref/out.test.toml b/acceptance/bundle/resource_deps/permission_ref/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/permission_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml index 6c76c91ec1b..8c738f635ac 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var/out.test.toml b/acceptance/bundle/resource_deps/resources_var/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index 0a43e097e33..dc29b70c320 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -4,10 +4,3 @@ Ignore = [ ".databricks", ".gitignore", ] - -# These tests exercise platform-independent bundle logic (resource dependency -# ordering and reference resolution) and behave identically on every OS, so on pull -# requests they run on Linux only to keep the windows/macOS CI jobs fast. They still -# run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml +++ b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index 7a9d26f12e5..c45d8e76a8e 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file/out.test.toml b/acceptance/bundle/resources/alerts/with_file/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/alerts/with_file/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/app/app.py b/acceptance/bundle/resources/apps/config-drift-stopped/app/app.py new file mode 100644 index 00000000000..ad1e64f9fb0 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/app/app.py @@ -0,0 +1,5 @@ +import http.server, os + +http.server.HTTPServer( + ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler +).serve_forever() diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/databricks.yml.tmpl b/acceptance/bundle/resources/apps/config-drift-stopped/databricks.yml.tmpl new file mode 100644 index 00000000000..d6e5da23106 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: config-drift-stopped-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app + source_code_path: ./app + config: + command: + - python + - app.py + env: + - name: MY_VAR + value: original_value + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/output.txt b/acceptance/bundle/resources/apps/config-drift-stopped/output.txt new file mode 100644 index 00000000000..f26277b5804 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/output.txt @@ -0,0 +1,73 @@ + +=== Deploy with started=true so the app has an active deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-stopped-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded +Updating deployment state... +Deployment complete! + +=== Change config while running: drift is detected (active deployment present) +>>> update_file.py databricks.yml original_value changed_value + +>>> [CLI] bundle plan -o json +{ + "config.env[0].value": { + "action": "update", + "old": "original_value", + "new": "changed_value", + "remote": "original_value" + } +} + +=== Stop the app: the backend clears the active deployment +>>> [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +=== Same config change is now skipped: no active deployment to compare against +>>> [CLI] bundle plan -o json +{ + "config": { + "action": "skip", + "reason": "no active deployment", + "old": { + "command": [ + "python", + "app.py" + ], + "env": [ + { + "name": "MY_VAR", + "value": "original_value" + } + ] + }, + "new": { + "command": [ + "python", + "app.py" + ], + "env": [ + { + "name": "MY_VAR", + "value": "changed_value" + } + ] + } + }, + "config.env[0].value": { + "action": "skip", + "reason": "no active deployment", + "old": "original_value", + "new": "changed_value" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/config-drift-stopped-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/script b/acceptance/bundle/resources/apps/config-drift-stopped/script new file mode 100644 index 00000000000..c449a3099a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy with started=true so the app has an active deployment" +trace $CLI bundle deploy + +title "Change config while running: drift is detected (active deployment present)" +trace update_file.py databricks.yml original_value changed_value +trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' + +title "Stop the app: the backend clears the active deployment" +trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' + +title "Same config change is now skipped: no active deployment to compare against" +trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/test.toml new file mode 100644 index 00000000000..5b3deb5b630 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift-stopped/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = false + +# Asserts plan classification, not requests; drop the inherited RecordRequests. +RecordRequests = false + +Ignore = [".databricks", "databricks.yml"] + +# Direct engine only: the skip logic lives in OverrideChangeDesc. Mirrors +# config-no-deployment. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/app/app.py b/acceptance/bundle/resources/apps/config-no-deployment/app/app.py new file mode 100644 index 00000000000..96fa00a4f14 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/app/app.py @@ -0,0 +1,10 @@ +import os + +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def home(): + return "Hello from config-no-deployment app!" diff --git a/acceptance/bundle/resources/apps/config-no-deployment/app/app.yaml b/acceptance/bundle/resources/apps/config-no-deployment/app/app.yaml new file mode 100644 index 00000000000..61471358dce --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/app/app.yaml @@ -0,0 +1,3 @@ +command: + - python + - app.py diff --git a/acceptance/bundle/resources/apps/config-no-deployment/databricks.yml b/acceptance/bundle/resources/apps/config-no-deployment/databricks.yml new file mode 100644 index 00000000000..c730a8d1dd5 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: app-config-no-deployment + +resources: + apps: + myapp: + name: test-app-config-no-deployment + description: my_app_description + source_code_path: ./app + config: + command: ["python", "app.py"] + env: + - name: MY_ENV_VAR + value: test_value diff --git a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/output.txt b/acceptance/bundle/resources/apps/config-no-deployment/output.txt new file mode 100644 index 00000000000..9b158ea51fd --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/output.txt @@ -0,0 +1,50 @@ + +=== Deploy: app created with no_compute; config is not deployed until the app starts +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-config-no-deployment/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op: config/source_code_path drift is skipped while the app has no active deployment +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Both deploy-only fields are skipped with reason "no active deployment" +>>> [CLI] bundle plan -o json +{ + "config": { + "action": "skip", + "reason": "no active deployment", + "old": { + "command": [ + "python", + "app.py" + ], + "env": [ + { + "name": "MY_ENV_VAR", + "value": "test_value" + } + ] + }, + "new": { + "command": [ + "python", + "app.py" + ], + "env": [ + { + "name": "MY_ENV_VAR", + "value": "test_value" + } + ] + } + }, + "source_code_path": { + "action": "skip", + "reason": "no active deployment", + "old": "/Workspace/Users/[USERNAME]/.bundle/app-config-no-deployment/default/files/app", + "new": "/Workspace/Users/[USERNAME]/.bundle/app-config-no-deployment/default/files/app" + } +} diff --git a/acceptance/bundle/resources/apps/config-no-deployment/script b/acceptance/bundle/resources/apps/config-no-deployment/script new file mode 100644 index 00000000000..f1970999645 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/script @@ -0,0 +1,10 @@ +echo "*" > .gitignore + +title "Deploy: app created with no_compute; config is not deployed until the app starts" +trace $CLI bundle deploy + +title "Plan is a no-op: config/source_code_path drift is skipped while the app has no active deployment" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Both deploy-only fields are skipped with reason \"no active deployment\"" +trace $CLI bundle plan -o json | jq '.plan[].changes | {config, source_code_path}' diff --git a/acceptance/bundle/resources/apps/config-no-deployment/test.toml b/acceptance/bundle/resources/apps/config-no-deployment/test.toml new file mode 100644 index 00000000000..6d58b36d864 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-no-deployment/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = false + +# Asserts plan classification, not requests; drop the inherited RecordRequests. +RecordRequests = false + +Ignore = [".databricks"] + +# Direct engine only: the skip logic lives in OverrideChangeDesc. Mirrors +# lifecycle-started-omitted. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml +++ b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/default_description/out.test.toml b/acceptance/bundle/resources/apps/default_description/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/apps/default_description/out.test.toml +++ b/acceptance/bundle/resources/apps/default_description/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/databricks.yml b/acceptance/bundle/resources/apps/git-source-no-deployment/databricks.yml new file mode 100644 index 00000000000..d3711e17833 --- /dev/null +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: app-git-source-no-deployment + +resources: + apps: + myapp: + name: test-app-git-source-no-deployment + description: my_app_description + git_source: + branch: main diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt b/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt new file mode 100644 index 00000000000..77e8df548da --- /dev/null +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt @@ -0,0 +1,26 @@ + +=== Deploy: app created with no_compute; git_source is not deployed until the app starts +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-git-source-no-deployment/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op: git_source drift is skipped while the app has no active deployment +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== git_source is skipped with reason "no active deployment" +>>> [CLI] bundle plan -o json +{ + "git_source": { + "action": "skip", + "reason": "no active deployment", + "old": { + "branch": "main" + }, + "new": { + "branch": "main" + } + } +} diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/script b/acceptance/bundle/resources/apps/git-source-no-deployment/script new file mode 100644 index 00000000000..63cce6312c3 --- /dev/null +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/script @@ -0,0 +1,10 @@ +echo "*" > .gitignore + +title "Deploy: app created with no_compute; git_source is not deployed until the app starts" +trace $CLI bundle deploy + +title "Plan is a no-op: git_source drift is skipped while the app has no active deployment" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "git_source is skipped with reason \"no active deployment\"" +trace $CLI bundle plan -o json | jq '.plan[].changes | {git_source}' diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/test.toml new file mode 100644 index 00000000000..b943555ebc1 --- /dev/null +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = false + +# Asserts plan classification, not requests; drop the inherited RecordRequests. +RecordRequests = false + +Ignore = [".databricks"] + +# Direct engine only: the skip logic lives in OverrideChangeDesc. Mirrors +# config-no-deployment. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/immutable/out.test.toml b/acceptance/bundle/resources/apps/immutable/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/apps/immutable/out.test.toml +++ b/acceptance/bundle/resources/apps/immutable/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/inline_config/out.test.toml b/acceptance/bundle/resources/apps/inline_config/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/apps/inline_config/out.test.toml +++ b/acceptance/bundle/resources/apps/inline_config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml index 4ebd478a982..42c0997090a 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml index 31759042f71..71970b719d4 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/resource-refs/out.test.toml b/acceptance/bundle/resources/apps/resource-refs/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/apps/resource-refs/out.test.toml +++ b/acceptance/bundle/resources/apps/resource-refs/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/update/out.test.toml b/acceptance/bundle/resources/apps/update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/apps/update/out.test.toml +++ b/acceptance/bundle/resources/apps/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/basic/out.test.toml b/acceptance/bundle/resources/catalogs/basic/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/catalogs/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/basic/script b/acceptance/bundle/resources/catalogs/basic/script index 3ce45404638..f5d2fd82486 100644 --- a/acceptance/bundle/resources/catalogs/basic/script +++ b/acceptance/bundle/resources/catalogs/basic/script @@ -15,7 +15,7 @@ title "Deploy bundle with catalog" trace $CLI bundle deploy title "Assert the catalog is created" -trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}" | normalize_uc_payload.py title "Update catalog comment" update_file.py databricks.yml "This catalog was created from DABs" "Updated comment from DABs" diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml +++ b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/script b/acceptance/bundle/resources/catalogs/with-schemas/script index 66930927295..039a20fd670 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/script +++ b/acceptance/bundle/resources/catalogs/with-schemas/script @@ -19,7 +19,7 @@ title "Deploy bundle with catalog and schema" trace $CLI bundle deploy title "Assert the catalog is created" -trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}" | normalize_uc_payload.py title "Assert schema is created in the custom catalog" trace $CLI schemas get "${SCHEMA_FULL_NAME}" | jq "{full_name, catalog_name, comment}" diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml index d84aa6ed4b0..5a821e39edc 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.aws = false CloudEnvs.azure = false CloudEnvs.gcp = true diff --git a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml index 4ebd478a982..42c0997090a 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml index 31759042f71..71970b719d4 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 993733d1e7c..475b179caed 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-name/out.test.toml b/acceptance/bundle/resources/dashboards/change-name/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/resources/dashboards/change-name/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-name/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index ee9ef8a8e64..c2ac722e76a 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml index f489904c6fa..96be4fdfe9d 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml index 871f53935b8..8f6c4a03c57 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml index 46450314509..a29f11b9ab2 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index 6eb512566f2..7edd52865f7 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index d0aa931adb0..1f9d1fc1b75 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -3,7 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml index 4e6b374da1b..c777e3ce206 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/recreate/out.test.toml b/acceptance/bundle/resources/database_instances/recreate/out.test.toml index 4e6b374da1b..c777e3ce206 100644 --- a/acceptance/bundle/resources/database_instances/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_instances/recreate/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml index 37502cdd88f..12ab4ea7f78 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml @@ -2,7 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/external_locations/out.test.toml +++ b/acceptance/bundle/resources/external_locations/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml index 46450314509..a29f11b9ab2 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index 46450314509..a29f11b9ab2 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml index 56db5043922..6072bc71acd 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml index 56db5043922..6072bc71acd 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml index 46450314509..a29f11b9ab2 100644 --- a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 56db5043922..6072bc71acd 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml index 56db5043922..6072bc71acd 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.test.toml b/acceptance/bundle/resources/grants/catalogs/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.test.toml +++ b/acceptance/bundle/resources/grants/catalogs/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/script b/acceptance/bundle/resources/grants/catalogs/script index b584555570f..067710b705e 100755 --- a/acceptance/bundle/resources/grants/catalogs/script +++ b/acceptance/bundle/resources/grants/catalogs/script @@ -1,8 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json -jq 'walk(if type == "object" then del(.created_at, .created_by, .updated_at, .updated_by, .metastore_id, .browse_only, .effective_predictive_optimization_flag, .enable_predictive_optimization, .isolation_mode, .securable_type) else . end)' out.plan1.$DATABRICKS_BUNDLE_ENGINE.json > tmp.json && mv tmp.json out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -14,8 +13,7 @@ trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA -trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json -jq 'walk(if type == "object" then del(.created_at, .created_by, .updated_at, .updated_by, .metastore_id, .browse_only, .effective_predictive_optimization_flag, .enable_predictive_optimization, .isolation_mode, .securable_type) else . end)' out.plan2.$DATABRICKS_BUNDLE_ENGINE.json > tmp.json && mv tmp.json out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/registered_models/out.test.toml b/acceptance/bundle/resources/grants/registered_models/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.test.toml +++ b/acceptance/bundle/resources/grants/registered_models/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/databricks.yml.tmpl b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/databricks.yml.tmpl new file mode 100644 index 00000000000..8266f099163 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: schema-grants-all-privileges-coexist-$UNIQUE_NAME + +resources: + schemas: + grants_schema: + name: schema_all_priv_coexist_$UNIQUE_NAME + catalog_name: main + grants: + # Config declares only ALL_PRIVILEGES for the principal. Reproduces + # issue #6030: the backend also reports a concrete privilege for this + # principal (added out of band below / materialized for owners), and + # buildGrantChanges skips the ALL_PRIVILEGES removal, so the concrete + # privilege is never reconciled and the plan reports perpetual update. + - principal: $CURRENT_USER_NAME + privileges: + - ALL_PRIVILEGES diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.direct.txt b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.direct.txt new file mode 100644 index 00000000000..068a177d51f --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.direct.txt @@ -0,0 +1,3 @@ + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.terraform.txt b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.terraform.txt new file mode 100644 index 00000000000..8c7f94dfa63 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.terraform.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle plan +update schemas.grants_schema.grants + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml new file mode 100644 index 00000000000..e849ec85ace --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/output.txt b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/output.txt new file mode 100644 index 00000000000..8c6cc0fcd0d --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/output.txt @@ -0,0 +1,29 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-all-privileges-coexist-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] grants update schema main.schema_all_priv_coexist_[UNIQUE_NAME] --json @update.resolved.json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-all-privileges-coexist-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> errcode [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.grants_schema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.grants_schema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-all-privileges-coexist-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/script b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/script new file mode 100644 index 00000000000..bf04258675f --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/script @@ -0,0 +1,26 @@ +SCHEMA_FULL_NAME=main.schema_all_priv_coexist_$UNIQUE_NAME + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +# Out of band, grant a concrete privilege in addition to the ALL_PRIVILEGES that +# the config declares for the same principal. This reproduces issue #6030: the +# backend then reports both ALL_PRIVILEGES and the concrete privilege. +envsubst < update.json > update.resolved.json +trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.resolved.json > /dev/null + +# The first plan differs between engines: the direct engine treats ALL_PRIVILEGES +# as implying every concrete privilege, so it reports no drift (issue #6030), +# while terraform revokes the extra privilege to match the config exactly. +trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 + +# After deploying, both engines converge: the follow-up plan reports no changes. +trace $CLI bundle deploy +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/test.toml new file mode 100644 index 00000000000..3411dded007 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/test.toml @@ -0,0 +1 @@ +Ignore = ['update.resolved.json'] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/update.json b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/update.json new file mode 100644 index 00000000000..d43aec70917 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/update.json @@ -0,0 +1,8 @@ +{ + "changes": [ + { + "principal": "$CURRENT_USER_NAME", + "add": ["USE_SCHEMA"] + } + ] +} diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/script b/acceptance/bundle/resources/grants/schemas/change_privilege/script index 0000f3eeb71..737f7491d3a 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/script +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/script @@ -13,8 +13,7 @@ trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml USE_SCHEMA APPLY_TAG -trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json -jq 'walk(if type == "object" then del(.effective_predictive_optimization_flag, .enable_predictive_optimization, .metastore_id, .schema_id, .updated_at, .updated_by) else . end)' out.plan2.$DATABRICKS_BUNDLE_ENGINE.json > tmp.json && mv tmp.json out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json index 84fca9f95c8..ff320b18213 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json @@ -10,16 +10,14 @@ "browse_only": false, "catalog_name": "main", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", "enable_predictive_optimization": "INHERIT", "full_name": "main.schema_dup_grants_[UNIQUE_NAME]", "metastore_id": "[UUID]", "name": "schema_dup_grants_[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" } }, "resources.schemas.apps_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script index 479b54f81e2..12a255773b1 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script @@ -9,7 +9,6 @@ trap cleanup EXIT # Same principal listed twice with the same privilege. trace $CLI bundle deploy print_requests.py --get //permissions --keep > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -tmp=$(mktemp) # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. -jq 'del(.plan["resources.schemas.apps_schema"].remote_state.effective_predictive_optimization_flag)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. +trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.direct.txt deleted file mode 100644 index 6a435e4733d..00000000000 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.direct.txt +++ /dev/null @@ -1,4 +0,0 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.terraform.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.terraform.txt deleted file mode 100644 index c4605e6b9d1..00000000000 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy.terraform.txt +++ /dev/null @@ -1,19 +0,0 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-[UNIQUE_NAME]/default/files... -Error: exit status 1 - -Error: Missing required argument - - with databricks_grants.schema_grants_schema, - on bundle.tf.json line 17, in resource.databricks_grants.schema_grants_schema: - 17: "grant": [ - 18: { - 19: "principal": "deco-test-user@databricks.com", - 20: "privileges": null - 21: } - 22: ] - -The argument "grant.0.privileges" is required, but no definition was found. - - - -Exit code: 1 diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy1.requests.direct.json b/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy1.requests.direct.json deleted file mode 100644 index 6c720ca4c87..00000000000 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.deploy1.requests.direct.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_grants_[UNIQUE_NAME]", - "body": { - "changes": [ - { - "principal": "deco-test-user@databricks.com", - "remove": [ - "ALL_PRIVILEGES" - ] - } - ] - } -} diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt deleted file mode 100644 index dca661cb0ba..00000000000 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "plan_version": 2, - "cli_version": "[CLI_VERSION]", - "plan": { - "resources.schemas.grants_schema": { - "action": "create", - "new_state": { - "value": { - "catalog_name": "main", - "name": "schema_grants_[UNIQUE_NAME]" - } - } - }, - "resources.schemas.grants_schema.grants": { - "depends_on": [ - { - "node": "resources.schemas.grants_schema", - "label": "${resources.schemas.grants_schema.id}" - } - ], - "action": "create", - "new_state": { - "value": { - "securable_type": "schema", - "full_name": "", - "__embed__": [ - { - "principal": "deco-test-user@databricks.com" - } - ] - }, - "vars": { - "full_name": "${resources.schemas.grants_schema.id}" - } - } - } - } -} diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.terraform.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.terraform.txt deleted file mode 100644 index c2e037a3a8f..00000000000 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.terraform.txt +++ /dev/null @@ -1,18 +0,0 @@ -Error: exit status 1 - -Error: Missing required argument - - with databricks_grants.schema_grants_schema, - on bundle.tf.json line 17, in resource.databricks_grants.schema_grants_schema: - 17: "grant": [ - 18: { - 19: "principal": "deco-test-user@databricks.com", - 20: "privileges": null - 21: } - 22: ] - -The argument "grant.0.privileges" is required, but no definition was found. - - - -Exit code: 1 diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/output.txt b/acceptance/bundle/resources/grants/schemas/empty_array/output.txt index e69de29bb2d..3305b087d98 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/output.txt +++ b/acceptance/bundle/resources/grants/schemas/empty_array/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle validate +Error: grant privileges is required + at resources.schemas.grants_schema.grants[0] + in databricks.yml:10:11 + +Name: schema-grants-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/schema-grants-[UNIQUE_NAME]/default + +Found 1 error + +>>> [CLI] bundle deploy +Error: grant privileges is required + at resources.schemas.grants_schema.grants[0] + in databricks.yml:10:11 + + +>>> print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/script b/acceptance/bundle/resources/grants/schemas/empty_array/script index cf8541f96c2..15f2c294adf 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/script +++ b/acceptance/bundle/resources/grants/schemas/empty_array/script @@ -1,6 +1,8 @@ envsubst < databricks.yml.tmpl > databricks.yml -errcode $CLI bundle plan -o json &> out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py --get //permissions -errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json +# Empty privileges never converge (the backend drops principals with none), so reject early. +musterr trace $CLI bundle validate + +# Deploy must abort at validation: no securable is created, so no partial deploy. +musterr trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json index fc7ceb399af..cc1ae0e9cf3 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json @@ -10,16 +10,14 @@ "browse_only": false, "catalog_name": "main", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", "enable_predictive_optimization": "INHERIT", "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]", "metastore_id": "[UUID]", "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" } }, "resources.schemas.grants_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json index 90586b63a61..bf38acd93fe 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json @@ -10,16 +10,14 @@ "browse_only": false, "catalog_name": "main", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", "enable_predictive_optimization": "INHERIT", "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]", "metastore_id": "[UUID]", "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" } }, "resources.schemas.grants_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index 4ae7b3e6843..6f8965cfe1a 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -16,9 +16,9 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) +# __embed__ is extracted above; drop it so its non-deterministic order does not churn this golden. # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. -jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__, .plan["resources.schemas.grants_schema"].remote_state.effective_predictive_optimization_flag)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. +jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json -tmp=$(mktemp) -jq 'del(.plan["resources.schemas.grants_schema"].remote_state.effective_predictive_optimization_flag)' out.plan2.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/volumes/out.test.toml b/acceptance/bundle/resources/grants/volumes/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/grants/volumes/out.test.toml +++ b/acceptance/bundle/resources/grants/volumes/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/volumes/script b/acceptance/bundle/resources/grants/volumes/script index 2da24f4fe83..725049cda8e 100644 --- a/acceptance/bundle/resources/grants/volumes/script +++ b/acceptance/bundle/resources/grants/volumes/script @@ -10,8 +10,7 @@ trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq -- update_file.py databricks.yml WRITE_VOLUME MANAGE # different on cloud vs local due to remote state -trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json -jq 'walk(if type == "object" then del(.effective_predictive_optimization_flag, .enable_predictive_optimization, .metastore_id, .schema_id, .updated_at, .updated_by, .volume_id) else . end)' out.plan2.$DATABRICKS_BUNDLE_ENGINE.json > tmp.json && mv tmp.json out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/independent/out.test.toml b/acceptance/bundle/resources/independent/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/independent/out.test.toml +++ b/acceptance/bundle/resources/independent/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/instance_pools/databricks.yml b/acceptance/bundle/resources/instance_pools/databricks.yml new file mode 100644 index 00000000000..dd58ba4b0d0 --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test_instance_pool + +resources: + instance_pools: + test_instance_pool: + instance_pool_name: my_instance_pool + node_type_id: Standard_DS3_v2 + min_idle_instances: 0 + max_capacity: 5 + idle_instance_autotermination_minutes: 60 diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/instance_pools/output.txt b/acceptance/bundle/resources/instance_pools/output.txt new file mode 100644 index 00000000000..30381ea5b70 --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/output.txt @@ -0,0 +1,135 @@ + +>>> [CLI] bundle validate +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "test_instance_pool": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "my_instance_pool", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the create request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/create"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/create", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "my_instance_pool", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + +=== Update the instance pool name +>>> update_file.py databricks.yml my_instance_pool my_instance_pool_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the update request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/edit", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_id": "[UUID]", + "instance_pool_name": "my_instance_pool_2", + "max_capacity": 5, + "min_idle_instances": 0, + "node_type_id": "Standard_DS3_v2" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool_2 + URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[NUMID] + +=== Destroy the instance pool +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.instance_pools.test_instance_pool + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default + +Deleting files... +Destroy complete! + +=== Verify the destroy request +>>> jq select(.method == "POST" and (.path | contains("/instance-pools/delete"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/instance-pools/delete", + "body": { + "instance_pool_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_instance_pool +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_instance_pool/default +Resources: + Instance Pools: + test_instance_pool: + Name: my_instance_pool_2 + URL: (not deployed) + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/instance_pools/script b/acceptance/bundle/resources/instance_pools/script new file mode 100644 index 00000000000..208a61abc1b --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/script @@ -0,0 +1,33 @@ +trace $CLI bundle validate +trace $CLI bundle validate -o json | jq ".resources.instance_pools" + +trace $CLI bundle summary + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Verify the create request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/create")))' out.requests.txt + +trace $CLI bundle summary + +title "Update the instance pool name" +trace update_file.py databricks.yml my_instance_pool my_instance_pool_2 +trace $CLI bundle deploy + +title "Verify the update request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/edit")))' out.requests.txt + +trace $CLI bundle summary + +title "Destroy the instance pool" +trace $CLI bundle destroy --auto-approve + +title "Verify the destroy request" +trace jq 'select(.method == "POST" and (.path | contains("/instance-pools/delete")))' out.requests.txt + +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/instance_pools/test.toml b/acceptance/bundle/resources/instance_pools/test.toml new file mode 100644 index 00000000000..96ce9798d0c --- /dev/null +++ b/acceptance/bundle/resources/instance_pools/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/job_runs/basic/out.test.toml +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml +++ b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/alert-task/out.test.toml b/acceptance/bundle/resources/jobs/alert-task/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/jobs/alert-task/out.test.toml +++ b/acceptance/bundle/resources/jobs/alert-task/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index 31759042f71..71970b719d4 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/create-error/out.test.toml b/acceptance/bundle/resources/jobs/create-error/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/jobs/create-error/out.test.toml +++ b/acceptance/bundle/resources/jobs/create-error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.test.toml b/acceptance/bundle/resources/jobs/delete_job/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_job/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index 500d0d44799..8ffbd40f24c 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/num_workers/out.test.toml +++ b/acceptance/bundle/resources/jobs/num_workers/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index 500d0d44799..8ffbd40f24c 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml +++ b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/task-source/out.test.toml b/acceptance/bundle/resources/jobs/task-source/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.test.toml +++ b/acceptance/bundle/resources/jobs/task-source/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml index dce3f0dca64..65156e0457c 100644 --- a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml +++ b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/jobs/update/out.state.terraform.json b/acceptance/bundle/resources/jobs/update/out.state.terraform.json index 02634009fad..e7c11656ae2 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.terraform.json +++ b/acceptance/bundle/resources/jobs/update/out.state.terraform.json @@ -61,6 +61,7 @@ "cluster_name": "", "custom_tags": null, "data_security_mode": "", + "dependency_mode": "", "docker_image": [], "driver_instance_pool_id": "", "driver_node_type_flexibility": [], @@ -90,7 +91,8 @@ "worker_node_type_flexibility": [], "workload_type": [] } - ] + ], + "serverless_compute_id": "" } ], "library": [], @@ -102,6 +104,7 @@ "notebook_task": [], "notification_settings": [], "parameter": [], + "parent_path": null, "performance_target": null, "pipeline_task": [], "provider_config": [ diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index 500d0d44799..8ffbd40f24c 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml +++ b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml index 37502cdd88f..12ab4ea7f78 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml @@ -2,7 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml index 31759042f71..71970b719d4 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml index 6db629e7043..7845c49f70c 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index c0589419173..581c975b773 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -2,7 +2,5 @@ Local = true Cloud = true CloudSlow = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml index f0ae7c3ccb0..887e4650a78 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index a379f8f6b3e..9b877617211 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RunsOnDbr = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index a379f8f6b3e..9b877617211 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RunsOnDbr = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/databricks.yml b/acceptance/bundle/resources/permissions/jobs/update/databricks.yml index f92bf6da15a..33b69c81894 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/databricks.yml +++ b/acceptance/bundle/resources/permissions/jobs/update/databricks.yml @@ -15,4 +15,4 @@ resources: user_name: viewer@example.com # PERMISSIONS - level: CAN_MANAGE # DELETE_ONE group_name: data-team # DELETE_ONE - # permissions: [] # EXPLICIT_EMPTY + # permissions: [] # EXPLICIT_EMPTY diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 0eec9bab6fc..57ee99c53f4 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -78,7 +78,7 @@ "label": "${resources.jobs.job_with_permissions.id}" } ], - "action": "skip", + "action": "delete", "remote_state": { "object_id": "/jobs/[JOB_WITH_PERMISSIONS_ID]", "__embed__": [ diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json index 08d1d1cf5ef..dea726555ec 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json @@ -5,7 +5,7 @@ "action": "skip" }, "resources.jobs.job_with_permissions.permissions": { - "action": "skip" + "action": "delete" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json index f66a05564c6..73495bdc38f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json @@ -1,15 +1,3 @@ -{ - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json index e69de29bb2d..95d354f99a4 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json @@ -0,0 +1,12 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/output.txt b/acceptance/bundle/resources/permissions/jobs/update/output.txt index 71e0e37f64d..24f9debf98c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/output.txt @@ -127,7 +127,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - user_name: viewer@example.com # PERMISSIONS - - level: CAN_MANAGE # DELETE_ONE - group_name: data-team # DELETE_ONE - # permissions: [] # EXPLICIT_EMPTY + # permissions: [] # EXPLICIT_EMPTY >>> [CLI] bundle plan -o json @@ -145,16 +145,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Set permissions: [] >>> [CLI] bundle plan -o json -Warning: unknown field: permissions - at resources.jobs.job_with_permissions.permissions[1] - in databricks.yml:18:11 - >>> [CLI] bundle deploy -Warning: unknown field: permissions - at resources.jobs.job_with_permissions.permissions[1] - in databricks.yml:18:11 - Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/files... Deploying resources... Updating deployment state... @@ -163,10 +155,6 @@ Deployment complete! >>> print_requests.py //jobs/ >>> [CLI] bundle destroy --auto-approve -Warning: unknown field: permissions - at resources.jobs.job_with_permissions.permissions[1] - in databricks.yml:18:11 - The following resources will be deleted: delete resources.jobs.job_with_permissions diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 647d085e598..38ac1a5b62e 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -42,7 +42,7 @@ trace print_requests.py //jobs/ > out.requests_restore_original.json trace $CLI bundle plan title "Set permissions: []\n" -grep -v '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml +grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' trace $CLI bundle plan -o json > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/out.test.toml b/acceptance/bundle/resources/permissions/out.test.toml index f164c442a11..be193812ec2 100644 --- a/acceptance/bundle/resources/permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false Phase = 1 -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 12a35fcfd86..85a16ad6a38 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -257,11 +257,12 @@ DIFF jobs/update/out.requests_delete_all.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -DIFF jobs/update/out.requests_destroy.direct.json ---- jobs/update/out.requests_destroy.direct.json -+++ jobs/update/out.requests_destroy.terraform.json -@@ -1,4 +1,16 @@ - [ +EXACT jobs/update/out.requests_destroy.direct.json +DIFF jobs/update/out.requests_set_empty.direct.json +--- jobs/update/out.requests_set_empty.direct.json ++++ jobs/update/out.requests_set_empty.terraform.json +@@ -1 +1,14 @@ +-[]+[ + { + "body": { + "access_control_list": [ @@ -273,11 +274,8 @@ DIFF jobs/update/out.requests_destroy.direct.json + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" -+ }, - { - "body": { - "job_id": "[JOB_WITH_PERMISSIONS_ID]" -EXACT jobs/update/out.requests_set_empty.direct.json ++ } ++] MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml index 4cc38e465ef..7ffa3d15391 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml index 4cc38e465ef..7ffa3d15391 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml index 4cc38e465ef..7ffa3d15391 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index 0e160d3e05c..5ad0addb75e 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml +++ b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml +++ b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index d18ae00ed55..6e3397efe53 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/update/out.test.toml b/acceptance/bundle/resources/pipelines/update/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/pipelines/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.no_change.direct.json index ff7df005d4e..35984abba61 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.no_change.direct.json @@ -9,7 +9,9 @@ "remote_state": { "branch_id": "dev-branch", "create_time": "[TIMESTAMP]", + "is_protected": false, "name": "[DEV_BRANCH_ID]", + "no_expiry": true, "parent": "projects/test-pg-proj-[UNIQUE_NAME]", "status": { "branch_id": "dev-branch", @@ -23,19 +25,5 @@ }, "uid": "[BRANCH_UID]", "update_time": "[TIMESTAMP]" - }, - "changes": { - "is_protected": { - "action": "skip", - "reason": "empty", - "old": false, - "new": false - }, - "no_expiry": { - "action": "skip", - "reason": "spec:input_only", - "old": true, - "new": true - } } } diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.direct.json index 64acdc88a68..8c543fd4230 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.direct.json @@ -17,7 +17,9 @@ "remote_state": { "branch_id": "dev-branch", "create_time": "[TIMESTAMP]", + "is_protected": true, "name": "[DEV_BRANCH_ID]", + "no_expiry": true, "parent": "projects/test-pg-proj-[UNIQUE_NAME]", "status": { "branch_id": "dev-branch", @@ -36,13 +38,8 @@ "is_protected": { "action": "update", "old": true, - "new": false - }, - "no_expiry": { - "action": "skip", - "reason": "spec:input_only", - "old": true, - "new": true + "new": false, + "remote": true } } } diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.update.direct.json index 6cc645dede6..d05e254629a 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.update.direct.json @@ -17,7 +17,9 @@ "remote_state": { "branch_id": "dev-branch", "create_time": "[TIMESTAMP]", + "is_protected": false, "name": "[DEV_BRANCH_ID]", + "no_expiry": true, "parent": "projects/test-pg-proj-[UNIQUE_NAME]", "status": { "branch_id": "dev-branch", @@ -36,13 +38,8 @@ "is_protected": { "action": "update", "old": false, - "new": true - }, - "no_expiry": { - "action": "skip", - "reason": "spec:input_only", - "old": true, - "new": true + "new": true, + "remote": false } } } diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/databricks.yml.tmpl deleted file mode 100644 index 9a4d3ed0dfa..00000000000 --- a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/databricks.yml.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -bundle: - name: missing-role-$UNIQUE_NAME - -sync: - paths: [] - -resources: - postgres_projects: - my_project: - project_id: test-pg-proj-$UNIQUE_NAME - display_name: "Missing-role error test" - pg_version: 16 - history_retention_duration: "604800s" - - postgres_branches: - main: - parent: ${resources.postgres_projects.my_project.id} - branch_id: main - no_expiry: true - - postgres_databases: - my_database: - parent: ${resources.postgres_branches.main.id} - database_id: my-database - postgres_database: app_db - # Deliberately omitting `role`; the live API rejects this with 400. diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/out.test.toml deleted file mode 100644 index 7ace96bd3a1..00000000000 --- a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/out.test.toml +++ /dev/null @@ -1,8 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false -CloudEnvs.azure = false -CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/output.txt b/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/output.txt deleted file mode 100644 index 65bf64d8e9d..00000000000 --- a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/output.txt +++ /dev/null @@ -1,28 +0,0 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/missing-role-[UNIQUE_NAME]/default/files... -Deploying resources... -Error: cannot create resources.postgres_databases.my_database: Field 'database.spec.role' is required, expected non-default value (not "")! (400 INVALID_PARAMETER_VALUE) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases?database_id=my-database -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Field 'database.spec.role' is required, expected non-default value (not "")! - -Updating deployment state... - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.postgres_branches.main - delete resources.postgres_projects.my_project - -This action will result in the deletion of the following Lakebase projects along with -all their branches, databases, and endpoints. All data stored in them will be permanently lost: - delete resources.postgres_projects.my_project - -This action will result in the deletion of the following Lakebase branches. -All data stored in them will be permanently lost: - delete resources.postgres_branches.main - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/missing-role-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/script b/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/script deleted file mode 100644 index a58467d6532..00000000000 --- a/acceptance/bundle/resources/postgres_databases/live_errors/missing_role/script +++ /dev/null @@ -1,8 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -musterr $CLI bundle deploy 2>&1 | contains.py "Field 'database.spec.role' is required, expected non-default value" diff --git a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/update/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_databases/update/out.plan.no_change.direct.json index 0f36efeb891..29c2d77a396 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_databases/update/out.plan.no_change.direct.json @@ -15,25 +15,13 @@ "database_id": "my-database", "name": "[MY_DATABASE_ID]", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_database": "initial_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", "status": { "database_id": "my-database", "postgres_database": "initial_db_name", "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" }, "update_time": "[TIMESTAMP]" - }, - "changes": { - "postgres_database": { - "action": "skip", - "reason": "spec:input_only", - "old": "initial_db_name", - "new": "initial_db_name" - }, - "role": { - "action": "skip", - "reason": "spec:input_only", - "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "new": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" - } } } diff --git a/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.direct.json index cb7b036c5eb..fdcd3a5c407 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.direct.json @@ -23,6 +23,8 @@ "database_id": "my-database", "name": "[MY_DATABASE_ID]", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_database": "renamed_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", "status": { "database_id": "my-database", "postgres_database": "renamed_db_name", @@ -34,13 +36,8 @@ "postgres_database": { "action": "update", "old": "renamed_db_name", - "new": "initial_db_name" - }, - "role": { - "action": "skip", - "reason": "spec:input_only", - "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "new": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + "new": "initial_db_name", + "remote": "renamed_db_name" } } } diff --git a/acceptance/bundle/resources/postgres_databases/update/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_databases/update/out.plan.update.direct.json index be6e9f20978..07f974b0725 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_databases/update/out.plan.update.direct.json @@ -23,6 +23,8 @@ "database_id": "my-database", "name": "[MY_DATABASE_ID]", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_database": "initial_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", "status": { "database_id": "my-database", "postgres_database": "initial_db_name", @@ -34,13 +36,8 @@ "postgres_database": { "action": "update", "old": "initial_db_name", - "new": "renamed_db_name" - }, - "role": { - "action": "skip", - "reason": "spec:input_only", - "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "new": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + "new": "renamed_db_name", + "remote": "initial_db_name" } } } diff --git a/acceptance/bundle/resources/postgres_databases/update/out.test.toml b/acceptance/bundle/resources/postgres_databases/update/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/update/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.no_change.direct.json index 7320ab94146..a8d0e19c543 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.no_change.direct.json @@ -5,32 +5,5 @@ "label": "${resources.postgres_branches.main.id}" } ], - "action": "skip", - "changes": { - "autoscaling_limit_max_cu": { - "action": "skip", - "reason": "spec:input_only", - "old": 8, - "new": 8 - }, - "autoscaling_limit_min_cu": { - "action": "skip", - "reason": "spec:input_only", - "old": 0.5, - "new": 0.5 - }, - "endpoint_type": { - "action": "skip", - "reason": "spec:input_only", - "old": "ENDPOINT_TYPE_READ_ONLY", - "new": "ENDPOINT_TYPE_READ_ONLY", - "remote": "" - }, - "suspend_timeout_duration": { - "action": "skip", - "reason": "empty", - "old": "300s", - "new": "300s" - } - } + "action": "skip" } diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json index 6aec05f1661..080ab2d3b9d 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.direct.json @@ -20,26 +20,8 @@ "autoscaling_limit_max_cu": { "action": "update", "old": 4, - "new": 8 - }, - "autoscaling_limit_min_cu": { - "action": "skip", - "reason": "spec:input_only", - "old": 0.5, - "new": 0.5 - }, - "endpoint_type": { - "action": "skip", - "reason": "spec:input_only", - "old": "ENDPOINT_TYPE_READ_ONLY", - "new": "ENDPOINT_TYPE_READ_ONLY", - "remote": "" - }, - "suspend_timeout_duration": { - "action": "skip", - "reason": "empty", - "old": "300s", - "new": "300s" + "new": 8, + "remote": 4 } } } diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.update.direct.json index 3d74497e283..018d27c54ac 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.update.direct.json @@ -20,26 +20,8 @@ "autoscaling_limit_max_cu": { "action": "update", "old": 8, - "new": 4 - }, - "autoscaling_limit_min_cu": { - "action": "skip", - "reason": "spec:input_only", - "old": 0.5, - "new": 0.5 - }, - "endpoint_type": { - "action": "skip", - "reason": "spec:input_only", - "old": "ENDPOINT_TYPE_READ_ONLY", - "new": "ENDPOINT_TYPE_READ_ONLY", - "remote": "" - }, - "suspend_timeout_duration": { - "action": "skip", - "reason": "empty", - "old": "300s", - "new": "300s" + "new": 4, + "remote": 8 } } } diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json index 02eada6bb5c..f5fe890073b 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.no_change.direct.json @@ -2,7 +2,15 @@ "action": "skip", "remote_state": { "create_time": "[TIMESTAMP]", + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Original Name", + "history_retention_duration": "604800s", "name": "[MY_PROJECT_ID]", + "pg_version": 16, "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], @@ -22,39 +30,5 @@ }, "uid": "[UUID]", "update_time": "[TIMESTAMP]" - }, - "changes": { - "default_endpoint_settings": { - "action": "skip", - "reason": "spec:input_only", - "old": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - }, - "new": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - } - }, - "display_name": { - "action": "skip", - "reason": "spec:input_only", - "old": "Original Name", - "new": "Original Name" - }, - "history_retention_duration": { - "action": "skip", - "reason": "empty", - "old": "604800s", - "new": "604800s" - }, - "pg_version": { - "action": "skip", - "reason": "spec:input_only", - "old": 16, - "new": 16 - } } } diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json index de9277d15f0..b7db638b8bf 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.direct.json @@ -15,7 +15,15 @@ }, "remote_state": { "create_time": "[TIMESTAMP]", + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Updated Name", + "history_retention_duration": "604800s", "name": "[MY_PROJECT_ID]", + "pg_version": 16, "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], @@ -37,36 +45,11 @@ "update_time": "[TIMESTAMP]" }, "changes": { - "default_endpoint_settings": { - "action": "skip", - "reason": "spec:input_only", - "old": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - }, - "new": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - } - }, "display_name": { "action": "update", "old": "Updated Name", - "new": "Original Name" - }, - "history_retention_duration": { - "action": "skip", - "reason": "empty", - "old": "604800s", - "new": "604800s" - }, - "pg_version": { - "action": "skip", - "reason": "spec:input_only", - "old": 16, - "new": 16 + "new": "Original Name", + "remote": "Updated Name" } } } diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json index d6307318bc8..b53d02beefd 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.update.direct.json @@ -15,7 +15,15 @@ }, "remote_state": { "create_time": "[TIMESTAMP]", + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Original Name", + "history_retention_duration": "604800s", "name": "[MY_PROJECT_ID]", + "pg_version": 16, "project_id": "test-pg-proj-[UNIQUE_NAME]", "status": { "branch_logical_size_limit_bytes": [NUMID], @@ -37,36 +45,11 @@ "update_time": "[TIMESTAMP]" }, "changes": { - "default_endpoint_settings": { - "action": "skip", - "reason": "spec:input_only", - "old": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - }, - "new": { - "autoscaling_limit_max_cu": 4, - "autoscaling_limit_min_cu": 0.5, - "suspend_timeout_duration": "300s" - } - }, "display_name": { "action": "update", "old": "Original Name", - "new": "Updated Name" - }, - "history_retention_duration": { - "action": "skip", - "reason": "empty", - "old": "604800s", - "new": "604800s" - }, - "pg_version": { - "action": "skip", - "reason": "spec:input_only", - "old": 16, - "new": 16 + "new": "Updated Name", + "remote": "Original Name" } } } diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index 0f311c53285..c5b8e7c8a71 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index 0f311c53285..c5b8e7c8a71 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml index 7ace96bd3a1..4fe23e297fe 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/update/out.plan.no_change.direct.json b/acceptance/bundle/resources/postgres_roles/update/out.plan.no_change.direct.json index 16cc6bd6a72..ec34567c0b6 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.plan.no_change.direct.json +++ b/acceptance/bundle/resources/postgres_roles/update/out.plan.no_change.direct.json @@ -7,9 +7,13 @@ ], "action": "skip", "remote_state": { + "attributes": { + "createdb": false + }, "create_time": "[TIMESTAMP]", "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_role": "app_role", "role_id": "test-role", "status": { "attributes": { @@ -23,23 +27,5 @@ "role_id": "test-role" }, "update_time": "[TIMESTAMP]" - }, - "changes": { - "attributes": { - "action": "skip", - "reason": "spec:input_only", - "old": { - "createdb": false - }, - "new": { - "createdb": false - } - }, - "postgres_role": { - "action": "skip", - "reason": "spec:input_only", - "old": "app_role", - "new": "app_role" - } } } diff --git a/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.direct.json b/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.direct.json index e6ff6664b92..d27c44c829b 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.direct.json +++ b/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.direct.json @@ -17,9 +17,13 @@ } }, "remote_state": { + "attributes": { + "createdb": true + }, "create_time": "[TIMESTAMP]", "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_role": "app_role", "role_id": "test-role", "status": { "attributes": { @@ -35,25 +39,11 @@ "update_time": "[TIMESTAMP]" }, "changes": { - "attributes": { - "action": "update", - "old": { - "createdb": true - }, - "new": { - "createdb": false - } - }, "attributes.createdb": { "action": "update", "old": true, - "new": false - }, - "postgres_role": { - "action": "skip", - "reason": "spec:input_only", - "old": "app_role", - "new": "app_role" + "new": false, + "remote": true } } } diff --git a/acceptance/bundle/resources/postgres_roles/update/out.plan.update.direct.json b/acceptance/bundle/resources/postgres_roles/update/out.plan.update.direct.json index 76b67ee2de5..908d69a00c6 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.plan.update.direct.json +++ b/acceptance/bundle/resources/postgres_roles/update/out.plan.update.direct.json @@ -17,9 +17,13 @@ } }, "remote_state": { + "attributes": { + "createdb": false + }, "create_time": "[TIMESTAMP]", "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "postgres_role": "app_role", "role_id": "test-role", "status": { "attributes": { @@ -35,25 +39,11 @@ "update_time": "[TIMESTAMP]" }, "changes": { - "attributes": { - "action": "update", - "old": { - "createdb": false - }, - "new": { - "createdb": true - } - }, "attributes.createdb": { "action": "update", "old": false, - "new": true - }, - "postgres_role": { - "action": "skip", - "reason": "spec:input_only", - "old": "app_role", - "new": "app_role" + "new": true, + "remote": false } } } diff --git a/acceptance/bundle/resources/postgres_roles/update/out.test.toml b/acceptance/bundle/resources/postgres_roles/update/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/update/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 4631717463a..110f841fa05 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index d18ae00ed55..6e3397efe53 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt b/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt index 2be28866986..e71b4e4fd2b 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt @@ -14,7 +14,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan --output json { "action": "skip", - "reason": "output_only", + "reason": "spec:output_only", "remote": true } diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index d18ae00ed55..6e3397efe53 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/out.test.toml b/acceptance/bundle/resources/schemas/recreate/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/schemas/recreate/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 12fedd022e4..a7175e28eda 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -71,7 +71,7 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "catalog_name": "newmain", "catalog_type": "MANAGED_CATALOG", "comment": "COMMENT1", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", "effective_predictive_optimization_flag": { "inherited_from_name": "[METASTORE_NAME]", @@ -83,7 +83,5 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" } diff --git a/acceptance/bundle/resources/schemas/recreate/script b/acceptance/bundle/resources/schemas/recreate/script index ef6ca3aa8e6..11b4e91c010 100644 --- a/acceptance/bundle/resources/schemas/recreate/script +++ b/acceptance/bundle/resources/schemas/recreate/script @@ -11,5 +11,6 @@ trace $CLI bundle deploy --auto-approve trace print_requests.py //unity trace musterr $CLI schemas get main.myschema -trace $CLI schemas get newmain.myschema +# updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them (the managed properties map is pruned automatically). +trace $CLI schemas get newmain.myschema | normalize_uc_payload.py updated_at updated_by rm out.requests.txt diff --git a/acceptance/bundle/resources/schemas/update/out.test.toml b/acceptance/bundle/resources/schemas/update/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/schemas/update/out.test.toml +++ b/acceptance/bundle/resources/schemas/update/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 670d22860b7..6fc644d5164 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index 4b54db5c50f..6b858c4df47 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml index 724dfe98dff..5bbfaf5e65a 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true CloudSlow = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt index 4379e7528b8..b6f0e4b538f 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt @@ -21,7 +21,8 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] @@ -51,7 +52,8 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml index 61052183da3..91e30e807cf 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false CloudSlow = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml index 0cf47bd2230..d0abd00ab97 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false CloudSlow = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt index b4551a6ba04..002a36777dd 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt @@ -21,7 +21,8 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] @@ -39,7 +40,8 @@ Deployment complete! >>> print_requests.py //sql/warehouses { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] @@ -57,7 +59,8 @@ Deployment complete! >>> print_requests.py //sql/warehouses { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml index 0cf47bd2230..d0abd00ab97 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false CloudSlow = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt index ee2c137c200..21c9214f16b 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt @@ -42,11 +42,13 @@ Deployment complete! >>> print_requests.py //sql/warehouses { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] @@ -67,7 +69,8 @@ Deployment complete! >>> print_requests.py //sql/warehouses { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] @@ -85,7 +88,8 @@ Deployment complete! >>> print_requests.py //sql/warehouses { "method": "POST", - "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start" + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/start", + "body": {} } >>> errcode [CLI] warehouses get [WAREHOUSE_ID] diff --git a/acceptance/bundle/resources/sql_warehouses/out.test.toml b/acceptance/bundle/resources/sql_warehouses/out.test.toml index 16f980b5c68..355ae0775bc 100644 --- a/acceptance/bundle/resources/sql_warehouses/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false CloudSlow = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index 74697b346de..e991fce9180 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -2,7 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml index 4e6b374da1b..c777e3ce206 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 3ea42e3c79c..159efe02696 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,8 +1 @@ RecordRequests = true - -# These tests exercise platform-independent bundle logic (resource CRUD, diffing, -# permissions) and behave identically on every OS, so on pull requests they run on -# Linux only to keep the windows/macOS CI jobs fast. They still run on every OS on -# push to main. Individual OS-sensitive tests below re-enable themselves. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml index 5e66915254d..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml index dacaf56fd57..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index 0612e293703..48203e833cd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index 86181a94195..af665307d39 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index 86181a94195..af665307d39 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index 0612e293703..48203e833cd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml index 0612e293703..48203e833cd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index 86181a94195..af665307d39 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml index 0612e293703..48203e833cd 100644 --- a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/change-comment/out.test.toml b/acceptance/bundle/resources/volumes/change-comment/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/change-comment/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-comment/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-name/out.test.toml b/acceptance/bundle/resources/volumes/change-name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index d18ae00ed55..6e3397efe53 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -2,6 +2,4 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml index 6c76c91ec1b..8c738f635ac 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml @@ -1,8 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml index 51f171b6132..e849ec85ace 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index 73723e9ff8b..b5403a291f8 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -17,7 +17,12 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt echo "=== Plan on no-op redeploy: mixed-case names are skipped, not recreated" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + # UC injects managed-default properties on the schema via an async system write + # seconds after create, so the schema's `properties` change is present + # nondeterministically. Prune it with normalize_uc_payload.py before reshaping so + # the golden is stable. See #6027 and #6040. $CLI bundle plan -o json \ + | normalize_uc_payload.py \ | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/root/env-not-a-directory/notadir.txt b/acceptance/bundle/root/env-not-a-directory/notadir.txt new file mode 100644 index 00000000000..c6f4f7ff362 --- /dev/null +++ b/acceptance/bundle/root/env-not-a-directory/notadir.txt @@ -0,0 +1 @@ +This file exists so DATABRICKS_BUNDLE_ROOT can point at a non-directory. diff --git a/acceptance/bundle/root/env-not-a-directory/out.test.toml b/acceptance/bundle/root/env-not-a-directory/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/root/env-not-a-directory/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/env-not-a-directory/output.txt b/acceptance/bundle/root/env-not-a-directory/output.txt new file mode 100644 index 00000000000..8b5cb62e60a --- /dev/null +++ b/acceptance/bundle/root/env-not-a-directory/output.txt @@ -0,0 +1,7 @@ + +>>> DATABRICKS_BUNDLE_ROOT=notadir.txt [CLI] bundle validate +Error: Invalid bundle root DATABRICKS_BUNDLE_ROOT="notadir.txt": not a directory + +The DATABRICKS_BUNDLE_ROOT environment variable must point to an existing directory that contains a bundle configuration file. + +Found 1 error diff --git a/acceptance/bundle/root/env-not-a-directory/script b/acceptance/bundle/root/env-not-a-directory/script new file mode 100644 index 00000000000..2e32754f685 --- /dev/null +++ b/acceptance/bundle/root/env-not-a-directory/script @@ -0,0 +1,2 @@ +# DATABRICKS_BUNDLE_ROOT points at a file rather than a directory. +musterr trace DATABRICKS_BUNDLE_ROOT=notadir.txt $CLI bundle validate diff --git a/acceptance/bundle/root/env-not-found/out.test.toml b/acceptance/bundle/root/env-not-found/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/root/env-not-found/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/env-not-found/output.txt b/acceptance/bundle/root/env-not-found/output.txt new file mode 100644 index 00000000000..96a2b6eaa6a --- /dev/null +++ b/acceptance/bundle/root/env-not-found/output.txt @@ -0,0 +1,7 @@ + +>>> DATABRICKS_BUNDLE_ROOT=doesnotexist [CLI] bundle validate +Error: Invalid bundle root DATABRICKS_BUNDLE_ROOT="doesnotexist": stat doesnotexist: no such file or directory + +The DATABRICKS_BUNDLE_ROOT environment variable must point to an existing directory that contains a bundle configuration file. + +Found 1 error diff --git a/acceptance/bundle/root/env-not-found/script b/acceptance/bundle/root/env-not-found/script new file mode 100644 index 00000000000..0ec0aa8be94 --- /dev/null +++ b/acceptance/bundle/root/env-not-found/script @@ -0,0 +1,2 @@ +# DATABRICKS_BUNDLE_ROOT points at a path that does not exist: os.Stat fails. +musterr trace DATABRICKS_BUNDLE_ROOT=doesnotexist $CLI bundle validate diff --git a/acceptance/bundle/root/env-not-found/test.toml b/acceptance/bundle/root/env-not-found/test.toml new file mode 100644 index 00000000000..ff98f4a0103 --- /dev/null +++ b/acceptance/bundle/root/env-not-found/test.toml @@ -0,0 +1,4 @@ +# Normalize the os.Stat error message from Windows. +[[Repls]] +Old = "GetFileAttributesEx doesnotexist: The system cannot find the file specified." +New = "stat doesnotexist: no such file or directory" diff --git a/acceptance/bundle/root/not-found/out.test.toml b/acceptance/bundle/root/not-found/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/root/not-found/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/not-found/output.txt b/acceptance/bundle/root/not-found/output.txt new file mode 100644 index 00000000000..92659e3f642 --- /dev/null +++ b/acceptance/bundle/root/not-found/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate +Error: Unable to locate the bundle root: databricks.yml not found. + +Run this command from a directory that contains a bundle, or set the DATABRICKS_BUNDLE_ROOT environment variable to the bundle root directory. +Alternatively, create a new bundle with 'databricks bundle init'. + +Found 1 error diff --git a/acceptance/bundle/root/not-found/script b/acceptance/bundle/root/not-found/script new file mode 100644 index 00000000000..2b0f66adc1c --- /dev/null +++ b/acceptance/bundle/root/not-found/script @@ -0,0 +1,2 @@ +# No databricks.yml in this directory or any parent: getRootWithTraversal fails. +musterr trace $CLI bundle validate diff --git a/acceptance/bundle/root/real-empty-dir/out.test.toml b/acceptance/bundle/root/real-empty-dir/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/root/real-empty-dir/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/real-empty-dir/output.txt b/acceptance/bundle/root/real-empty-dir/output.txt new file mode 100644 index 00000000000..ef8ba923919 --- /dev/null +++ b/acceptance/bundle/root/real-empty-dir/output.txt @@ -0,0 +1,7 @@ + +>>> DATABRICKS_BUNDLE_ROOT=emptydir [CLI] bundle validate +Error: Invalid bundle root DATABRICKS_BUNDLE_ROOT="emptydir": databricks.yml not found + +The DATABRICKS_BUNDLE_ROOT environment variable must point to an existing directory that contains a bundle configuration file. + +Found 1 error diff --git a/acceptance/bundle/root/real-empty-dir/script b/acceptance/bundle/root/real-empty-dir/script new file mode 100644 index 00000000000..e533074c03c --- /dev/null +++ b/acceptance/bundle/root/real-empty-dir/script @@ -0,0 +1,3 @@ +# DATABRICKS_BUNDLE_ROOT points at an existing directory that has no databricks.yml. +mkdir emptydir +musterr trace DATABRICKS_BUNDLE_ROOT=emptydir $CLI bundle validate diff --git a/acceptance/bundle/root/real-empty-dir/test.toml b/acceptance/bundle/root/real-empty-dir/test.toml new file mode 100644 index 00000000000..0ca19575506 --- /dev/null +++ b/acceptance/bundle/root/real-empty-dir/test.toml @@ -0,0 +1 @@ +Ignore = ["emptydir"] diff --git a/acceptance/bundle/root/test.toml b/acceptance/bundle/root/test.toml new file mode 100644 index 00000000000..347d428e8c5 --- /dev/null +++ b/acceptance/bundle/root/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# These errors originate in bundle/root.go before any engine is selected, +# so they are identical for both deployment engines. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run/inline-script/no-bundle/output.txt b/acceptance/bundle/run/inline-script/no-bundle/output.txt index 870f682634f..d8c327966c1 100644 --- a/acceptance/bundle/run/inline-script/no-bundle/output.txt +++ b/acceptance/bundle/run/inline-script/no-bundle/output.txt @@ -1,6 +1,9 @@ >>> [CLI] bundle run -- echo hello -Error: unable to locate bundle root: databricks.yml not found +Error: Unable to locate the bundle root: databricks.yml not found. + +Run this command from a directory that contains a bundle, or set the DATABRICKS_BUNDLE_ROOT environment variable to the bundle root directory. +Alternatively, create a new bundle with 'databricks bundle init'. Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/no-separator/output.txt b/acceptance/bundle/run/inline-script/no-separator/output.txt index 7a35cab72e4..a592daf3a53 100644 --- a/acceptance/bundle/run/inline-script/no-separator/output.txt +++ b/acceptance/bundle/run/inline-script/no-separator/output.txt @@ -1,6 +1,9 @@ >>> [CLI] bundle run echo hello -Error: unable to locate bundle root: databricks.yml not found +Error: Unable to locate the bundle root: databricks.yml not found. + +Run this command from a directory that contains a bundle, or set the DATABRICKS_BUNDLE_ROOT environment variable to the bundle root directory. +Alternatively, create a new bundle with 'databricks bundle init'. Exit code: 1 diff --git a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/dashboard_embed/out.test.toml b/acceptance/bundle/run_as/dashboard_embed/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/dashboard_embed/out.test.toml +++ b/acceptance/bundle/run_as/dashboard_embed/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_override/out.test.toml b/acceptance/bundle/run_as/empty_override/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_override/out.test.toml +++ b/acceptance/bundle/run_as/empty_override/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as/out.test.toml b/acceptance/bundle/run_as/empty_run_as/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_run_as/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_sp/out.test.toml b/acceptance/bundle/run_as/empty_sp/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_sp/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user/out.test.toml b/acceptance/bundle/run_as/empty_user/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_user/out.test.toml +++ b/acceptance/bundle/run_as/empty_user/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml +++ b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/job_default/out.test.toml b/acceptance/bundle/run_as/job_default/out.test.toml index 39941902726..9cfad3fb0d5 100644 --- a/acceptance/bundle/run_as/job_default/out.test.toml +++ b/acceptance/bundle/run_as/job_default/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run_as/model_serving_different/out.test.toml b/acceptance/bundle/run_as/model_serving_different/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/model_serving_different/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_different/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/model_serving_matching/out.test.toml b/acceptance/bundle/run_as/model_serving_matching/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/model_serving_matching/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_matching/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/out.test.toml +++ b/acceptance/bundle/run_as/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml +++ b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/test.toml b/acceptance/bundle/run_as/test.toml deleted file mode 100644 index 588038827f2..00000000000 --- a/acceptance/bundle/run_as/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# These tests exercise platform-independent bundle logic (run_as identity -# resolution) and behave identically on every OS, so on pull requests they run on -# Linux only to keep the windows/macOS CI jobs fast. They still run on every OS on -# push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/state/bad_env/out.test.toml b/acceptance/bundle/state/bad_env/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/bad_env/out.test.toml +++ b/acceptance/bundle/state/bad_env/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/basic/out.test.toml b/acceptance/bundle/state/basic/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/basic/out.test.toml +++ b/acceptance/bundle/state/basic/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/engine_default/out.test.toml b/acceptance/bundle/state/engine_default/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/state/engine_default/out.test.toml +++ b/acceptance/bundle/state/engine_default/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml deleted file mode 100644 index 74e061d27e7..00000000000 --- a/acceptance/bundle/state/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# These tests exercise platform-independent bundle logic (deployment state -# tracking and lineage) and behave identically on every OS, so on pull requests they -# run on Linux only to keep the windows/macOS CI jobs fast. They still run on every -# OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml +++ b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/modified_status/out.test.toml b/acceptance/bundle/summary/modified_status/out.test.toml index 6eccde12289..7f4e2c0ca80 100644 --- a/acceptance/bundle/summary/modified_status/out.test.toml +++ b/acceptance/bundle/summary/modified_status/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"] diff --git a/acceptance/bundle/summary/test.toml b/acceptance/bundle/summary/test.toml deleted file mode 100644 index 1a027567a9a..00000000000 --- a/acceptance/bundle/summary/test.toml +++ /dev/null @@ -1,5 +0,0 @@ -# These tests exercise platform-independent bundle logic (summary rendering) and -# behave identically on every OS, so on pull requests they run on Linux only to keep -# the windows/macOS CI jobs fast. They still run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 255f7408030..4e136c6838f 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false GOOS.windows = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index 4742e08f771..f7c4cf648a9 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false GOOS.windows = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index 4742e08f771..f7c4cf648a9 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false GOOS.windows = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index 4742e08f771..f7c4cf648a9 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false GOOS.windows = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.engine.direct.txt b/acceptance/bundle/telemetry/deploy/out.engine.direct.txt new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.engine.direct.txt @@ -0,0 +1 @@ +[] diff --git a/acceptance/bundle/telemetry/deploy/out.engine.terraform.txt b/acceptance/bundle/telemetry/deploy/out.engine.terraform.txt new file mode 100644 index 00000000000..60cb8fdc893 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.engine.terraform.txt @@ -0,0 +1,6 @@ +[ + { + "key": "engine_terraform_env", + "value": true + } +] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/output.txt b/acceptance/bundle/telemetry/deploy/output.txt index 98e1d643081..e99a645bfa5 100644 --- a/acceptance/bundle/telemetry/deploy/output.txt +++ b/acceptance/bundle/telemetry/deploy/output.txt @@ -11,4 +11,5 @@ Deployment complete! >>> cat telemetry.json true -=== Assert the dry-run migration to the direct engine is reported \ No newline at end of file +=== Assert the dry-run migration to the direct engine is reported +=== Assert the terraform engine opt-in is reported \ No newline at end of file diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index feb65287206..557034032f3 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -25,12 +25,19 @@ cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources title "Assert the dry-run migration to the direct engine is reported" cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values[] | select(.key | startswith("direct_drymigrate_"))]' > out.migration.$DATABRICKS_BUNDLE_ENGINE.txt +# engine_terraform_env reflects DATABRICKS_BUNDLE_ENGINE, which the acceptance +# matrix sets per variant, so it diverges across the matrix. Capture the +# engine_terraform_* keys per-engine and drop them from the engine-agnostic +# out.telemetry.txt below. +title "Assert the terraform engine opt-in is reported" +cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values[] | select(.key | startswith("engine_terraform_"))]' > out.engine.$DATABRICKS_BUNDLE_ENGINE.txt + # bundle_mutator_execution_time_ms can have variable number of entries depending upon the runtime of the mutators. Thus we omit it from # being asserted here. # # upload_file_count and upload_file_sizes are asserted here and kept deterministic by # the sync.paths restriction in databricks.yml (see the comment there). -cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("direct_drymigrate_") | not))' > out.telemetry.txt +cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | (startswith("direct_drymigrate_") or startswith("engine_terraform_")) | not))' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 0e2247eeced..14453c07f92 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,13 +1,6 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] -# These tests assert the telemetry payload emitted on deploy; the output is -# OS-normalized (see the (linux|darwin|windows) -> [OS] replacement below), so it -# behaves identically on every OS. On pull requests they run on Linux only to keep -# the windows/macOS CI jobs fast; they still run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false - [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/AGENTS.md b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/CLAUDE.md b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-minimal/python/input.json b/acceptance/bundle/templates/default-minimal/python/input.json new file mode 100644 index 00000000000..c620c3159b5 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/input.json @@ -0,0 +1,4 @@ +{ + "project_name": "my_default_minimal", + "language_choice": "python" +} diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/output.txt b/acceptance/bundle/templates/default-minimal/python/output.txt similarity index 100% rename from acceptance/bundle/templates/default-minimal/output.txt rename to acceptance/bundle/templates/default-minimal/python/output.txt diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/__builtins__.pyi similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/__builtins__.pyi rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/__builtins__.pyi diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json similarity index 53% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/settings.json similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/.vscode/settings.json rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/.vscode/settings.json diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/AGENTS.md b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/CLAUDE.md b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md similarity index 87% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md index 4304687ae33..8af9ac7e000 100644 --- a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/README.md +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/README.md @@ -2,8 +2,11 @@ The 'my_default_minimal' project was generated by using the default-minimal template. -* `src/`: SQL source code for this project. +* `src/`: Python source code for this project. * `resources/`: Resource configurations (jobs, pipelines, etc.) +* `tests/`: Unit tests for the shared Python code. +* `fixtures/`: Fixtures for data sets (primarily used for testing). + ## Getting started @@ -52,3 +55,8 @@ with this project. It's also possible to interact with it directly using the CLI ``` $ databricks bundle run ``` + +5. Finally, to run tests locally, use `pytest`: + ``` + $ uv run pytest + ``` diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/databricks.yml similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/databricks.yml rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/databricks.yml diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep new file mode 100644 index 00000000000..77a906614cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/fixtures/.gitkeep @@ -0,0 +1,9 @@ +# Test fixtures directory + +Add JSON or CSV files here. In tests, use them with `load_fixture()`: + +``` +def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert len(data) >= 1 +``` diff --git a/acceptance/bundle/templates/default-minimal/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-minimal/output/my_default_minimal/out.gitignore rename to acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/out.gitignore diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml new file mode 100644 index 00000000000..4678f9c4368 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "my_default_minimal" +version = "0.0.1" +authors = [{ name = "[USERNAME]" }] +requires-python = ">=3.10,<3.13" +dependencies = [ + # Any dependencies for jobs and pipelines in this project can be added here + # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies + # + # LIMITATION: for pipelines, dependencies are cached during development; + # add dependencies to the 'environment' section of your pipeline.yml file instead +] + +[dependency-groups] +dev = [ + "pytest", + "ruff", + "databricks-dlt", + "databricks-connect>=15.4,<15.5", + "ipykernel", +] + +[project.scripts] +main = "my_default_minimal.main:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src"] + +[tool.ruff] +line-length = 120 diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py new file mode 100644 index 00000000000..72ebfeb5666 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/tests/conftest.py @@ -0,0 +1,94 @@ +"""This file configures pytest, initializes Databricks Connect, and provides fixtures for Spark and loading test data.""" + +import os, sys, pathlib +from contextlib import contextmanager + + +try: + from databricks.connect import DatabricksSession + from databricks.sdk import WorkspaceClient + from pyspark.sql import SparkSession + import pytest + import json + import csv + import os +except ImportError: + raise ImportError( + "Test dependencies not found.\n\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv." + ) + + +@pytest.fixture() +def spark() -> SparkSession: + """Provide a SparkSession fixture for tests. + + Minimal example: + def test_uses_spark(spark): + df = spark.createDataFrame([(1,)], ["x"]) + assert df.count() == 1 + """ + return DatabricksSession.builder.getOrCreate() + + +@pytest.fixture() +def load_fixture(spark: SparkSession): + """Provide a callable to load JSON or CSV from fixtures/ directory. + + Example usage: + + def test_using_fixture(load_fixture): + data = load_fixture("my_data.json") + assert data.count() >= 1 + """ + + def _loader(filename: str): + path = pathlib.Path(__file__).parent.parent / "fixtures" / filename + suffix = path.suffix.lower() + if suffix == ".json": + rows = json.loads(path.read_text()) + return spark.createDataFrame(rows) + if suffix == ".csv": + with path.open(newline="") as f: + rows = list(csv.DictReader(f)) + return spark.createDataFrame(rows) + raise ValueError(f"Unsupported fixture type for: {filename}") + + return _loader + + +def _enable_fallback_compute(): + """Enable serverless compute if no compute is specified.""" + conf = WorkspaceClient().config + if conf.serverless_compute_id or conf.cluster_id or os.environ.get("SPARK_REMOTE"): + return + + url = "https://docs.databricks.com/dev-tools/databricks-connect/cluster-config" + print("☁️ no compute specified, falling back to serverless compute", file=sys.stderr) + print(f" see {url} for manual configuration", file=sys.stdout) + + os.environ["DATABRICKS_SERVERLESS_COMPUTE_ID"] = "auto" + + +@contextmanager +def _allow_stderr_output(config: pytest.Config): + """Temporarily disable pytest output capture.""" + capman = config.pluginmanager.get_plugin("capturemanager") + if capman: + with capman.global_and_fixture_disabled(): + yield + else: + yield + + +def pytest_configure(config: pytest.Config): + """Configure pytest session.""" + with _allow_stderr_output(config): + _enable_fallback_compute() + + # Initialize Spark session eagerly, so it is available even when + # SparkSession.builder.getOrCreate() is used. For DB Connect 15+, + # we validate version compatibility with the remote cluster. + if hasattr(DatabricksSession.builder, "validateSession"): + DatabricksSession.builder.validateSession().getOrCreate() + else: + DatabricksSession.builder.getOrCreate() diff --git a/acceptance/bundle/templates/default-minimal/script b/acceptance/bundle/templates/default-minimal/python/script similarity index 100% rename from acceptance/bundle/templates/default-minimal/script rename to acceptance/bundle/templates/default-minimal/python/script diff --git a/acceptance/bundle/templates/default-minimal/skip/input.json b/acceptance/bundle/templates/default-minimal/skip/input.json new file mode 100644 index 00000000000..b66e679822d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/input.json @@ -0,0 +1,4 @@ +{ + "project_name": "my_default_minimal", + "language_choice": "skip" +} diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/output.txt b/acceptance/bundle/templates/default-minimal/skip/output.txt new file mode 100644 index 00000000000..06eb69b9a11 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle init default-minimal --config-file ./input.json --output-dir output +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [DATABRICKS_URL] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle validate -t prod +Name: my_default_minimal +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/prod + +Validation OK! diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json similarity index 53% rename from libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json rename to acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json new file mode 100644 index 00000000000..d3e814885cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "**/.gitkeep": "markdown" + }, +} diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/AGENTS.md b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/CLAUDE.md b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md new file mode 100644 index 00000000000..420f11cf53b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/README.md @@ -0,0 +1,47 @@ +# my_default_minimal + +The 'my_default_minimal' project was generated by using the default-minimal template. + +* `src/`: Any source code for this project. +* `resources/`: Resource configurations (jobs, pipelines, etc.) + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml new file mode 100644 index 00000000000..ec72bde8b31 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/databricks.yml @@ -0,0 +1,41 @@ +# This is a Declarative Automation Bundle definition for my_default_minimal. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_minimal + uuid: [UUID] + +include: + - resources/*.yml + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore new file mode 100644 index 00000000000..622a8811a7b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/output/my_default_minimal/out.gitignore @@ -0,0 +1,5 @@ +.databricks/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-minimal/skip/script b/acceptance/bundle/templates/default-minimal/skip/script new file mode 100644 index 00000000000..bce20c6ed9d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/skip/script @@ -0,0 +1,16 @@ +trace $CLI bundle init default-minimal --config-file ./input.json --output-dir output + +cd output/my_default_minimal + +# Verify that empty directories are preserved +[ -d "src" ] || exit 1 +[ -d "resources" ] || exit 1 + +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm -r .databricks + +cd ../../ diff --git a/acceptance/bundle/templates/default-minimal/input.json b/acceptance/bundle/templates/default-minimal/sql/input.json similarity index 100% rename from acceptance/bundle/templates/default-minimal/input.json rename to acceptance/bundle/templates/default-minimal/sql/input.json diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/output.txt b/acceptance/bundle/templates/default-minimal/sql/output.txt new file mode 100644 index 00000000000..06eb69b9a11 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle init default-minimal --config-file ./input.json --output-dir output +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [DATABRICKS_URL] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle validate -t prod +Name: my_default_minimal +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/prod + +Validation OK! diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json new file mode 100644 index 00000000000..1f39c330871 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json new file mode 100644 index 00000000000..d3e814885cb --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "**/.gitkeep": "markdown" + }, +} diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/AGENTS.md b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/CLAUDE.md b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md new file mode 100644 index 00000000000..ace4bdc7d74 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/README.md @@ -0,0 +1,47 @@ +# my_default_minimal + +The 'my_default_minimal' project was generated by using the default-minimal template. + +* `src/`: SQL source code for this project. +* `resources/`: Resource configurations (jobs, pipelines, etc.) + +## Getting started + +Choose how you want to work on this project: + +(a) Directly in your Databricks workspace, see + https://docs.databricks.com/dev-tools/bundles/workspace. + +(b) Locally with an IDE like Cursor or VS Code, see + https://docs.databricks.com/dev-tools/vscode-ext.html. + +(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +# Using this project using the CLI + +The Databricks workspace and IDE extensions provide a graphical interface for working +with this project. It's also possible to interact with it directly using the CLI: + +1. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +2. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + +3. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml new file mode 100644 index 00000000000..ec72bde8b31 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/databricks.yml @@ -0,0 +1,41 @@ +# This is a Declarative Automation Bundle definition for my_default_minimal. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_minimal + uuid: [UUID] + +include: + - resources/*.yml + +# Variable declarations. These variables are assigned in the dev/prod targets below. +variables: + catalog: + description: The catalog to use + schema: + description: The schema to use + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + variables: + catalog: hive_metastore + schema: ${workspace.current_user.short_name} + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + variables: + catalog: hive_metastore + schema: prod + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore new file mode 100644 index 00000000000..622a8811a7b --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/output/my_default_minimal/out.gitignore @@ -0,0 +1,5 @@ +.databricks/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-minimal/sql/script b/acceptance/bundle/templates/default-minimal/sql/script new file mode 100644 index 00000000000..bce20c6ed9d --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/sql/script @@ -0,0 +1,16 @@ +trace $CLI bundle init default-minimal --config-file ./input.json --output-dir output + +cd output/my_default_minimal + +# Verify that empty directories are preserved +[ -d "src" ] || exit 1 +[ -d "resources" ] || exit 1 + +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm -r .databricks + +cd ../../ diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/AGENTS.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/CLAUDE.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index ac6ddc1e6d9..9f9b4934ffe 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index ac6ddc1e6d9..9f9b4934ffe 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,7 +1,5 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/test.toml b/acceptance/bundle/templates/default-python/combinations/test.toml index 8344b5d48a7..d851f0a5bed 100644 --- a/acceptance/bundle/templates/default-python/combinations/test.toml +++ b/acceptance/bundle/templates/default-python/combinations/test.toml @@ -1,13 +1,5 @@ Cloud = true -# This matrix (notebook/DLT/python inclusion) tests OS-independent bundle logic; the -# only OS-specific surface is the wheel build (uv venv layout, bdist_wheel), which is -# covered on every OS by templates/default-python/integration_classic. So on pull -# requests these run on Linux only to keep the windows/macOS CI jobs fast; they still -# run on every OS on push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false - Ignore = ["input.json"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/AGENTS.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/CLAUDE.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-scala/output/my_default_scala/AGENTS.md b/acceptance/bundle/templates/default-scala/output/my_default_scala/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-scala/output/my_default_scala/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-scala/output/my_default_scala/CLAUDE.md b/acceptance/bundle/templates/default-scala/output/my_default_scala/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-scala/output/my_default_scala/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/AGENTS.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/CLAUDE.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/AGENTS.md b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/CLAUDE.md b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc5..00000000000 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json index d73c73b5705..d3e814885cb 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/AGENTS.md b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/CLAUDE.md b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore index e566c51f740..622a8811a7b 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/out.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml index eea1c50a649..184b495a64a 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml @@ -18,7 +18,7 @@ dev = [ "databricks-dlt", "databricks-connect>=15.4,<15.5", "ipykernel", - "databricks-bundles==x.y.z", + "databricks-bundles==0.0.0", ] [project.scripts] diff --git a/acceptance/bundle/templates/pydabs/test.toml b/acceptance/bundle/templates/pydabs/test.toml index fc66c79de7c..a7b1c1bcb78 100644 --- a/acceptance/bundle/templates/pydabs/test.toml +++ b/acceptance/bundle/templates/pydabs/test.toml @@ -4,6 +4,10 @@ TimeoutWindows = '120s' Local = true Cloud = false +# Mask the real databricks-bundles version so the golden pyproject.toml does not +# churn on every release. The placeholder must stay valid PEP 440: Dependabot's +# dependency-graph submission scans this generated manifest and aborts the whole +# pip/uv graph job on an unparseable version (the old "x.y.z" did exactly that). [[Repls]] Old = '"databricks-bundles==\d+\.\d+\.\d+"' -New = '"databricks-bundles==x.y.z"' +New = '"databricks-bundles==0.0.0"' diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 167b459b2e9..3fe73103691 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -41,8 +41,7 @@ OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' -MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat' -MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat' +MISS deploy.terraform /api/2.0/preview/scim/v2/Me 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /api/2.1/unity-catalog/schemas 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' @@ -94,7 +93,6 @@ MISS plan.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks- MISS plan.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' OK plan.terraform /api/2.0/workspace/get-status engine/terraform MISS plan.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' -MISS plan.terraform /api/2.0/preview/scim/v2/Me 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat' MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index 22d78dca557..9bfe34ca4dd 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -342,19 +342,7 @@ { "headers": { "User-Agent": [ - "databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me", - "q": { - "excludedAttributes": "entitlements" - } -} -{ - "headers": { - "User-Agent": [ - "databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat" + "databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat" ] }, "method": "GET", diff --git a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json index 6047b9055cc..ba790f18a0a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.plan.terraform.json @@ -55,18 +55,6 @@ "method": "GET", "path": "/.well-known/databricks-config" } -{ - "headers": { - "User-Agent": [ - "databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 auth/pat" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me", - "q": { - "excludedAttributes": "entitlements" - } -} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/anchor_containers/databricks.yml new file mode 100644 index 00000000000..a82d4fe7202 --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/databricks.yml @@ -0,0 +1,48 @@ +bundle: + name: test-bundle + +# This test verifies that anchors grouped in a list or a map do not trigger +# "unknown field" warnings. The suppression recurses into containers whose +# elements are all anchors. A container that also holds a non-anchor element is +# not suppressed: it is reported as an unknown field, since it is more likely a +# misplaced configuration value than a block of reusable anchors. + +task_defaults: &task_defaults + max_retries: 3 + +# List of anchors. +parameter_items: + - &p_first + name: first + default: one + - &p_second + name: second + default: two + +# Map of anchors. +parameter_map: + entry: &p_map_entry + name: first + +# Container nested inside a container: a map whose value is a list of anchors. +nested_parameters: + group: + - &p_nested + name: nested + +# Mixed container: one anchor and one non-anchor element. This is reported as an +# unknown field because not all elements are anchor containers. +mixed_parameters: + - &p_mixed + name: mixed + - name: plain + +resources: + jobs: + my_job: + name: "test job with anchors" + tasks: + - task_key: "main" + <<: *task_defaults + notebook_task: + notebook_path: "/some/notebook" diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/anchor_containers/output.txt b/acceptance/bundle/validate/anchor_containers/output.txt new file mode 100644 index 00000000000..86ae905e85f --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle validate +Warning: unknown field: mixed_parameters + in databricks.yml:35:1 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 warning diff --git a/acceptance/bundle/validate/anchor_containers/script b/acceptance/bundle/validate/anchor_containers/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml index dce3f0dca64..65156e0457c 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/validate/dashboard_defaults/out.test.toml b/acceptance/bundle/validate/dashboard_defaults/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/dashboard_defaults/out.test.toml +++ b/acceptance/bundle/validate/dashboard_defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_name/out.test.toml b/acceptance/bundle/validate/dashboard_required_name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/dashboard_required_name/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml +++ b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/null/out.test.toml b/acceptance/bundle/validate/empty_resources/null/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_resources/null/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/null/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_tasks/out.test.toml b/acceptance/bundle/validate/empty_tasks/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/empty_tasks/out.test.toml +++ b/acceptance/bundle/validate/empty_tasks/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/engine-config-valid/out.test.toml b/acceptance/bundle/validate/engine-config-valid/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/engine-config-valid/out.test.toml +++ b/acceptance/bundle/validate/engine-config-valid/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/engine-config-valid/output.txt b/acceptance/bundle/validate/engine-config-valid/output.txt index e07d8c553f2..253bb571cd6 100644 --- a/acceptance/bundle/validate/engine-config-valid/output.txt +++ b/acceptance/bundle/validate/engine-config-valid/output.txt @@ -9,10 +9,15 @@ Workspace: Validation OK! >>> [CLI] bundle validate -t terraform-target +Warning: the terraform deployment engine is deprecated and will stop working in a future version of the CLI + in databricks.yml:10:15 + +See https://docs.databricks.com/aws/en/dev-tools/bundles/direct for how to migrate to the direct deployment engine + Name: test-engine-config-valid Target: terraform-target Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/test-engine-config-valid/terraform-target -Validation OK! +Found 1 warning diff --git a/acceptance/bundle/validate/enum/out.test.toml b/acceptance/bundle/validate/enum/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/enum/out.test.toml +++ b/acceptance/bundle/validate/enum/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum_resource_refs/out.test.toml b/acceptance/bundle/validate/enum_resource_refs/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/enum_resource_refs/out.test.toml +++ b/acceptance/bundle/validate/enum_resource_refs/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/databricks.yml b/acceptance/bundle/validate/grants_required_principal/databricks.yml index 832ec6ec0bd..9db93a59066 100644 --- a/acceptance/bundle/validate/grants_required_principal/databricks.yml +++ b/acceptance/bundle/validate/grants_required_principal/databricks.yml @@ -6,7 +6,7 @@ resources: my_catalog: name: my_catalog grants: - # Missing principal (warns). + # Missing principal: rejected. - privileges: - ALL_PRIVILEGES schemas: diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/grants_required_principal/out.test.toml +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/output.txt b/acceptance/bundle/validate/grants_required_principal/output.txt index 79629ec21d5..3e838701d52 100644 --- a/acceptance/bundle/validate/grants_required_principal/output.txt +++ b/acceptance/bundle/validate/grants_required_principal/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle validate -Warning: grant principal is required +Error: grant principal is required at resources.catalogs.my_catalog.grants[0] in databricks.yml:10:11 @@ -10,4 +10,12 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Found 1 warning +Found 1 error + +>>> [CLI] bundle deploy +Error: grant principal is required + at resources.catalogs.my_catalog.grants[0] + in databricks.yml:10:11 + + +>>> print_requests.py //unity-catalog diff --git a/acceptance/bundle/validate/grants_required_principal/script b/acceptance/bundle/validate/grants_required_principal/script index 5350876150f..8e28a75e4c7 100644 --- a/acceptance/bundle/validate/grants_required_principal/script +++ b/acceptance/bundle/validate/grants_required_principal/script @@ -1 +1,7 @@ -trace $CLI bundle validate +musterr trace $CLI bundle validate + +# Deploy must abort at validation: no securable is created, so no partial deploy. +musterr trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +rm -f out.requests.txt diff --git a/acceptance/bundle/validate/grants_required_principal/test.toml b/acceptance/bundle/validate/grants_required_principal/test.toml index fc2b3f50667..1af7758fd37 100644 --- a/acceptance/bundle/validate/grants_required_principal/test.toml +++ b/acceptance/bundle/validate/grants_required_principal/test.toml @@ -1,3 +1,5 @@ +RecordRequests = true + # Catalogs and schemas are only supported by the direct engine. [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml +++ b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/include_locations/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/include_locations/out.test.toml +++ b/acceptance/bundle/validate/include_locations/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-target/out.test.toml b/acceptance/bundle/validate/invalid-engine-target/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/invalid-engine-target/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-target/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/job-references/out.test.toml b/acceptance/bundle/validate/job-references/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/job-references/out.test.toml +++ b/acceptance/bundle/validate/job-references/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml +++ b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_conversion/out.test.toml b/acceptance/bundle/validate/model_serving_conversion/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/model_serving_conversion/out.test.toml +++ b/acceptance/bundle/validate/model_serving_conversion/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/missing_name/out.test.toml b/acceptance/bundle/validate/models/missing_name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/models/missing_name/out.test.toml +++ b/acceptance/bundle/validate/models/missing_name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/user_id/out.test.toml b/acceptance/bundle/validate/models/user_id/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/models/user_id/out.test.toml +++ b/acceptance/bundle/validate/models/user_id/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml +++ b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/permissions/out.test.toml b/acceptance/bundle/validate/permissions/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/permissions/out.test.toml +++ b/acceptance/bundle/validate/permissions/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/permissions_overlap/out.test.toml b/acceptance/bundle/validate/permissions_overlap/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/permissions_overlap/out.test.toml +++ b/acceptance/bundle/validate/permissions_overlap/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml +++ b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix/out.test.toml b/acceptance/bundle/validate/presets_name_prefix/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/presets_name_prefix/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml index 1a3e24fa574..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/presets_tags/out.test.toml b/acceptance/bundle/validate/presets_tags/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/presets_tags/out.test.toml +++ b/acceptance/bundle/validate/presets_tags/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/required/out.test.toml b/acceptance/bundle/validate/required/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/required/out.test.toml +++ b/acceptance/bundle/validate/required/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml +++ b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/strict/out.test.toml b/acceptance/bundle/validate/strict/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/strict/out.test.toml +++ b/acceptance/bundle/validate/strict/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sync_patterns/out.test.toml b/acceptance/bundle/validate/sync_patterns/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/sync_patterns/out.test.toml +++ b/acceptance/bundle/validate/sync_patterns/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/test.toml b/acceptance/bundle/validate/test.toml deleted file mode 100644 index fa0df3d789e..00000000000 --- a/acceptance/bundle/validate/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# These tests exercise platform-independent bundle logic (config validation and -# diagnostics) and behave identically on every OS, so on pull requests they run on -# Linux only to keep the windows/macOS CI jobs fast. They still run on every OS on -# push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml +++ b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/volume_defaults/out.test.toml b/acceptance/bundle/validate/volume_defaults/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/validate/volume_defaults/out.test.toml +++ b/acceptance/bundle/validate/volume_defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/arg-repeat/out.test.toml b/acceptance/bundle/variables/arg-repeat/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/arg-repeat/out.test.toml +++ b/acceptance/bundle/variables/arg-repeat/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cross-ref/out.test.toml b/acceptance/bundle/variables/complex-cross-ref/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-cross-ref/out.test.toml +++ b/acceptance/bundle/variables/complex-cross-ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle-self/out.test.toml b/acceptance/bundle/variables/complex-cycle-self/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-cycle-self/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle-self/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle/out.test.toml b/acceptance/bundle/variables/complex-cycle/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-cycle/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-simple/out.test.toml b/acceptance/bundle/variables/complex-simple/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-simple/out.test.toml +++ b/acceptance/bundle/variables/complex-simple/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive/out.test.toml b/acceptance/bundle/variables/complex-transitive/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-transitive/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml +++ b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-within-complex/out.test.toml b/acceptance/bundle/variables/complex-within-complex/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex-within-complex/out.test.toml +++ b/acceptance/bundle/variables/complex-within-complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex/out.test.toml b/acceptance/bundle/variables/complex/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex/out.test.toml +++ b/acceptance/bundle/variables/complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex_multiple_files/out.test.toml b/acceptance/bundle/variables/complex_multiple_files/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/complex_multiple_files/out.test.toml +++ b/acceptance/bundle/variables/complex_multiple_files/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/cycle/out.test.toml b/acceptance/bundle/variables/cycle/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/cycle/out.test.toml +++ b/acceptance/bundle/variables/cycle/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/double_underscore/out.test.toml b/acceptance/bundle/variables/double_underscore/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/double_underscore/out.test.toml +++ b/acceptance/bundle/variables/double_underscore/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/empty/out.test.toml b/acceptance/bundle/variables/empty/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/empty/out.test.toml +++ b/acceptance/bundle/variables/empty/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/env_overrides/out.test.toml b/acceptance/bundle/variables/env_overrides/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/env_overrides/out.test.toml +++ b/acceptance/bundle/variables/env_overrides/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/file-defaults/out.test.toml b/acceptance/bundle/variables/file-defaults/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/file-defaults/out.test.toml +++ b/acceptance/bundle/variables/file-defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/git-branch/out.test.toml b/acceptance/bundle/variables/git-branch/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/git-branch/out.test.toml +++ b/acceptance/bundle/variables/git-branch/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/host/out.test.toml b/acceptance/bundle/variables/host/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/host/out.test.toml +++ b/acceptance/bundle/variables/host/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/int/out.test.toml b/acceptance/bundle/variables/int/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/int/out.test.toml +++ b/acceptance/bundle/variables/int/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_2436/out.test.toml b/acceptance/bundle/variables/issue_2436/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/issue_2436/out.test.toml +++ b/acceptance/bundle/variables/issue_2436/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/lookup/out.test.toml b/acceptance/bundle/variables/lookup/out.test.toml index 5bd2a7c1f08..bbc7fcfd1bd 100644 --- a/acceptance/bundle/variables/lookup/out.test.toml +++ b/acceptance/bundle/variables/lookup/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = true -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml +++ b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-builtin/out.test.toml b/acceptance/bundle/variables/resolve-builtin/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-builtin/out.test.toml +++ b/acceptance/bundle/variables/resolve-builtin/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-empty/out.test.toml b/acceptance/bundle/variables/resolve-empty/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-empty/out.test.toml +++ b/acceptance/bundle/variables/resolve-empty/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml +++ b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml +++ b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml +++ b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml deleted file mode 100644 index a37d2699f6a..00000000000 --- a/acceptance/bundle/variables/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# These tests exercise platform-independent bundle logic (variable resolution and -# interpolation) and behave identically on every OS, so on pull requests they run on -# Linux only to keep the windows/macOS CI jobs fast. They still run on every OS on -# push to main. -GOOSOnPR.windows = false -GOOSOnPR.darwin = false diff --git a/acceptance/bundle/variables/unicode_reference/out.test.toml b/acceptance/bundle/variables/unicode_reference/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/unicode_reference/out.test.toml +++ b/acceptance/bundle/variables/unicode_reference/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/vanilla/out.test.toml b/acceptance/bundle/variables/vanilla/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/vanilla/out.test.toml +++ b/acceptance/bundle/variables/vanilla/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/var_in_var/out.test.toml b/acceptance/bundle/variables/var_in_var/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/var_in_var/out.test.toml +++ b/acceptance/bundle/variables/var_in_var/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml +++ b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml +++ b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/without_definition/out.test.toml b/acceptance/bundle/variables/without_definition/out.test.toml index 67759662971..f784a183258 100644 --- a/acceptance/bundle/variables/without_definition/out.test.toml +++ b/acceptance/bundle/variables/without_definition/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false -GOOSOnPR.darwin = false -GOOSOnPR.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/cmd/account/account-help/output.txt b/acceptance/cmd/account/account-help/output.txt index 2425182699f..0c1f01187cf 100644 --- a/acceptance/cmd/account/account-help/output.txt +++ b/acceptance/cmd/account/account-help/output.txt @@ -55,7 +55,7 @@ OAuth service-principal-secrets These APIs enable administrators to manage service principal secrets. Disaster Recovery - disaster-recovery *Public Preview* Manage disaster recovery configurations and execute failover operations. + disaster-recovery Manage disaster recovery configurations and execute failover operations. Flags: -h, --help help for account diff --git a/acceptance/cmd/bundle/dms-read-only/out.test.toml b/acceptance/cmd/bundle/dms-read-only/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/bundle/dms-read-only/out.test.toml +++ b/acceptance/cmd/bundle/dms-read-only/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/bundle/dms-read-only/test.toml b/acceptance/cmd/bundle/dms-read-only/test.toml index f4984b3c5b7..aa916970fa8 100644 --- a/acceptance/cmd/bundle/dms-read-only/test.toml +++ b/acceptance/cmd/bundle/dms-read-only/test.toml @@ -10,8 +10,9 @@ Local = true Cloud = false # bundle-deployments is a workspace-service command independent of the bundle -# deploy engine, so opt out of the inherited terraform/direct matrix. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +# deploy engine, so pin to a single value instead of the inherited matrix so +# it runs once (empty list would run on both direct and terraform CI runners). +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # Normalize the git commit SHA so the golden output isn't tied to a specific # commit value. diff --git a/acceptance/cmd/sandbox/config/idle-timeout-bounds/out.test.toml b/acceptance/cmd/sandbox/config/idle-timeout-bounds/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/config/idle-timeout-bounds/out.test.toml +++ b/acceptance/cmd/sandbox/config/idle-timeout-bounds/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/config/no-flags/out.test.toml b/acceptance/cmd/sandbox/config/no-flags/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/config/no-flags/out.test.toml +++ b/acceptance/cmd/sandbox/config/no-flags/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/config/update-name/out.test.toml b/acceptance/cmd/sandbox/config/update-name/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/config/update-name/out.test.toml +++ b/acceptance/cmd/sandbox/config/update-name/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/create/with-name/out.test.toml b/acceptance/cmd/sandbox/create/with-name/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/create/with-name/out.test.toml +++ b/acceptance/cmd/sandbox/create/with-name/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/default/not-found/out.test.toml b/acceptance/cmd/sandbox/default/not-found/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/default/not-found/out.test.toml +++ b/acceptance/cmd/sandbox/default/not-found/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/default/set/out.test.toml b/acceptance/cmd/sandbox/default/set/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/default/set/out.test.toml +++ b/acceptance/cmd/sandbox/default/set/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/delete/no-tty-no-auto-approve/out.test.toml b/acceptance/cmd/sandbox/delete/no-tty-no-auto-approve/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/delete/no-tty-no-auto-approve/out.test.toml +++ b/acceptance/cmd/sandbox/delete/no-tty-no-auto-approve/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/delete/not-found/out.test.toml b/acceptance/cmd/sandbox/delete/not-found/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/delete/not-found/out.test.toml +++ b/acceptance/cmd/sandbox/delete/not-found/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/delete/success/out.test.toml b/acceptance/cmd/sandbox/delete/success/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/delete/success/out.test.toml +++ b/acceptance/cmd/sandbox/delete/success/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/list/empty/out.test.toml b/acceptance/cmd/sandbox/list/empty/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/list/empty/out.test.toml +++ b/acceptance/cmd/sandbox/list/empty/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/list/json/out.test.toml b/acceptance/cmd/sandbox/list/json/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/list/json/out.test.toml +++ b/acceptance/cmd/sandbox/list/json/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/list/region-unavailable/out.test.toml b/acceptance/cmd/sandbox/list/region-unavailable/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/list/region-unavailable/out.test.toml +++ b/acceptance/cmd/sandbox/list/region-unavailable/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/list/with-entries/out.test.toml b/acceptance/cmd/sandbox/list/with-entries/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/list/with-entries/out.test.toml +++ b/acceptance/cmd/sandbox/list/with-entries/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/register/success/out.test.toml b/acceptance/cmd/sandbox/register/success/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/register/success/out.test.toml +++ b/acceptance/cmd/sandbox/register/success/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/ssh-key/delete/success/out.test.toml b/acceptance/cmd/sandbox/ssh-key/delete/success/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/ssh-key/delete/success/out.test.toml +++ b/acceptance/cmd/sandbox/ssh-key/delete/success/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/ssh-key/list/empty/out.test.toml b/acceptance/cmd/sandbox/ssh-key/list/empty/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/ssh-key/list/empty/out.test.toml +++ b/acceptance/cmd/sandbox/ssh-key/list/empty/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/ssh-key/list/with-entries/out.test.toml b/acceptance/cmd/sandbox/ssh-key/list/with-entries/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/ssh-key/list/with-entries/out.test.toml +++ b/acceptance/cmd/sandbox/ssh-key/list/with-entries/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/start/already-running/out.test.toml b/acceptance/cmd/sandbox/start/already-running/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/start/already-running/out.test.toml +++ b/acceptance/cmd/sandbox/start/already-running/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/status/json/out.test.toml b/acceptance/cmd/sandbox/status/json/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/status/json/out.test.toml +++ b/acceptance/cmd/sandbox/status/json/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/status/not-found/out.test.toml b/acceptance/cmd/sandbox/status/not-found/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/status/not-found/out.test.toml +++ b/acceptance/cmd/sandbox/status/not-found/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/status/running/out.test.toml b/acceptance/cmd/sandbox/status/running/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/status/running/out.test.toml +++ b/acceptance/cmd/sandbox/status/running/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/stop/success/out.test.toml b/acceptance/cmd/sandbox/stop/success/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/sandbox/stop/success/out.test.toml +++ b/acceptance/cmd/sandbox/stop/success/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/sandbox/test.toml b/acceptance/cmd/sandbox/test.toml index fd492ebe7af..d1e2b7f3550 100644 --- a/acceptance/cmd/sandbox/test.toml +++ b/acceptance/cmd/sandbox/test.toml @@ -5,8 +5,9 @@ # otherwise produce these on every run. Ignore = [".databricks", ".ssh"] -# Sandbox commands are independent of the bundle engine. Override the -# root EnvMatrix so the two engines don't both run in parallel for -# every sandbox test — when they did, they raced each other writing -# to the same ~/.databricks/sandbox.json file. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +# Sandbox commands are independent of the bundle engine. Pin to a single +# value so the two engines don't both run in parallel for every sandbox +# test — when they did, they raced each other writing to the same +# ~/.databricks/sandbox.json file. An empty list would run once locally +# but on both direct and terraform CI runners, defeating the point. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/version/out.test.toml b/acceptance/cmd/version/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/cmd/version/out.test.toml +++ b/acceptance/cmd/version/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/cmd/version/test.toml b/acceptance/cmd/version/test.toml index 97179048cb4..5321e27a6de 100644 --- a/acceptance/cmd/version/test.toml +++ b/acceptance/cmd/version/test.toml @@ -1,3 +1,2 @@ # The update check is not bundle-aware; run it once instead of per-engine. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/cancel/out.test.toml b/acceptance/experimental/air/cancel/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/cancel/out.test.toml +++ b/acceptance/experimental/air/cancel/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/get-ai-runtime/out.test.toml b/acceptance/experimental/air/get-ai-runtime/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/get-ai-runtime/out.test.toml +++ b/acceptance/experimental/air/get-ai-runtime/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/get/out.test.toml b/acceptance/experimental/air/get/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/get/out.test.toml +++ b/acceptance/experimental/air/get/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/help/out.test.toml b/acceptance/experimental/air/help/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/help/out.test.toml +++ b/acceptance/experimental/air/help/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/list/out.test.toml b/acceptance/experimental/air/list/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/list/out.test.toml +++ b/acceptance/experimental/air/list/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/run-submit/out.test.toml b/acceptance/experimental/air/run-submit/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/run-submit/out.test.toml +++ b/acceptance/experimental/air/run-submit/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/run/out.test.toml b/acceptance/experimental/air/run/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/run/out.test.toml +++ b/acceptance/experimental/air/run/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/air/unimplemented/out.test.toml b/acceptance/experimental/air/unimplemented/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/air/unimplemented/out.test.toml +++ b/acceptance/experimental/air/unimplemented/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/install-experimental-empty/out.test.toml b/acceptance/experimental/aitools/skills/install-experimental-empty/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/aitools/skills/install-experimental-empty/out.test.toml +++ b/acceptance/experimental/aitools/skills/install-experimental-empty/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt b/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt index 71fa9e5604a..c1a6249bf76 100644 --- a/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt +++ b/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt @@ -7,6 +7,4 @@ Installing Databricks skills for Claude Code... Using skills version test-ref Fetching skills manifest... Warn: --experimental was set but the manifest at test-ref exposes no experimental skills. Set DATABRICKS_SKILLS_REF to a release or ref that includes them. -Downloading test-stable... -Exposing test-stable to 1 agent... Installed 1 skill. diff --git a/acceptance/experimental/aitools/skills/install-experimental-empty/test.toml b/acceptance/experimental/aitools/skills/install-experimental-empty/test.toml index d60200402db..96fb36a3e73 100644 --- a/acceptance/experimental/aitools/skills/install-experimental-empty/test.toml +++ b/acceptance/experimental/aitools/skills/install-experimental-empty/test.toml @@ -5,8 +5,7 @@ Ignore = [ "home", ] -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # Stable-only manifest (no repo_dir=experimental). [[Server]] diff --git a/acceptance/experimental/aitools/skills/install-specific/out.test.toml b/acceptance/experimental/aitools/skills/install-specific/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/aitools/skills/install-specific/out.test.toml +++ b/acceptance/experimental/aitools/skills/install-specific/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/install-specific/output.txt b/acceptance/experimental/aitools/skills/install-specific/output.txt index 3200ae3727c..dc972dd17d3 100644 --- a/acceptance/experimental/aitools/skills/install-specific/output.txt +++ b/acceptance/experimental/aitools/skills/install-specific/output.txt @@ -6,8 +6,6 @@ Flag --global has been deprecated, use --scope=global Installing Databricks skills for Claude Code... Using skills version test-ref Fetching skills manifest... -Downloading test-stable-a... -Exposing test-stable-a to 1 agent... Installed 1 skill. === install a specific experimental skill @@ -17,8 +15,6 @@ Flag --global has been deprecated, use --scope=global Installing Databricks skills for Claude Code... Using skills version test-ref Fetching skills manifest... -Downloading test-exp... -Exposing test-exp to 1 agent... Installed 1 skill. === asking for an experimental skill without --experimental flag errors out diff --git a/acceptance/experimental/aitools/skills/install-specific/test.toml b/acceptance/experimental/aitools/skills/install-specific/test.toml index 37a42c8441c..28aa4f13a8b 100644 --- a/acceptance/experimental/aitools/skills/install-specific/test.toml +++ b/acceptance/experimental/aitools/skills/install-specific/test.toml @@ -5,8 +5,7 @@ Ignore = [ "home", ] -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [[Server]] Pattern = "GET /test-ref/manifest.json" diff --git a/acceptance/experimental/aitools/skills/install/out.test.toml b/acceptance/experimental/aitools/skills/install/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/aitools/skills/install/out.test.toml +++ b/acceptance/experimental/aitools/skills/install/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/install/output.txt b/acceptance/experimental/aitools/skills/install/output.txt index 77cd8551597..bcd43f22227 100644 --- a/acceptance/experimental/aitools/skills/install/output.txt +++ b/acceptance/experimental/aitools/skills/install/output.txt @@ -6,8 +6,6 @@ Flag --global has been deprecated, use --scope=global Installing Databricks skills for Claude Code... Using skills version test-ref Fetching skills manifest... -Downloading test-stable... -Exposing test-stable to 1 agent... Installed 1 skill. === re-run with --experimental installs the experimental one too @@ -17,8 +15,6 @@ Flag --global has been deprecated, use --scope=global Installing Databricks skills for Claude Code... Using skills version test-ref Fetching skills manifest... -Downloading test-exp... -Exposing test-exp to 1 agent... Installed 2 skills. === no-op re-run is idempotent (no new fetches, no errors) diff --git a/acceptance/experimental/aitools/skills/install/test.toml b/acceptance/experimental/aitools/skills/install/test.toml index bd2bc414ae0..bfc9d34e3e6 100644 --- a/acceptance/experimental/aitools/skills/install/test.toml +++ b/acceptance/experimental/aitools/skills/install/test.toml @@ -6,8 +6,7 @@ Ignore = [ "home", ] -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # Stable + experimental skills via repo_dir. [[Server]] diff --git a/acceptance/experimental/aitools/skills/path-dump/out.test.toml b/acceptance/experimental/aitools/skills/path-dump/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/aitools/skills/path-dump/out.test.toml +++ b/acceptance/experimental/aitools/skills/path-dump/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/path-dump/test.toml b/acceptance/experimental/aitools/skills/path-dump/test.toml index 6cd88669924..b5e1bc4d625 100644 --- a/acceptance/experimental/aitools/skills/path-dump/test.toml +++ b/acceptance/experimental/aitools/skills/path-dump/test.toml @@ -7,8 +7,7 @@ Ignore = [ "dumped", ] -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [[Server]] Pattern = "GET /test-ref/manifest.json" diff --git a/acceptance/experimental/aitools/skills/update-prune/out.test.toml b/acceptance/experimental/aitools/skills/update-prune/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/aitools/skills/update-prune/out.test.toml +++ b/acceptance/experimental/aitools/skills/update-prune/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/update-prune/output.txt b/acceptance/experimental/aitools/skills/update-prune/output.txt index 744e53b0d39..564957813a4 100644 --- a/acceptance/experimental/aitools/skills/update-prune/output.txt +++ b/acceptance/experimental/aitools/skills/update-prune/output.txt @@ -6,17 +6,11 @@ Flag --global has been deprecated, use --scope=global Installing Databricks skills for Cursor... Using skills version test-ref Fetching skills manifest... -Downloading alpha... -Exposing alpha to 1 agent... -Downloading beta... -Exposing beta to 1 agent... Installed 2 skills. === update against a release where beta is gone: alpha updates, beta is pruned >>> DATABRICKS_SKILLS_REF=v2-ref [CLI] experimental aitools update --scope global Command "update" is deprecated, use "databricks aitools update" instead. -Downloading alpha... -Exposing alpha to 1 agent... updated alpha v1.0.0 -> v2.0.0 removed beta (no longer in this release) Updated 1 skill. diff --git a/acceptance/experimental/aitools/skills/update-prune/test.toml b/acceptance/experimental/aitools/skills/update-prune/test.toml index dee7f170173..13ac797b451 100644 --- a/acceptance/experimental/aitools/skills/update-prune/test.toml +++ b/acceptance/experimental/aitools/skills/update-prune/test.toml @@ -6,8 +6,7 @@ Ignore = [ "home", ] -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # Initial release: alpha + beta. [[Server]] diff --git a/acceptance/experimental/genie/ask-deprecated/out.test.toml b/acceptance/experimental/genie/ask-deprecated/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/experimental/genie/ask-deprecated/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/genie/ask-deprecated/output.txt b/acceptance/experimental/genie/ask-deprecated/output.txt new file mode 100644 index 00000000000..475c7538824 --- /dev/null +++ b/acceptance/experimental/genie/ask-deprecated/output.txt @@ -0,0 +1,12 @@ + +=== the deprecated experimental alias still runs but prints a move notice +>>> [CLI] experimental genie ask What are total sales by franchise? +Command "ask" is deprecated, use "databricks genie ask" instead; this experimental alias will be removed in a future release +Alpha leads with 1,250 total sales. + + Total by Franchise + ────────────────── + + Alpha ██████████████████████████████████████████████████ 1,250 + Beta █████████████████████████▌ 640 + diff --git a/acceptance/experimental/genie/ask-deprecated/script b/acceptance/experimental/genie/ask-deprecated/script new file mode 100644 index 00000000000..c3338131a74 --- /dev/null +++ b/acceptance/experimental/genie/ask-deprecated/script @@ -0,0 +1,2 @@ +title "the deprecated experimental alias still runs but prints a move notice" +trace $CLI experimental genie ask "What are total sales by franchise?" diff --git a/acceptance/experimental/genie/ask/test.toml b/acceptance/experimental/genie/ask-deprecated/test.toml similarity index 98% rename from acceptance/experimental/genie/ask/test.toml rename to acceptance/experimental/genie/ask-deprecated/test.toml index 124119ddb56..fa19f460549 100644 --- a/acceptance/experimental/genie/ask/test.toml +++ b/acceptance/experimental/genie/ask-deprecated/test.toml @@ -1,6 +1,5 @@ # No bundle engine needed for this command. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # One canned Genie SSE response covers all invocations in the script. The # stream exercises the full pipeline: a reasoning item, a THOUGHT message, an diff --git a/acceptance/experimental/genie/ask-endpoint-gone/out.test.toml b/acceptance/experimental/genie/ask-endpoint-gone/out.test.toml deleted file mode 100644 index d6187dcb046..00000000000 --- a/acceptance/experimental/genie/ask-endpoint-gone/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/genie/ask-endpoint-gone/script b/acceptance/experimental/genie/ask-endpoint-gone/script deleted file mode 100644 index 7dfda2348e7..00000000000 --- a/acceptance/experimental/genie/ask-endpoint-gone/script +++ /dev/null @@ -1,2 +0,0 @@ -title "a removed endpoint tells the user to update the CLI" -musterr trace $CLI experimental genie ask "What are total sales by franchise?" diff --git a/acceptance/experimental/genie/ask-protocol-drift/out.test.toml b/acceptance/experimental/genie/ask-protocol-drift/out.test.toml deleted file mode 100644 index d6187dcb046..00000000000 --- a/acceptance/experimental/genie/ask-protocol-drift/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/genie/ask-protocol-drift/script b/acceptance/experimental/genie/ask-protocol-drift/script deleted file mode 100644 index 1cc2a7db0d4..00000000000 --- a/acceptance/experimental/genie/ask-protocol-drift/script +++ /dev/null @@ -1,5 +0,0 @@ -title "a drifted protocol with no renderable answer tells the user to update the CLI" -errcode trace $CLI experimental genie ask "What are total sales by franchise?" - -title "json output also reports the drift" -errcode trace $CLI experimental genie ask "What are total sales by franchise?" --output json diff --git a/acceptance/experimental/genie/ask-request-drift/out.test.toml b/acceptance/experimental/genie/ask-request-drift/out.test.toml deleted file mode 100644 index d6187dcb046..00000000000 --- a/acceptance/experimental/genie/ask-request-drift/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/genie/ask-request-drift/script b/acceptance/experimental/genie/ask-request-drift/script deleted file mode 100644 index a0aaffdb615..00000000000 --- a/acceptance/experimental/genie/ask-request-drift/script +++ /dev/null @@ -1,2 +0,0 @@ -title "a 500 with no message points at a possible request format change" -musterr trace $CLI experimental genie ask "What are total sales by franchise?" diff --git a/acceptance/experimental/genie/ask/out.test.toml b/acceptance/experimental/genie/ask/out.test.toml deleted file mode 100644 index d6187dcb046..00000000000 --- a/acceptance/experimental/genie/ask/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/genie/ask/script b/acceptance/experimental/genie/ask/script deleted file mode 100644 index ef0081f1455..00000000000 --- a/acceptance/experimental/genie/ask/script +++ /dev/null @@ -1,15 +0,0 @@ -title "ask renders the final answer and a chart" -trace $CLI experimental genie ask "What are total sales by franchise?" - -title "ask with --include-sql also shows the executed SQL" -trace $CLI experimental genie ask "What are total sales by franchise?" --include-sql - -title "ask with --output json emits structured output" -trace $CLI experimental genie ask "What are total sales by franchise?" --output json - -title "ask with --raw dumps raw SSE events" -trace $CLI experimental genie ask "What are total sales by franchise?" --raw - -title "incompatible flags are rejected" -musterr trace $CLI experimental genie ask "q" --raw --output json -musterr trace $CLI experimental genie ask "q" --raw --include-sql diff --git a/acceptance/experimental/open/out.test.toml b/acceptance/experimental/open/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/experimental/open/out.test.toml +++ b/acceptance/experimental/open/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f634d630fc8..08d3a757c83 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -22,6 +22,7 @@ database_catalogs database_instances experiments genie_spaces +instance_pools jobs model_serving_endpoints models diff --git a/acceptance/experimental/open/test.toml b/acceptance/experimental/open/test.toml index e83f5fafb97..7fd7a8d22a3 100644 --- a/acceptance/experimental/open/test.toml +++ b/acceptance/experimental/open/test.toml @@ -1,3 +1,2 @@ # No bundle engine needed for this command. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/genie/ask-endpoint-gone/out.test.toml b/acceptance/genie/ask-endpoint-gone/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/genie/ask-endpoint-gone/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/genie/ask-endpoint-gone/output.txt b/acceptance/genie/ask-endpoint-gone/output.txt similarity index 82% rename from acceptance/experimental/genie/ask-endpoint-gone/output.txt rename to acceptance/genie/ask-endpoint-gone/output.txt index 9862aba8e86..b6e02a96ce7 100644 --- a/acceptance/experimental/genie/ask-endpoint-gone/output.txt +++ b/acceptance/genie/ask-endpoint-gone/output.txt @@ -1,4 +1,4 @@ === a removed endpoint tells the user to update the CLI ->>> [CLI] experimental genie ask What are total sales by franchise? +>>> [CLI] genie ask What are total sales by franchise? Error: the Genie API is not available on this workspace: No API found for 'POST /data-rooms/tools/onechat/responses'; the endpoint may have moved since this CLI release: update the Databricks CLI to the latest version (run 'databricks version --check') diff --git a/acceptance/genie/ask-endpoint-gone/script b/acceptance/genie/ask-endpoint-gone/script new file mode 100644 index 00000000000..1a30a0e9fa7 --- /dev/null +++ b/acceptance/genie/ask-endpoint-gone/script @@ -0,0 +1,2 @@ +title "a removed endpoint tells the user to update the CLI" +musterr trace $CLI genie ask "What are total sales by franchise?" diff --git a/acceptance/experimental/genie/ask-endpoint-gone/test.toml b/acceptance/genie/ask-endpoint-gone/test.toml similarity index 90% rename from acceptance/experimental/genie/ask-endpoint-gone/test.toml rename to acceptance/genie/ask-endpoint-gone/test.toml index 577d5f6c7d4..9d92fcfb7a1 100644 --- a/acceptance/experimental/genie/ask-endpoint-gone/test.toml +++ b/acceptance/genie/ask-endpoint-gone/test.toml @@ -1,6 +1,5 @@ # No bundle engine needed for this command. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # The genie route is an undocumented API that can disappear between releases. # This is the wire shape a live workspace gateway returns for a removed route. diff --git a/acceptance/genie/ask-protocol-drift/out.test.toml b/acceptance/genie/ask-protocol-drift/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/genie/ask-protocol-drift/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/genie/ask-protocol-drift/output.txt b/acceptance/genie/ask-protocol-drift/output.txt similarity index 85% rename from acceptance/experimental/genie/ask-protocol-drift/output.txt rename to acceptance/genie/ask-protocol-drift/output.txt index 9e071ff783b..00e7f7f5689 100644 --- a/acceptance/experimental/genie/ask-protocol-drift/output.txt +++ b/acceptance/genie/ask-protocol-drift/output.txt @@ -1,12 +1,12 @@ === a drifted protocol with no renderable answer tells the user to update the CLI ->>> [CLI] experimental genie ask What are total sales by franchise? +>>> [CLI] genie ask What are total sales by franchise? Error: the stream ended without an answer (received 2 events); the API may have changed: update the Databricks CLI to the latest version (run 'databricks version --check'), or re-run with --raw to inspect the raw stream Exit code: 1 === json output also reports the drift ->>> [CLI] experimental genie ask What are total sales by franchise? --output json +>>> [CLI] genie ask What are total sales by franchise? --output json { "status": "error", "conversation_id": "conv_1", diff --git a/acceptance/genie/ask-protocol-drift/script b/acceptance/genie/ask-protocol-drift/script new file mode 100644 index 00000000000..e9e5f17c26f --- /dev/null +++ b/acceptance/genie/ask-protocol-drift/script @@ -0,0 +1,5 @@ +title "a drifted protocol with no renderable answer tells the user to update the CLI" +errcode trace $CLI genie ask "What are total sales by franchise?" + +title "json output also reports the drift" +errcode trace $CLI genie ask "What are total sales by franchise?" --output json diff --git a/acceptance/experimental/genie/ask-protocol-drift/test.toml b/acceptance/genie/ask-protocol-drift/test.toml similarity index 92% rename from acceptance/experimental/genie/ask-protocol-drift/test.toml rename to acceptance/genie/ask-protocol-drift/test.toml index 65cd1d283b6..d604382b1ca 100644 --- a/acceptance/experimental/genie/ask-protocol-drift/test.toml +++ b/acceptance/genie/ask-protocol-drift/test.toml @@ -1,6 +1,5 @@ # No bundle engine needed for this command. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # A protocol change that renames item types (or moves the answer elsewhere) # leaves the stream syntactically valid but free of anything this build can diff --git a/acceptance/genie/ask-request-drift/out.test.toml b/acceptance/genie/ask-request-drift/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/genie/ask-request-drift/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/genie/ask-request-drift/output.txt b/acceptance/genie/ask-request-drift/output.txt similarity index 82% rename from acceptance/experimental/genie/ask-request-drift/output.txt rename to acceptance/genie/ask-request-drift/output.txt index e5d0e7bd29a..7be83fde191 100644 --- a/acceptance/experimental/genie/ask-request-drift/output.txt +++ b/acceptance/genie/ask-request-drift/output.txt @@ -1,4 +1,4 @@ === a 500 with no message points at a possible request format change ->>> [CLI] experimental genie ask What are total sales by franchise? +>>> [CLI] genie ask What are total sales by franchise? Error: the Genie backend could not process the request (500 with no details); if this keeps happening, the request format may have changed since this CLI release: update the Databricks CLI to the latest version (run 'databricks version --check') diff --git a/acceptance/genie/ask-request-drift/script b/acceptance/genie/ask-request-drift/script new file mode 100644 index 00000000000..1fa5c84e49d --- /dev/null +++ b/acceptance/genie/ask-request-drift/script @@ -0,0 +1,2 @@ +title "a 500 with no message points at a possible request format change" +musterr trace $CLI genie ask "What are total sales by franchise?" diff --git a/acceptance/experimental/genie/ask-request-drift/test.toml b/acceptance/genie/ask-request-drift/test.toml similarity index 90% rename from acceptance/experimental/genie/ask-request-drift/test.toml rename to acceptance/genie/ask-request-drift/test.toml index 59339f04f5b..952f2fdd918 100644 --- a/acceptance/experimental/genie/ask-request-drift/test.toml +++ b/acceptance/genie/ask-request-drift/test.toml @@ -1,6 +1,5 @@ # No bundle engine needed for this command. -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # Wire shape observed live when the backend cannot interpret the request body # (e.g. the expected request shape changed): a 500 INTERNAL_ERROR with an diff --git a/acceptance/genie/ask/out.test.toml b/acceptance/genie/ask/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/genie/ask/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/genie/ask/output.txt b/acceptance/genie/ask/output.txt similarity index 89% rename from acceptance/experimental/genie/ask/output.txt rename to acceptance/genie/ask/output.txt index 3613bf02ee6..397a096c2fb 100644 --- a/acceptance/experimental/genie/ask/output.txt +++ b/acceptance/genie/ask/output.txt @@ -1,6 +1,6 @@ === ask renders the final answer and a chart ->>> [CLI] experimental genie ask What are total sales by franchise? +>>> [CLI] genie ask What are total sales by franchise? Alpha leads with 1,250 total sales. Total by Franchise @@ -11,7 +11,7 @@ Alpha leads with 1,250 total sales. === ask with --include-sql also shows the executed SQL ->>> [CLI] experimental genie ask What are total sales by franchise? --include-sql +>>> [CLI] genie ask What are total sales by franchise? --include-sql SQL executed (Total by Franchise): SELECT franchise, SUM(total) AS total FROM sales GROUP BY franchise @@ -25,7 +25,7 @@ Alpha leads with 1,250 total sales. === ask with --output json emits structured output ->>> [CLI] experimental genie ask What are total sales by franchise? --output json +>>> [CLI] genie ask What are total sales by franchise? --output json { "status": "completed", "conversation_id": "conv_1", @@ -41,7 +41,7 @@ Alpha leads with 1,250 total sales. } === ask with --raw dumps raw SSE events ->>> [CLI] experimental genie ask What are total sales by franchise? --raw +>>> [CLI] genie ask What are total sales by franchise? --raw {"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"r1","status":"in_progress"}} {"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"t1","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Looking at the sales data...","annotations":[]}],"metadata":{"ui_type":"THOUGHT","source_internal_ids":["m1"]}}} {"type":"response.output_item.added","output_index":2,"item":{"type":"function_call","id":"f1","call_id":"c1","name":"execute_sql","arguments":""}} @@ -52,8 +52,8 @@ Alpha leads with 1,250 total sales. {"type":"response.completed","response":{"id":"resp_1","status":"completed","conversation_id":"conv_1"}} === incompatible flags are rejected ->>> [CLI] experimental genie ask q --raw --output json +>>> [CLI] genie ask q --raw --output json Error: --raw cannot be used with --output json ->>> [CLI] experimental genie ask q --raw --include-sql +>>> [CLI] genie ask q --raw --include-sql Error: --include-sql cannot be used with --raw diff --git a/acceptance/genie/ask/script b/acceptance/genie/ask/script new file mode 100644 index 00000000000..eb95620c77e --- /dev/null +++ b/acceptance/genie/ask/script @@ -0,0 +1,15 @@ +title "ask renders the final answer and a chart" +trace $CLI genie ask "What are total sales by franchise?" + +title "ask with --include-sql also shows the executed SQL" +trace $CLI genie ask "What are total sales by franchise?" --include-sql + +title "ask with --output json emits structured output" +trace $CLI genie ask "What are total sales by franchise?" --output json + +title "ask with --raw dumps raw SSE events" +trace $CLI genie ask "What are total sales by franchise?" --raw + +title "incompatible flags are rejected" +musterr trace $CLI genie ask "q" --raw --output json +musterr trace $CLI genie ask "q" --raw --include-sql diff --git a/acceptance/genie/ask/test.toml b/acceptance/genie/ask/test.toml new file mode 100644 index 00000000000..fa19f460549 --- /dev/null +++ b/acceptance/genie/ask/test.toml @@ -0,0 +1,28 @@ +# No bundle engine needed for this command. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# One canned Genie SSE response covers all invocations in the script. The +# stream exercises the full pipeline: a reasoning item, a THOUGHT message, an +# execute_sql call whose .added event carries empty arguments (filled in on +# .done), the QUERY_EXECUTION result data, a VIZ spec joined to it, the +# FINAL_RESPONSE message with a viz image reference, and the completion event. +[[Server]] +Pattern = "POST /api/2.0/data-rooms/tools/onechat/responses" +Response.Body = ''' +data: {"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"r1","status":"in_progress"}} + +data: {"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"t1","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Looking at the sales data...","annotations":[]}],"metadata":{"ui_type":"THOUGHT","source_internal_ids":["m1"]}}} + +data: {"type":"response.output_item.added","output_index":2,"item":{"type":"function_call","id":"f1","call_id":"c1","name":"execute_sql","arguments":""}} + +data: {"type":"response.output_item.done","output_index":2,"item":{"type":"function_call","id":"f1","call_id":"c1","status":"completed","name":"execute_sql","arguments":"{\"sql\":\"SELECT franchise, SUM(total) AS total FROM sales GROUP BY franchise\",\"title\":\"Total by Franchise\"}"}} + +data: {"type":"response.output_item.done","output_index":3,"item":{"type":"function_call_output","id":"o1","call_id":"c1","status":"completed","metadata":{"ui_type":"QUERY_EXECUTION","statement_id":"stmt1","result_data":{"columns":[{"name":"franchise"},{"name":"total"}],"preview_rows":[["Alpha",1250],["Beta",640]]}}}} + +data: {"type":"response.output_item.done","output_index":4,"item":{"type":"function_call_output","id":"o2","call_id":"c2","status":"completed","metadata":{"ui_type":"VIZ","sql_id":"stmt1","embed_id":"viz_1","viz_definition":"{\"renderSpec\":{\"widgetType\":\"bar\",\"frame\":{\"title\":\"Total by Franchise\"},\"encodings\":{\"x\":{\"fieldName\":\"franchise\"},\"y\":{\"fieldName\":\"total\"}}}}"}}} + +data: {"type":"response.output_item.added","output_index":5,"item":{"type":"message","id":"m1","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Alpha leads with 1,250 total sales.\n\n![Total by Franchise](#viz_1)","annotations":[]}],"metadata":{"ui_type":"FINAL_RESPONSE","source_internal_ids":["x1"]}}} + +data: {"type":"response.completed","response":{"id":"resp_1","status":"completed","conversation_id":"conv_1"}} + +''' diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index c6642670d17..c37d9f87064 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -40,7 +40,7 @@ Vector Search vector-search-indexes **Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries. AI Search - ai-search *Beta* **AI Search Endpoint**: Represents the compute resources to host AI Search indexes. + ai-search *Public Preview* **AI Search Endpoint**: Represents the compute resources to host AI Search indexes. Identity and Access Management current-user *Public Preview* This API allows retrieving information about currently authenticated user or service principal. @@ -56,7 +56,7 @@ Identity and Access Management Databricks SQL alerts The alerts API can be used to perform CRUD operations on alerts. alerts-legacy *Public Preview* The alerts API can be used to perform CRUD operations on alerts. - alerts-v2 *Public Preview* New version of SQL Alerts. + alerts-v2 New version of SQL Alerts. dashboards In general, there is little need to modify dashboards using the API. data-sources This API is provided to assist you in making new query objects. queries The queries API can be used to perform CRUD operations on queries. @@ -137,7 +137,7 @@ Clean Rooms clean-room-asset-revisions *Beta* Clean Room Asset Revisions denote new versions of uploaded assets (e.g. clean-room-assets Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room. clean-room-auto-approval-rules *Beta* Clean room auto-approval rules automatically create an approval on your behalf when an asset (e.g. - clean-room-task-runs Clean room task runs are the executions of notebooks in a clean room. + clean-room-task-runs Clean room task runs are the executions of notebooks and JAR analyses in a clean room. clean-rooms A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other's data. Quality Monitor diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index b2d1a9f578f..f981e790e0b 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -37,12 +37,6 @@ type TestConfig struct { // If absent, default to true. GOOS map[string]bool - // Like GOOS, but only consulted when the test run is triggered by a GitHub - // pull_request event. Each string is compared against runtime.GOOS; if absent, - // default to true. This lets an OS-independent test run on Linux only for pull - // requests while still running on every OS on push to main. - GOOSOnPR map[string]bool - // Which Clouds the test is enabled on. Allowed values: "aws", "azure", "gcp". // If absent, default to true. // Only checked if CLOUD_ENV is not empty. @@ -257,6 +251,22 @@ func validateConfig(t *testing.T, config TestConfig, configPath string) { "Output files must not be ignored.", configPath, pattern) } } + + // Reject EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []. It runs the test once + // locally with the variable unset, but on CI (which splits work by + // filtering ENVFILTER=DATABRICKS_BUNDLE_ENGINE=) the test has no + // engine tag, so checkEnvFilters lets it through on BOTH the direct and + // terraform runners, duplicating the run. Use ["direct"] to pin it to a + // single CI runner, and unset DATABRICKS_BUNDLE_ENGINE in the script if + // the test needs to exercise the default engine. + // + // selftests intentionally exercise the empty-list mechanic and are + // exempt. + if vals, ok := config.EnvMatrix["DATABRICKS_BUNDLE_ENGINE"]; ok && len(vals) == 0 && !strings.Contains(configPath, "selftest/") { + t.Fatalf("Invalid config %s: EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] "+ + "runs on both direct and terraform CI runners. Use "+ + `EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] instead`, configPath) + } } func DoLoadConfig(t *testing.T, path string) TestConfig { diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go index cd6d34739d5..0fa5d43657d 100644 --- a/acceptance/internal/materialized_config.go +++ b/acceptance/internal/materialized_config.go @@ -54,9 +54,6 @@ func GenerateMaterializedConfig(config *TestConfig) string { for _, k := range slices.Sorted(maps.Keys(config.GOOS)) { fmt.Fprintf(&buf, "GOOS.%s = %v\n", k, config.GOOS[k]) } - for _, k := range slices.Sorted(maps.Keys(config.GOOSOnPR)) { - fmt.Fprintf(&buf, "GOOSOnPR.%s = %v\n", k, config.GOOSOnPR[k]) - } for _, k := range slices.Sorted(maps.Keys(config.CloudEnvs)) { fmt.Fprintf(&buf, "CloudEnvs.%s = %v\n", k, config.CloudEnvs[k]) } diff --git a/acceptance/localenv/cluster-name-ambiguous-json/out.test.toml b/acceptance/localenv/cluster-name-ambiguous-json/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous-json/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-name-ambiguous-json/output.txt b/acceptance/localenv/cluster-name-ambiguous-json/output.txt new file mode 100644 index 00000000000..ff557c87dea --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous-json/output.txt @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "command": "environments setup-local", + "ok": false, + "mode": "default", + "dryRun": true, + "greenfield": false, + "phases": [ + { + "phase": "preflight", + "status": "ok" + }, + { + "phase": "resolve", + "status": "error" + }, + { + "phase": "fetch", + "status": "pending" + }, + { + "phase": "merge", + "status": "pending" + }, + { + "phase": "provision", + "status": "pending" + }, + { + "phase": "validate", + "status": "pending" + } + ], + "warnings": [], + "error": { + "code": "E_RESOLVE", + "failurePhase": "resolve", + "message": "resolving cluster name \"dup\": there are 2 active clusters named \"dup\"; use --cluster-id to disambiguate", + "diskMutated": false + }, + "durationMs": 0 +} diff --git a/acceptance/localenv/cluster-name-ambiguous-json/script b/acceptance/localenv/cluster-name-ambiguous-json/script new file mode 100644 index 00000000000..434245a7820 --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous-json/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --cluster-name dup --dry-run --output json diff --git a/acceptance/localenv/cluster-name-ambiguous-json/test.toml b/acceptance/localenv/cluster-name-ambiguous-json/test.toml new file mode 100644 index 00000000000..be79d13bed8 --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous-json/test.toml @@ -0,0 +1,20 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# Two active clusters share the same name, so name→ID resolution is genuinely +# ambiguous and errors as an actionable E_RESOLVE at the resolve phase. +# (A terminated cluster sharing the name would be filtered out server-side via +# ListClustersFilterBy, so it does not cause a spurious collision — see +# GetClusterByName.) +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "dup", "state": "RUNNING", "spark_version": "15.4.x-scala2.12"}, + {"cluster_id": "cid-999", "cluster_name": "dup", "state": "RUNNING", "spark_version": "14.3.x-scala2.12"} + ] +} +''' diff --git a/acceptance/localenv/cluster-name-ambiguous/out.test.toml b/acceptance/localenv/cluster-name-ambiguous/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-name-ambiguous/output.txt b/acceptance/localenv/cluster-name-ambiguous/output.txt new file mode 100644 index 00000000000..ece61571e20 --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous/output.txt @@ -0,0 +1,7 @@ +preflight ok check +resolve error resolving cluster name "dup": there are 2 active clusters named "dup"; use --cluster-id to disambiguate +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/cluster-name-ambiguous/script b/acceptance/localenv/cluster-name-ambiguous/script new file mode 100644 index 00000000000..a7a8d77eb72 --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --cluster-name dup --dry-run diff --git a/acceptance/localenv/cluster-name-ambiguous/test.toml b/acceptance/localenv/cluster-name-ambiguous/test.toml new file mode 100644 index 00000000000..be79d13bed8 --- /dev/null +++ b/acceptance/localenv/cluster-name-ambiguous/test.toml @@ -0,0 +1,20 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# Two active clusters share the same name, so name→ID resolution is genuinely +# ambiguous and errors as an actionable E_RESOLVE at the resolve phase. +# (A terminated cluster sharing the name would be filtered out server-side via +# ListClustersFilterBy, so it does not cause a spurious collision — see +# GetClusterByName.) +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "dup", "state": "RUNNING", "spark_version": "15.4.x-scala2.12"}, + {"cluster_id": "cid-999", "cluster_name": "dup", "state": "RUNNING", "spark_version": "14.3.x-scala2.12"} + ] +} +''' diff --git a/acceptance/localenv/cluster-name-check/out.test.toml b/acceptance/localenv/cluster-name-check/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/cluster-name-check/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-name-check/output.txt b/acceptance/localenv/cluster-name-check/output.txt new file mode 100644 index 00000000000..b8f50099434 --- /dev/null +++ b/acceptance/localenv/cluster-name-check/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] environments setup-local --cluster-name my-cluster --dry-run +preflight ok check +resolve ok source=cluster envKey=dbr/15.4.x-scala2.12 +fetch ok source=[DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml fromCache=false +merge ok +provision ok +validate ok +Plan: [TEST_TMP_DIR]/pyproject.toml + changed region: requires-python + changed region: tool.uv.constraint-dependencies + changed region: databricks-connect +Check complete. No files were modified. diff --git a/acceptance/localenv/cluster-name-check/script b/acceptance/localenv/cluster-name-check/script new file mode 100644 index 00000000000..ab8b82e1d02 --- /dev/null +++ b/acceptance/localenv/cluster-name-check/script @@ -0,0 +1 @@ +trace $CLI environments setup-local --cluster-name my-cluster --dry-run diff --git a/acceptance/localenv/cluster-name-check/test.toml b/acceptance/localenv/cluster-name-check/test.toml new file mode 100644 index 00000000000..0a49bc20a41 --- /dev/null +++ b/acceptance/localenv/cluster-name-check/test.toml @@ -0,0 +1,34 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# --cluster-name resolves the name to an ID via the Clusters API (which lists +# clusters and matches on name), then resolves identically to --cluster-id. +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "my-cluster", "spark_version": "15.4.x-scala2.12"}, + {"cluster_id": "cid-999", "cluster_name": "other-cluster", "spark_version": "14.3.x-scala2.12"} + ] +} +''' + +[[Server]] +Pattern = "GET /dbr/15.4.x-scala2.12/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +''' + +[[Repls]] +Old = 'uv uv \S+(?: \([^)]+\))?' +New = 'uv [UV_VERSION]' diff --git a/acceptance/localenv/cluster-name-unknown/out.test.toml b/acceptance/localenv/cluster-name-unknown/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/cluster-name-unknown/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-name-unknown/output.txt b/acceptance/localenv/cluster-name-unknown/output.txt new file mode 100644 index 00000000000..6a158bb005a --- /dev/null +++ b/acceptance/localenv/cluster-name-unknown/output.txt @@ -0,0 +1,7 @@ +preflight ok check +resolve error resolving cluster name "nope": no active cluster named "nope" +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/cluster-name-unknown/script b/acceptance/localenv/cluster-name-unknown/script new file mode 100644 index 00000000000..96ae54a868e --- /dev/null +++ b/acceptance/localenv/cluster-name-unknown/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --cluster-name nope --dry-run diff --git a/acceptance/localenv/cluster-name-unknown/test.toml b/acceptance/localenv/cluster-name-unknown/test.toml new file mode 100644 index 00000000000..154e747bbd5 --- /dev/null +++ b/acceptance/localenv/cluster-name-unknown/test.toml @@ -0,0 +1,18 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# No active cluster matches the requested name, so name→ID resolution finds +# nothing and errors as an actionable E_RESOLVE at the resolve phase. This is +# the unknown-name counterpart to cluster-name-ambiguous; the two share the +# single wrapping branch in GetClusterByName. +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "my-cluster", "state": "RUNNING", "spark_version": "15.4.x-scala2.12"} + ] +} +''' diff --git a/acceptance/localenv/constraints-only/out.test.toml b/acceptance/localenv/constraints-only/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/constraints-only/out.test.toml +++ b/acceptance/localenv/constraints-only/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/constraints-only/output.txt b/acceptance/localenv/constraints-only/output.txt index d2f12be5db6..a85265f8ccc 100644 --- a/acceptance/localenv/constraints-only/output.txt +++ b/acceptance/localenv/constraints-only/output.txt @@ -1,8 +1,8 @@ ->>> [CLI] local-env python sync --serverless v4 --constraints-only --check --output json +>>> [CLI] environments setup-local --serverless-version 4 --constraints-only --dry-run --output json { "schemaVersion": 1, - "command": "local-env python sync", + "command": "environments setup-local", "ok": true, "mode": "constraints-only", "dryRun": true, @@ -19,7 +19,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,15 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+# managed by databricks local-env python sync — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks local-env python sync\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,15 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/constraints-only/script b/acceptance/localenv/constraints-only/script index 354eac49963..b4f777e7980 100644 --- a/acceptance/localenv/constraints-only/script +++ b/acceptance/localenv/constraints-only/script @@ -1 +1 @@ -trace $CLI local-env python sync --serverless v4 --constraints-only --check --output json +trace $CLI environments setup-local --serverless-version 4 --constraints-only --dry-run --output json diff --git a/acceptance/localenv/constraints-only/test.toml b/acceptance/localenv/constraints-only/test.toml index e2b12fc95ba..c636f7a9cef 100644 --- a/acceptance/localenv/constraints-only/test.toml +++ b/acceptance/localenv/constraints-only/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" diff --git a/acceptance/localenv/env-unsupported/out.test.toml b/acceptance/localenv/env-unsupported/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/env-unsupported/out.test.toml +++ b/acceptance/localenv/env-unsupported/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/env-unsupported/output.txt b/acceptance/localenv/env-unsupported/output.txt index 0dbb61fee4d..c13d1a1d7d8 100644 --- a/acceptance/localenv/env-unsupported/output.txt +++ b/acceptance/localenv/env-unsupported/output.txt @@ -1,6 +1,6 @@ preflight ok check resolve ok source=cluster envKey=dbr/15.4.x-scala2.12 -fetch error no published environment for "dbr/15.4.x-scala2.12". If this is a new runtime, try the latest LTS target (e.g. --serverless v4 or a supported --cluster DBR): GET [DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml: environment key not found +fetch error no published environment for "dbr/15.4.x-scala2.12". If this is a new runtime, try the latest LTS target (e.g. --serverless-version 5 or a supported --cluster-id DBR): GET [DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml: environment key not found merge pending provision pending validate pending diff --git a/acceptance/localenv/env-unsupported/script b/acceptance/localenv/env-unsupported/script index 924f04b9d97..6a84cf8172e 100644 --- a/acceptance/localenv/env-unsupported/script +++ b/acceptance/localenv/env-unsupported/script @@ -1 +1 @@ -musterr $CLI local-env python sync --cluster test-cluster-id --check +musterr $CLI environments setup-local --cluster-id test-cluster-id --dry-run diff --git a/acceptance/localenv/env-unsupported/test.toml b/acceptance/localenv/env-unsupported/test.toml index 7823db924d7..c6bb9f30340 100644 --- a/acceptance/localenv/env-unsupported/test.toml +++ b/acceptance/localenv/env-unsupported/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" diff --git a/acceptance/localenv/flag-conflict-json/out.test.toml b/acceptance/localenv/flag-conflict-json/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/flag-conflict-json/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/flag-conflict-json/output.txt b/acceptance/localenv/flag-conflict-json/output.txt new file mode 100644 index 00000000000..84840872e3f --- /dev/null +++ b/acceptance/localenv/flag-conflict-json/output.txt @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "command": "environments setup-local", + "ok": false, + "mode": "default", + "dryRun": false, + "greenfield": false, + "phases": [ + { + "phase": "preflight", + "status": "error" + }, + { + "phase": "resolve", + "status": "pending" + }, + { + "phase": "fetch", + "status": "pending" + }, + { + "phase": "merge", + "status": "pending" + }, + { + "phase": "provision", + "status": "pending" + }, + { + "phase": "validate", + "status": "pending" + } + ], + "warnings": [], + "error": { + "code": "E_USAGE", + "failurePhase": "preflight", + "message": "invalid compute target flags: flags --cluster-id and --serverless-version are mutually exclusive; specify at most one", + "diskMutated": false + }, + "durationMs": 0 +} diff --git a/acceptance/localenv/flag-conflict-json/script b/acceptance/localenv/flag-conflict-json/script new file mode 100644 index 00000000000..408aae874bd --- /dev/null +++ b/acceptance/localenv/flag-conflict-json/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --cluster-id abc --serverless-version 4 --output json diff --git a/acceptance/localenv/flag-conflict-json/test.toml b/acceptance/localenv/flag-conflict-json/test.toml new file mode 100644 index 00000000000..9609e1af299 --- /dev/null +++ b/acceptance/localenv/flag-conflict-json/test.toml @@ -0,0 +1 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/flag-conflict/out.test.toml b/acceptance/localenv/flag-conflict/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/flag-conflict/out.test.toml +++ b/acceptance/localenv/flag-conflict/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/flag-conflict/output.txt b/acceptance/localenv/flag-conflict/output.txt index 141152ae1cb..85a4bd285e0 100644 --- a/acceptance/localenv/flag-conflict/output.txt +++ b/acceptance/localenv/flag-conflict/output.txt @@ -1 +1,7 @@ -Error: if any flags in the group [cluster serverless job] are set none of the others can be; [cluster serverless] were all set +preflight error invalid compute target flags: flags --cluster-id and --serverless-version are mutually exclusive; specify at most one +resolve pending +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/flag-conflict/script b/acceptance/localenv/flag-conflict/script index 3309cd52205..b251350e3c4 100644 --- a/acceptance/localenv/flag-conflict/script +++ b/acceptance/localenv/flag-conflict/script @@ -1 +1 @@ -musterr $CLI local-env python sync --cluster abc --serverless v4 +musterr $CLI environments setup-local --cluster-id abc --serverless-version 4 diff --git a/acceptance/localenv/flag-conflict/test.toml b/acceptance/localenv/flag-conflict/test.toml index c63fe3fe108..9609e1af299 100644 --- a/acceptance/localenv/flag-conflict/test.toml +++ b/acceptance/localenv/flag-conflict/test.toml @@ -1 +1 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/help/out.test.toml b/acceptance/localenv/help/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/help/out.test.toml +++ b/acceptance/localenv/help/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/help/output.txt b/acceptance/localenv/help/output.txt index 03fffb2be77..b024e671819 100644 --- a/acceptance/localenv/help/output.txt +++ b/acceptance/localenv/help/output.txt @@ -1,22 +1,3 @@ -Manage the local Python environment matched to a Databricks compute target. - -Usage: - databricks local-env python [flags] - databricks local-env python [command] - -Available Commands: - sync Provision a local Python environment matched to a Databricks compute target - -Flags: - -h, --help help for python - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - -Use "databricks local-env python [command] --help" for more information about a command. Provision (or update) a local Python environment matched to a Databricks compute target. Resolves the target to an environment key, fetches the pinned Python version, @@ -26,15 +7,16 @@ initialized from scratch; an existing pyproject.toml is merged in place (its env-owned sections are refreshed, user-owned content is preserved). Usage: - databricks local-env python sync [flags] + databricks environments setup-local [flags] Flags: - --check compute the plan without writing files or provisioning - --cluster string cluster ID to use as the compute target - --constraints-only apply the Python version and constraints without adding the databricks-connect dependency - -h, --help help for sync - --job string job ID to use as the compute target - --serverless string serverless version to use as the compute target (e.g. v4) + --cluster-id string cluster ID to use as the compute target + --cluster-name string cluster name to use as the compute target (resolved to an ID via the Clusters API) + --constraints-only apply the Python version and constraints without adding the databricks-connect dependency + --dry-run compute the plan without writing files or provisioning + -h, --help help for setup-local + --job-task string job task to use as the compute target, as . (the task key is required) + --serverless-version string serverless version to use as the compute target (e.g. 5) Global Flags: --debug enable debug logging diff --git a/acceptance/localenv/help/script b/acceptance/localenv/help/script index cca1e85aad4..d8c393d010a 100644 --- a/acceptance/localenv/help/script +++ b/acceptance/localenv/help/script @@ -1,2 +1 @@ -$CLI local-env python --help -$CLI local-env python sync --help +$CLI environments setup-local --help diff --git a/acceptance/localenv/help/test.toml b/acceptance/localenv/help/test.toml index c63fe3fe108..9609e1af299 100644 --- a/acceptance/localenv/help/test.toml +++ b/acceptance/localenv/help/test.toml @@ -1 +1 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-classic-check/out.test.toml b/acceptance/localenv/job-classic-check/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/job-classic-check/out.test.toml +++ b/acceptance/localenv/job-classic-check/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-classic-check/output.txt b/acceptance/localenv/job-classic-check/output.txt index b4481ce0484..b3fa6654679 100644 --- a/acceptance/localenv/job-classic-check/output.txt +++ b/acceptance/localenv/job-classic-check/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] local-env python sync --job 12345 --check +>>> [CLI] environments setup-local --job-task 12345.ingest --dry-run preflight ok check resolve ok source=job envKey=dbr/15.4.x-scala2.12 fetch ok source=[DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml fromCache=false diff --git a/acceptance/localenv/job-classic-check/script b/acceptance/localenv/job-classic-check/script index 7469c95a844..c7c56c97f7b 100644 --- a/acceptance/localenv/job-classic-check/script +++ b/acceptance/localenv/job-classic-check/script @@ -1 +1 @@ -trace $CLI local-env python sync --job 12345 --check +trace $CLI environments setup-local --job-task 12345.ingest --dry-run diff --git a/acceptance/localenv/job-classic-check/test.toml b/acceptance/localenv/job-classic-check/test.toml index 9d123c84547..33fc90a5e37 100644 --- a/acceptance/localenv/job-classic-check/test.toml +++ b/acceptance/localenv/job-classic-check/test.toml @@ -1,9 +1,9 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" -# A classic-compute job (a single job cluster, no serverless environments) +# A classic task carries its own new_cluster, so --job-task . # resolves to that cluster's DBR-derived environment key. [[Server]] Pattern = "GET /api/2.2/jobs/get" @@ -12,8 +12,8 @@ Response.Body = ''' "job_id": 12345, "settings": { "name": "classic-job", - "job_clusters": [ - {"job_cluster_key": "main", "new_cluster": {"spark_version": "15.4.x-scala2.12", "num_workers": 1}} + "tasks": [ + {"task_key": "ingest", "new_cluster": {"spark_version": "15.4.x-scala2.12", "num_workers": 1}} ] } } diff --git a/acceptance/localenv/job-serverless-check/out.test.toml b/acceptance/localenv/job-serverless-check/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/job-serverless-check/out.test.toml +++ b/acceptance/localenv/job-serverless-check/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-serverless-check/output.txt b/acceptance/localenv/job-serverless-check/output.txt index 0b8d44d2ae7..a86c32d2cb3 100644 --- a/acceptance/localenv/job-serverless-check/output.txt +++ b/acceptance/localenv/job-serverless-check/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] local-env python sync --job 12345 --check +>>> [CLI] environments setup-local --job-task 12345.transform --dry-run preflight ok check resolve ok source=job envKey=serverless/serverless-v3 fetch ok source=[DATABRICKS_URL]/serverless/serverless-v3/pyproject.toml fromCache=false diff --git a/acceptance/localenv/job-serverless-check/script b/acceptance/localenv/job-serverless-check/script index 7469c95a844..572709bfadf 100644 --- a/acceptance/localenv/job-serverless-check/script +++ b/acceptance/localenv/job-serverless-check/script @@ -1 +1 @@ -trace $CLI local-env python sync --job 12345 --check +trace $CLI environments setup-local --job-task 12345.transform --dry-run diff --git a/acceptance/localenv/job-serverless-check/test.toml b/acceptance/localenv/job-serverless-check/test.toml index 34f8f09eb73..89ef2ce5794 100644 --- a/acceptance/localenv/job-serverless-check/test.toml +++ b/acceptance/localenv/job-serverless-check/test.toml @@ -1,10 +1,11 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" -# A serverless job pins its environment_version on the environment spec, so the -# target resolves to that serverless-vN (here v3) rather than defaulting to v4. +# A serverless task binds an environment_key to one of the job's environments; +# that environment pins the version, so --job-task . resolves to +# the matching serverless-vN (here v3) directly — no default fallback. [[Server]] Pattern = "GET /api/2.2/jobs/get" Response.Body = ''' @@ -12,6 +13,9 @@ Response.Body = ''' "job_id": 12345, "settings": { "name": "serverless-job", + "tasks": [ + {"task_key": "transform", "environment_key": "default"} + ], "environments": [ {"environment_key": "default", "spec": {"environment_version": "3"}} ] diff --git a/acceptance/localenv/job-task-foreach/out.test.toml b/acceptance/localenv/job-task-foreach/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/job-task-foreach/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-task-foreach/output.txt b/acceptance/localenv/job-task-foreach/output.txt new file mode 100644 index 00000000000..6587b3e2135 --- /dev/null +++ b/acceptance/localenv/job-task-foreach/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] environments setup-local --job-task 12345.fanout --dry-run +preflight ok check +resolve ok source=job envKey=serverless/serverless-v3 +fetch ok source=[DATABRICKS_URL]/serverless/serverless-v3/pyproject.toml fromCache=false +merge ok +provision ok +validate ok +Plan: [TEST_TMP_DIR]/pyproject.toml + changed region: requires-python + changed region: tool.uv.constraint-dependencies + changed region: databricks-connect +Check complete. No files were modified. diff --git a/acceptance/localenv/job-task-foreach/script b/acceptance/localenv/job-task-foreach/script new file mode 100644 index 00000000000..46ed21b157e --- /dev/null +++ b/acceptance/localenv/job-task-foreach/script @@ -0,0 +1 @@ +trace $CLI environments setup-local --job-task 12345.fanout --dry-run diff --git a/acceptance/localenv/job-task-foreach/test.toml b/acceptance/localenv/job-task-foreach/test.toml new file mode 100644 index 00000000000..8c7e836acca --- /dev/null +++ b/acceptance/localenv/job-task-foreach/test.toml @@ -0,0 +1,47 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# A for_each_task wraps the real per-iteration task; its compute lives on the +# nested task. --job-task resolves against that nested task (here a serverless +# environment binding), not the empty outer wrapper. +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 12345, + "settings": { + "name": "foreach-job", + "tasks": [ + { + "task_key": "fanout", + "for_each_task": { + "inputs": "[1,2,3]", + "task": {"task_key": "inner", "environment_key": "default"} + } + } + ], + "environments": [ + {"environment_key": "default", "spec": {"environment_version": "3"}} + ] + } +} +''' + +[[Server]] +Pattern = "GET /serverless/serverless-v3/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +''' + +[[Repls]] +Old = 'uv uv \S+(?: \([^)]+\))?' +New = 'uv [UV_VERSION]' diff --git a/acceptance/localenv/job-task-jobcluster/out.test.toml b/acceptance/localenv/job-task-jobcluster/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/job-task-jobcluster/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-task-jobcluster/output.txt b/acceptance/localenv/job-task-jobcluster/output.txt new file mode 100644 index 00000000000..b3fa6654679 --- /dev/null +++ b/acceptance/localenv/job-task-jobcluster/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] environments setup-local --job-task 12345.ingest --dry-run +preflight ok check +resolve ok source=job envKey=dbr/15.4.x-scala2.12 +fetch ok source=[DATABRICKS_URL]/dbr/15.4.x-scala2.12/pyproject.toml fromCache=false +merge ok +provision ok +validate ok +Plan: [TEST_TMP_DIR]/pyproject.toml + changed region: requires-python + changed region: tool.uv.constraint-dependencies + changed region: databricks-connect +Check complete. No files were modified. diff --git a/acceptance/localenv/job-task-jobcluster/script b/acceptance/localenv/job-task-jobcluster/script new file mode 100644 index 00000000000..c7c56c97f7b --- /dev/null +++ b/acceptance/localenv/job-task-jobcluster/script @@ -0,0 +1 @@ +trace $CLI environments setup-local --job-task 12345.ingest --dry-run diff --git a/acceptance/localenv/job-task-jobcluster/test.toml b/acceptance/localenv/job-task-jobcluster/test.toml new file mode 100644 index 00000000000..7eb76463420 --- /dev/null +++ b/acceptance/localenv/job-task-jobcluster/test.toml @@ -0,0 +1,41 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# A classic task references a shared job cluster by job_cluster_key (rather than +# an inline new_cluster), so --job-task resolves the version from the job's +# job_clusters entry. +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 12345, + "settings": { + "name": "jobcluster-job", + "tasks": [ + {"task_key": "ingest", "job_cluster_key": "shared"} + ], + "job_clusters": [ + {"job_cluster_key": "shared", "new_cluster": {"spark_version": "15.4.x-scala2.12", "num_workers": 1}} + ] + } +} +''' + +[[Server]] +Pattern = "GET /dbr/15.4.x-scala2.12/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +''' + +[[Repls]] +Old = 'uv uv \S+(?: \([^)]+\))?' +New = 'uv [UV_VERSION]' diff --git a/acceptance/localenv/job-task-missing-key/out.test.toml b/acceptance/localenv/job-task-missing-key/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/job-task-missing-key/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-task-missing-key/output.txt b/acceptance/localenv/job-task-missing-key/output.txt new file mode 100644 index 00000000000..75f44248d43 --- /dev/null +++ b/acceptance/localenv/job-task-missing-key/output.txt @@ -0,0 +1,7 @@ +preflight ok check +resolve error specify a job task: job 12345 has multiple tasks; specify one: --job-task 12345. (available: ingest, transform) +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/job-task-missing-key/script b/acceptance/localenv/job-task-missing-key/script new file mode 100644 index 00000000000..266ee6163aa --- /dev/null +++ b/acceptance/localenv/job-task-missing-key/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --job-task 12345 --dry-run diff --git a/acceptance/localenv/job-task-missing-key/test.toml b/acceptance/localenv/job-task-missing-key/test.toml new file mode 100644 index 00000000000..048d3a6f478 --- /dev/null +++ b/acceptance/localenv/job-task-missing-key/test.toml @@ -0,0 +1,26 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# A bare --job-task (no .) is a usage error: a job can bind +# tasks to different environment versions, so the CLI will not guess which one. +# The error lists the job's task keys. Reported as E_USAGE at the resolve phase. +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 12345, + "settings": { + "name": "multi-task-job", + "tasks": [ + {"task_key": "ingest", "environment_key": "e1"}, + {"task_key": "transform", "environment_key": "e2"} + ], + "environments": [ + {"environment_key": "e1", "spec": {"environment_version": "3"}}, + {"environment_key": "e2", "spec": {"environment_version": "4"}} + ] + } +} +''' diff --git a/acceptance/localenv/job-task-unknown/out.test.toml b/acceptance/localenv/job-task-unknown/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/job-task-unknown/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-task-unknown/output.txt b/acceptance/localenv/job-task-unknown/output.txt new file mode 100644 index 00000000000..7bd03a24d26 --- /dev/null +++ b/acceptance/localenv/job-task-unknown/output.txt @@ -0,0 +1,7 @@ +preflight ok check +resolve error resolving job task 12345.nope: job 12345 has no task "nope" (available: ingest) +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/job-task-unknown/script b/acceptance/localenv/job-task-unknown/script new file mode 100644 index 00000000000..6d43be42211 --- /dev/null +++ b/acceptance/localenv/job-task-unknown/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --job-task 12345.nope --dry-run diff --git a/acceptance/localenv/job-task-unknown/test.toml b/acceptance/localenv/job-task-unknown/test.toml new file mode 100644 index 00000000000..2a5cd68d54c --- /dev/null +++ b/acceptance/localenv/job-task-unknown/test.toml @@ -0,0 +1,23 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# A --job-task naming a task key that the job does not define fails at resolve +# (E_RESOLVE); the message lists the job's actual task keys. +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 12345, + "settings": { + "name": "job-with-tasks", + "tasks": [ + {"task_key": "ingest", "environment_key": "default"} + ], + "environments": [ + {"environment_key": "default", "spec": {"environment_version": "3"}} + ] + } +} +''' diff --git a/acceptance/localenv/job-task-unpinned/out.test.toml b/acceptance/localenv/job-task-unpinned/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/localenv/job-task-unpinned/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/job-task-unpinned/output.txt b/acceptance/localenv/job-task-unpinned/output.txt new file mode 100644 index 00000000000..b9a0f8ebc4b --- /dev/null +++ b/acceptance/localenv/job-task-unpinned/output.txt @@ -0,0 +1,7 @@ +preflight ok check +resolve error resolving job task 12345.transform: task "transform" of job 12345 binds environment "default", which records no environment version +fetch pending +merge pending +provision pending +validate pending +For more detail, re-run with --debug, or --output json to share a structured report. diff --git a/acceptance/localenv/job-task-unpinned/script b/acceptance/localenv/job-task-unpinned/script new file mode 100644 index 00000000000..6bec20c0289 --- /dev/null +++ b/acceptance/localenv/job-task-unpinned/script @@ -0,0 +1 @@ +musterr $CLI environments setup-local --job-task 12345.transform --dry-run diff --git a/acceptance/localenv/job-task-unpinned/test.toml b/acceptance/localenv/job-task-unpinned/test.toml new file mode 100644 index 00000000000..292dae64b5a --- /dev/null +++ b/acceptance/localenv/job-task-unpinned/test.toml @@ -0,0 +1,25 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" + +# A task whose bound serverless environment records no version cannot be resolved: +# the job-task path reads the version directly and never falls back to a default +# (the serverless-vN fallback applies only to the bundle-target path). Fails at +# resolve (E_RESOLVE); nothing is fetched. +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 12345, + "settings": { + "name": "serverless-job-unpinned", + "tasks": [ + {"task_key": "transform", "environment_key": "default"} + ], + "environments": [ + {"environment_key": "default", "spec": {}} + ] + } +} +''' diff --git a/acceptance/localenv/json-error/out.test.toml b/acceptance/localenv/json-error/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/json-error/out.test.toml +++ b/acceptance/localenv/json-error/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/json-error/output.txt b/acceptance/localenv/json-error/output.txt index e76a8f74798..b2e5a0ba43e 100644 --- a/acceptance/localenv/json-error/output.txt +++ b/acceptance/localenv/json-error/output.txt @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "command": "local-env python sync", + "command": "environments setup-local", "ok": false, "mode": "default", "dryRun": false, @@ -35,7 +35,7 @@ "error": { "code": "E_NO_TARGET", "failurePhase": "resolve", - "message": "No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job", + "message": "No compute target is selected. Select a cluster or serverless target, or pass --cluster-id / --cluster-name / --serverless-version / --job-task", "diskMutated": false }, "durationMs": 0 diff --git a/acceptance/localenv/json-error/script b/acceptance/localenv/json-error/script index bffcc264af6..fc3931eadd5 100644 --- a/acceptance/localenv/json-error/script +++ b/acceptance/localenv/json-error/script @@ -1 +1 @@ -musterr $CLI local-env python sync --output json +musterr $CLI environments setup-local --output json diff --git a/acceptance/localenv/json-error/test.toml b/acceptance/localenv/json-error/test.toml index 0d0481fb836..270ce8c5b79 100644 --- a/acceptance/localenv/json-error/test.toml +++ b/acceptance/localenv/json-error/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [[Repls]] Old = 'uv uv \S+(?: \([^)]+\))?' diff --git a/acceptance/localenv/manager-unsupported/out.test.toml b/acceptance/localenv/manager-unsupported/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/manager-unsupported/out.test.toml +++ b/acceptance/localenv/manager-unsupported/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/manager-unsupported/script b/acceptance/localenv/manager-unsupported/script index aae187d00cb..f7a1e15df54 100644 --- a/acceptance/localenv/manager-unsupported/script +++ b/acceptance/localenv/manager-unsupported/script @@ -1 +1 @@ -musterr $CLI local-env python sync --serverless v4 --check +musterr $CLI environments setup-local --serverless-version 4 --dry-run diff --git a/acceptance/localenv/manager-unsupported/test.toml b/acceptance/localenv/manager-unsupported/test.toml index 0d0481fb836..270ce8c5b79 100644 --- a/acceptance/localenv/manager-unsupported/test.toml +++ b/acceptance/localenv/manager-unsupported/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [[Repls]] Old = 'uv uv \S+(?: \([^)]+\))?' diff --git a/acceptance/localenv/no-target/out.test.toml b/acceptance/localenv/no-target/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/no-target/out.test.toml +++ b/acceptance/localenv/no-target/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/no-target/output.txt b/acceptance/localenv/no-target/output.txt index 7ffd3040a3b..eba9aad3918 100644 --- a/acceptance/localenv/no-target/output.txt +++ b/acceptance/localenv/no-target/output.txt @@ -1,5 +1,5 @@ preflight ok uv [UV_VERSION] -resolve error No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job +resolve error No compute target is selected. Select a cluster or serverless target, or pass --cluster-id / --cluster-name / --serverless-version / --job-task fetch pending merge pending provision pending diff --git a/acceptance/localenv/no-target/script b/acceptance/localenv/no-target/script index 3460d1b8cf1..712272f0e56 100644 --- a/acceptance/localenv/no-target/script +++ b/acceptance/localenv/no-target/script @@ -1 +1 @@ -musterr $CLI local-env python sync +musterr $CLI environments setup-local diff --git a/acceptance/localenv/no-target/test.toml b/acceptance/localenv/no-target/test.toml index 0d0481fb836..270ce8c5b79 100644 --- a/acceptance/localenv/no-target/test.toml +++ b/acceptance/localenv/no-target/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [[Repls]] Old = 'uv uv \S+(?: \([^)]+\))?' diff --git a/acceptance/localenv/serverless-check/out.test.toml b/acceptance/localenv/serverless-check/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/serverless-check/out.test.toml +++ b/acceptance/localenv/serverless-check/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/serverless-check/output.txt b/acceptance/localenv/serverless-check/output.txt index 927da068d45..2b36b789ace 100644 --- a/acceptance/localenv/serverless-check/output.txt +++ b/acceptance/localenv/serverless-check/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] local-env python sync --serverless v4 --check +>>> [CLI] environments setup-local --serverless-version 4 --dry-run preflight ok check resolve ok source=serverless envKey=serverless/serverless-v4 fetch ok source=[DATABRICKS_URL]/serverless/serverless-v4/pyproject.toml fromCache=false diff --git a/acceptance/localenv/serverless-check/script b/acceptance/localenv/serverless-check/script index 8c202ab129c..c02eb922237 100644 --- a/acceptance/localenv/serverless-check/script +++ b/acceptance/localenv/serverless-check/script @@ -1 +1 @@ -trace $CLI local-env python sync --serverless v4 --check +trace $CLI environments setup-local --serverless-version 4 --dry-run diff --git a/acceptance/localenv/serverless-check/test.toml b/acceptance/localenv/serverless-check/test.toml index e2b12fc95ba..c636f7a9cef 100644 --- a/acceptance/localenv/serverless-check/test.toml +++ b/acceptance/localenv/serverless-check/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" diff --git a/acceptance/localenv/serverless-json/out.test.toml b/acceptance/localenv/serverless-json/out.test.toml index d6187dcb046..e90b6d5d1ba 100644 --- a/acceptance/localenv/serverless-json/out.test.toml +++ b/acceptance/localenv/serverless-json/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/serverless-json/output.txt b/acceptance/localenv/serverless-json/output.txt index c53fa61031a..056a3fa34f2 100644 --- a/acceptance/localenv/serverless-json/output.txt +++ b/acceptance/localenv/serverless-json/output.txt @@ -1,8 +1,8 @@ ->>> [CLI] local-env python sync --serverless v4 --check --output json +>>> [CLI] environments setup-local --serverless-version 4 --dry-run --output json { "schemaVersion": 1, - "command": "local-env python sync", + "command": "environments setup-local", "ok": true, "mode": "default", "dryRun": true, @@ -20,7 +20,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+# managed by databricks local-env python sync — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks local-env python sync\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/serverless-json/script b/acceptance/localenv/serverless-json/script index 84ff5620456..0bda75a032b 100644 --- a/acceptance/localenv/serverless-json/script +++ b/acceptance/localenv/serverless-json/script @@ -1 +1 @@ -trace $CLI local-env python sync --serverless v4 --check --output json +trace $CLI environments setup-local --serverless-version 4 --dry-run --output json diff --git a/acceptance/localenv/serverless-json/test.toml b/acceptance/localenv/serverless-json/test.toml index e2b12fc95ba..c636f7a9cef 100644 --- a/acceptance/localenv/serverless-json/test.toml +++ b/acceptance/localenv/serverless-json/test.toml @@ -1,4 +1,4 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json b/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json +++ b/acceptance/pipelines/e2e/output/lakeflow_project/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/AGENTS.md b/acceptance/pipelines/e2e/output/lakeflow_project/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/pipelines/e2e/output/lakeflow_project/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/CLAUDE.md b/acceptance/pipelines/e2e/output/lakeflow_project/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/pipelines/e2e/output/lakeflow_project/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json b/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json index 5ba48e79c9b..b958aacfcb4 100644 --- a/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json +++ b/acceptance/pipelines/init/python/output/my_python_project/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/init/python/output/my_python_project/AGENTS.md b/acceptance/pipelines/init/python/output/my_python_project/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/pipelines/init/python/output/my_python_project/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/pipelines/init/python/output/my_python_project/CLAUDE.md b/acceptance/pipelines/init/python/output/my_python_project/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/pipelines/init/python/output/my_python_project/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc5..00000000000 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json index 5ba48e79c9b..1f39c330871 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json +++ b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json index d73c73b5705..d3e814885cb 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json +++ b/acceptance/pipelines/init/sql/output/my_sql_project/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/AGENTS.md b/acceptance/pipelines/init/sql/output/my_sql_project/AGENTS.md new file mode 100644 index 00000000000..fdcca98a432 --- /dev/null +++ b/acceptance/pipelines/init/sql/output/my_sql_project/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/CLAUDE.md b/acceptance/pipelines/init/sql/output/my_sql_project/CLAUDE.md new file mode 100644 index 00000000000..5612c9bde2f --- /dev/null +++ b/acceptance/pipelines/init/sql/output/my_sql_project/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore b/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore index e566c51f740..622a8811a7b 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore +++ b/acceptance/pipelines/init/sql/output/my_sql_project/out.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 115352791b2..c5dc9612120 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -109,8 +109,12 @@ retry() { env -u MSYS_NO_PATHCONV retry.py "$@" } +# The engine_terraform_* keys reflect the deployment engine opt-in; engine_terraform_env +# in particular mirrors $DATABRICKS_BUNDLE_ENGINE, which the acceptance matrix sets per +# variant, so it diverges across the matrix. Drop both here so this shared helper stays +# engine-agnostic; they are asserted per-engine in bundle/telemetry/deploy. print_telemetry_bool_values() { - jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' out.requests.txt | sort + jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' out.requests.txt | grep -v '^engine_terraform_' | sort } sethome() { diff --git a/acceptance/skiplocal_test.go b/acceptance/skiplocal_test.go index c72f7c39a3f..7f90653099b 100644 --- a/acceptance/skiplocal_test.go +++ b/acceptance/skiplocal_test.go @@ -1,10 +1,12 @@ package acceptance_test import ( + "errors" "os/exec" "path/filepath" "slices" "strings" + "testing" ) // DATABRICKS_TEST_SKIPLOCAL controls skipping of Local acceptance tests. @@ -61,8 +63,19 @@ func testDirForFile(repoRelPath string, testDirs map[string]bool) string { // committed. The three-dot form origin/main...HEAD only covers committed // changes and misses unstaged edits, which breaks the "touch a config, run // the test" local dev workflow (same reason lintdiff.py uses --merge-base). -func selectChangedLocalTests(testDirs map[string]bool) map[string][]string { - out, _ := exec.Command("git", "diff", "--name-status", "--merge-base", "-M", "origin/main").Output() +func selectChangedLocalTests(t *testing.T, testDirs map[string]bool) map[string][]string { + out, err := exec.Command("git", "diff", "--name-status", "--merge-base", "-M", "origin/main").Output() + if err != nil { + // A failed diff (most commonly a missing origin/main in a shallow CI + // checkout) must not be silently treated as "nothing changed": that + // disables change detection and lets newly added tests skip. Fail loudly. + // Every caller (push.yml PR cells, integration runs) now fetches origin/main. + stderr := "" + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { + stderr = strings.TrimSpace(string(exitErr.Stderr)) + } + t.Fatalf("git diff --merge-base origin/main failed: %v\n%s", err, stderr) + } diff := strings.TrimSpace(string(out)) // result accumulates dirs with their filters; added tracks brand-new dirs. diff --git a/acceptance/ssh/connect-serverless-cpu/output.txt b/acceptance/ssh/connect-serverless-cpu/output.txt index 5519f43fc58..994a35cbb88 100644 --- a/acceptance/ssh/connect-serverless-cpu/output.txt +++ b/acceptance/ssh/connect-serverless-cpu/output.txt @@ -26,6 +26,7 @@ "serverless": "true", "sessionId": "[CPU_CONN]", "shutdownDelay": "10m0s", + "usagePolicyId": "", "version": "[CLI_VERSION]" }, "notebook_path": "/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/[CPU_CONN]/ssh-server-bootstrap" diff --git a/acceptance/ssh/connect-serverless-gpu/output.txt b/acceptance/ssh/connect-serverless-gpu/output.txt index b7013545a8b..7c213823257 100644 --- a/acceptance/ssh/connect-serverless-gpu/output.txt +++ b/acceptance/ssh/connect-serverless-gpu/output.txt @@ -27,6 +27,7 @@ "serverless": "true", "sessionId": "serverless-gpu-test", "shutdownDelay": "10m0s", + "usagePolicyId": "", "version": "[CLI_VERSION]" }, "notebook_path": "/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/serverless-gpu-test/ssh-server-bootstrap" diff --git a/acceptance/ssh/connection/output.txt b/acceptance/ssh/connection/output.txt index 6c4551026b5..58babb6e4dc 100644 --- a/acceptance/ssh/connection/output.txt +++ b/acceptance/ssh/connection/output.txt @@ -16,6 +16,7 @@ "serverless": "false", "sessionId": "[TEST_DEFAULT_CLUSTER_ID]", "shutdownDelay": "10m0s", + "usagePolicyId": "", "version": "[CLI_VERSION]" }, "notebook_path": "/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/[TEST_DEFAULT_CLUSTER_ID]/ssh-server-bootstrap" diff --git a/acceptance/subset_test.go b/acceptance/subset_test.go new file mode 100644 index 00000000000..22eda164b3d --- /dev/null +++ b/acceptance/subset_test.go @@ -0,0 +1,131 @@ +package acceptance_test + +import ( + "hash/fnv" + "os" + "strconv" + "strings" + "testing" + + "github.com/google/uuid" +) + +// TESTS_SELECT_SUBSET_PCT/SEED select a random subset of subtests to run, so a CI +// cell can cover a fraction of the suite in less time. CI sets these only on the +// windows/macOS pull_request cells (see .github/workflows/push.yml); on Linux they +// are unset and the full suite runs. +const ( + // SubsetPctEnvVar is an integer 0..100: the percentage of subtests to run. + // Unset (or empty) disables subsetting entirely. + SubsetPctEnvVar = "TESTS_SELECT_SUBSET_PCT" + + // SubsetSeedEnvVar seeds the selection. CI sets it to the HEAD commit hash, so + // a new commit reshuffles the subset while a retry of the same commit repeats it. + // If unset, a random seed is generated and logged so the run can be reproduced. + SubsetSeedEnvVar = "TESTS_SELECT_SUBSET_SEED" +) + +// subsetSelector decides, per subtest, whether it runs under TESTS_SELECT_SUBSET_PCT. +// A subtest runs if it is an added/modified test on this branch (always kept, reusing +// the same change detection as SkipLocalWithChanged), or if its seeded hash falls +// under the percentage. The decision is independent per subtest, so added/modified +// tests run on top of the hash-selected subset rather than displacing anything. +type subsetSelector struct { + enabled bool + pct int + seed string + + // changed maps test dir -> variant filters for added/modified tests (nil filters + // means all variants of that dir are changed). Same shape as changedTests. + changed map[string][]string +} + +// newSubsetSelector reads the subset env vars. Subsetting is disabled in update mode +// and under -forcerun so that every output is regenerated and forced runs are honored. +// The caller assigns .changed (the added/modified tests to always keep) so that the +// change detection is shared with SkipLocalWithChanged and runs at most once. +func newSubsetSelector(t *testing.T, overwrite, forcerun bool) subsetSelector { + raw := os.Getenv(SubsetPctEnvVar) + if raw == "" || overwrite || forcerun { + return subsetSelector{} + } + + pct, err := strconv.Atoi(raw) + if err != nil || pct < 0 || pct > 100 { + t.Fatalf("Invalid %s=%q, expected an integer 0..100", SubsetPctEnvVar, raw) + } + + seed := os.Getenv(SubsetSeedEnvVar) + if seed == "" { + // Reproduce this run's subset by re-running with the logged seed. + seed = uuid.NewString() + t.Logf("%s not set, using random seed %q", SubsetSeedEnvVar, seed) + } + + return subsetSelector{ + enabled: true, + pct: pct, + seed: seed, + } +} + +// skipReason returns a non-empty skip message if the subtest identified by dir and +// envset should be skipped by subsetting, or "" if it should run. +func (s subsetSelector) skipReason(dir string, envset []string) string { + if !s.enabled { + return "" + } + if s.isChanged(dir, envset) { + return "" + } + id := dir + if len(envset) > 0 { + id += "/" + strings.Join(envset, "/") + } + if hashPercent(s.seed, id) < s.pct { + return "" + } + return "Skipped by " + SubsetPctEnvVar +} + +// isChanged reports whether the subtest belongs to an added/modified test dir. For an +// invariant dir re-enabled by a specific config change, only the matching variants +// count as changed. +func (s subsetSelector) isChanged(dir string, envset []string) bool { + filters, ok := s.changed[dir] + if !ok { + return false + } + if filters == nil { + return true + } + return envMatchesFilters(envset, filters) +} + +// envMatchesFilters reports whether envset satisfies every KEY=value filter, mirroring +// the skip semantics of checkEnvFilters: a filter matches unless its key is present in +// envset with a different value. +func envMatchesFilters(envset, filters []string) bool { + envMap := make(map[string]string, len(envset)) + for _, kv := range envset { + key, value, _ := strings.Cut(kv, "=") + envMap[key] = value + } + for _, filter := range filters { + key, expected, _ := strings.Cut(filter, "=") + if actual, ok := envMap[key]; ok && actual != expected { + return false + } + } + return true +} + +// hashPercent maps (seed, id) deterministically to 0..99. A subtest runs when this is +// below the percentage, so pct=100 runs everything and pct=0 runs nothing. +func hashPercent(seed, id string) int { + h := fnv.New64a() + h.Write([]byte(seed)) + h.Write([]byte{0}) + h.Write([]byte(id)) + return int(h.Sum64() % 100) +} diff --git a/acceptance/subset_unit_test.go b/acceptance/subset_unit_test.go new file mode 100644 index 00000000000..6f83e859841 --- /dev/null +++ b/acceptance/subset_unit_test.go @@ -0,0 +1,106 @@ +package acceptance_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSubsetDisabled(t *testing.T) { + var s subsetSelector + assert.Empty(t, s.skipReason("bundle/foo", nil)) + assert.Empty(t, s.skipReason("bundle/foo", []string{"DATABRICKS_BUNDLE_ENGINE=direct"})) +} + +func TestSubsetPctBoundaries(t *testing.T) { + all := subsetSelector{enabled: true, pct: 100, seed: "seed"} + none := subsetSelector{enabled: true, pct: 0, seed: "seed"} + + for _, dir := range []string{"bundle/a", "bundle/b", "cmd/c"} { + assert.Empty(t, all.skipReason(dir, nil), "pct=100 must run everything") + assert.NotEmpty(t, none.skipReason(dir, nil), "pct=0 must skip everything") + } +} + +func TestSubsetChangedAlwaysRuns(t *testing.T) { + // pct=0 would skip everything, but a changed dir is always kept. + s := subsetSelector{ + enabled: true, + pct: 0, + seed: "seed", + changed: map[string][]string{"bundle/changed": nil}, + } + assert.Empty(t, s.skipReason("bundle/changed", []string{"DATABRICKS_BUNDLE_ENGINE=direct"})) + assert.NotEmpty(t, s.skipReason("bundle/other", nil)) +} + +func TestSubsetChangedVariantFilter(t *testing.T) { + // A changed dir restricted to a specific config only keeps matching variants. + s := subsetSelector{ + enabled: true, + pct: 0, + seed: "seed", + changed: map[string][]string{"bundle/invariant/x": {"INPUT_CONFIG=job.yml.tmpl"}}, + } + assert.Empty(t, s.skipReason("bundle/invariant/x", []string{"INPUT_CONFIG=job.yml.tmpl"})) + assert.NotEmpty(t, s.skipReason("bundle/invariant/x", []string{"INPUT_CONFIG=pipeline.yml.tmpl"})) +} + +func TestSubsetSeedDeterministic(t *testing.T) { + // Same seed selects the same subset; the decision does not depend on ordering. + a := subsetSelector{enabled: true, pct: 50, seed: "abc"} + b := subsetSelector{enabled: true, pct: 50, seed: "abc"} + for _, dir := range []string{"bundle/a", "bundle/b", "bundle/c", "bundle/d"} { + assert.Equal(t, a.skipReason(dir, nil), b.skipReason(dir, nil)) + } +} + +func TestSubsetSeedChangesSelection(t *testing.T) { + // Different seeds should produce different selections across a set of dirs. + var dirs []string + for i := range 100 { + dirs = append(dirs, "bundle/dir"+string(rune('a'+i%26))+string(rune('0'+i/26))) + } + s1 := subsetSelector{enabled: true, pct: 50, seed: "seed-1"} + s2 := subsetSelector{enabled: true, pct: 50, seed: "seed-2"} + diff := 0 + for _, dir := range dirs { + if (s1.skipReason(dir, nil) == "") != (s2.skipReason(dir, nil) == "") { + diff++ + } + } + assert.NotZero(t, diff, "different seeds should select different subsets") +} + +func TestSubsetVariantsIndependent(t *testing.T) { + // Two variants of one dir are decided independently: at least one differs from the + // other across a spread of dirs (they are not forced to the same outcome). + s := subsetSelector{enabled: true, pct: 50, seed: "seed"} + sawDifferent := false + for i := range 50 { + dir := "bundle/d" + string(rune('a'+i%26)) + a := s.skipReason(dir, []string{"DATABRICKS_BUNDLE_ENGINE=direct"}) == "" + b := s.skipReason(dir, []string{"DATABRICKS_BUNDLE_ENGINE=terraform"}) == "" + if a != b { + sawDifferent = true + break + } + } + assert.True(t, sawDifferent, "variants of the same dir should be decided independently") +} + +func TestSubsetApproximatesPct(t *testing.T) { + // Over many subtests the selected fraction should be close to pct. + s := subsetSelector{enabled: true, pct: 30, seed: "seed"} + run := 0 + total := 1000 + for i := range total { + id := "bundle/pkg" + string(rune('a'+i%26)) + "/case" + string(rune('0'+i/26%10)) + if s.skipReason(id, []string{"V=" + string(rune('a'+i%7))}) == "" { + run++ + } + } + // Allow a generous margin; this guards against gross bias, not exact proportion. + assert.Greater(t, run, 200) + assert.Less(t, run, 400) +} diff --git a/bundle/bundle.go b/bundle/bundle.go index 651efb873cd..94f55389b26 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -197,9 +197,11 @@ func Load(ctx context.Context, path string) (*Bundle, error) { // MustLoad returns a bundle configuration. // The errors are recorded by logdiag, check with logdiag.HasError(). func MustLoad(ctx context.Context) *Bundle { - root, err := mustGetRoot(ctx) - if err != nil { - logdiag.LogError(ctx, err) + root, diags := mustGetRoot(ctx) + if diags.HasError() { + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } return nil } @@ -217,9 +219,11 @@ func MustLoad(ctx context.Context) *Bundle { // The errors are recorded by logdiag, check with logdiag.HasError(). // It returns a `nil` bundle if a bundle was not found. func TryLoad(ctx context.Context) *Bundle { - root, err := tryGetRoot(ctx) - if err != nil { - logdiag.LogError(ctx, err) + root, diags := tryGetRoot(ctx) + if diags.HasError() { + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } return nil } diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 37928cb8801..9bd667afe62 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -1,7 +1,6 @@ package bundle import ( - "io/fs" "os" "path/filepath" "testing" @@ -38,7 +37,7 @@ func tryLoad(t *testing.T) (*Bundle, []diag.Diagnostic) { func TestLoadNotExists(t *testing.T) { b, err := Load(t.Context(), "/doesntexist") - assert.ErrorIs(t, err, fs.ErrNotExist) + assert.ErrorContains(t, err, "databricks.yml not found") assert.Nil(t, b) } @@ -110,7 +109,7 @@ func TestBundleMustLoadFailureWithEnv(t *testing.T) { b, diags := mustLoad(t) require.Nil(t, b) require.Len(t, diags, 1, "expected diagnostics") - assert.Contains(t, diags[0].Summary, "invalid bundle root") + assert.Contains(t, diags[0].Summary, "Invalid bundle root") assert.Equal(t, diag.Error, diags[0].Severity) } @@ -119,7 +118,7 @@ func TestBundleMustLoadFailureIfNotFound(t *testing.T) { b, diags := mustLoad(t) require.Nil(t, b) require.Len(t, diags, 1, "expected diagnostics") - assert.Contains(t, diags[0].Summary, "unable to locate bundle root") + assert.Contains(t, diags[0].Summary, "Unable to locate the bundle root") assert.Equal(t, diag.Error, diags[0].Severity) } @@ -136,7 +135,7 @@ func TestBundleTryLoadFailureWithEnv(t *testing.T) { b, diags := tryLoad(t) require.Nil(t, b) require.Len(t, diags, 1, "expected diagnostics") - assert.Contains(t, diags[0].Summary, "invalid bundle root") + assert.Contains(t, diags[0].Summary, "Invalid bundle root") assert.Equal(t, diag.Error, diags[0].Severity) } diff --git a/bundle/config/filename.go b/bundle/config/filename.go index 11af34d9cf7..ce72cff2ce2 100644 --- a/bundle/config/filename.go +++ b/bundle/config/filename.go @@ -18,7 +18,6 @@ var FileNames = ConfigFileNames{ func (c ConfigFileNames) FindInPath(path string) (string, error) { result := "" - var firstErr error for _, file := range c { filePath := filepath.Join(path, file) @@ -28,15 +27,11 @@ func (c ConfigFileNames) FindInPath(path string) (string, error) { return "", fmt.Errorf("multiple bundle root configuration files found in %s", path) } result = filePath - } else { - if firstErr == nil { - firstErr = err - } } } if result == "" { - return "", firstErr + return "", fmt.Errorf("%s not found", c[0]) } return result, nil diff --git a/bundle/config/filename_test.go b/bundle/config/filename_test.go index 9d71fa7e867..47d4fd22aa4 100644 --- a/bundle/config/filename_test.go +++ b/bundle/config/filename_test.go @@ -3,7 +3,6 @@ package config import ( "os" "path/filepath" - "runtime" "strings" "testing" @@ -39,14 +38,10 @@ func TestConfigFileNames_FindInPath(t *testing.T) { name: "file not found", files: []string{}, expected: "", - err: "no such file or directory", + err: "databricks.yml not found", }, } - if runtime.GOOS == "windows" { - testCases[3].err = "The system cannot find the file specified." - } - for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { projectDir := t.TempDir() diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go new file mode 100644 index 00000000000..0d4eba75f0b --- /dev/null +++ b/bundle/config/mutator/drop_empty_strings.go @@ -0,0 +1,62 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" +) + +type dropEmptyStrings struct{} + +// DropEmptyStrings removes empty-string values on omitempty resource fields, so +// they are not force-sent to the backend. An empty string reaches this point +// either literally (policy_id: "") or via a variable that resolved to "", so +// this must run after variable resolution. +// +// Both engines convert the resolved config through convert.ToTyped, which +// force-sends explicitly-set zero values, defeating the omitempty tag. Dropping +// here fixes it uniformly for terraform and direct and makes the result visible +// in `bundle validate -o json`, which serializes the dynamic value. +func DropEmptyStrings() bundle.Mutator { + return &dropEmptyStrings{} +} + +func (m *dropEmptyStrings) Name() string { + return "DropEmptyStrings" +} + +func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + root, err := dyn.Map(root, "resources", func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) { + // Normalize against the resources type so omitempty is known per field. + // Only DropEmptyStrings is set: existing values are kept as-is otherwise. + out, normDiags := convert.Normalize(config.Resources{}, resources, convert.DropEmptyStrings) + diags = diags.Extend(normDiags) + return out, nil + }) + if err != nil { + return root, err + } + + // It seems safe to send an empty description "" to the backend, but for apps + // it causes drift in terraform: the Apps API always returns "" for an unset + // description, so bundle/deploy/terraform/tfdyn/convert_app.go injects it + // anyway. To avoid an unnecessary difference between the direct and terraform + // engines, keep the empty apps description here instead of dropping it. + return dyn.MapByPattern(root, dyn.NewPattern(dyn.Key("resources"), dyn.Key("apps"), dyn.AnyKey()), func(_ dyn.Path, app dyn.Value) (dyn.Value, error) { + if _, err := dyn.Get(app, "description"); err != nil { + return dyn.Set(app, "description", dyn.V("")) + } + return app, nil + }) + }) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + return diags +} diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 8ac31e53488..5c3e6921b93 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -179,6 +179,7 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia RunAs: dbalertFromFile.RunAs, RunAsUserName: dbalertFromFile.RunAsUserName, ParentPath: dbalertFromFile.ParentPath, + Parameters: dbalertFromFile.Parameters, // Output only fields. CreateTime: dbalertFromFile.CreateTime, diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index c81d452d641..f2a6d857090 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -89,6 +89,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", }, + "instance_pools": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_ATTACH_TO", + }, } ) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 912d70bdd40..4a3b5e298d0 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -89,6 +89,10 @@ func TestApplyBundlePermissions(t *testing.T) { "vs_1": {}, "vs_2": {}, }, + InstancePools: map[string]*resources.InstancePool{ + "instance_pool_1": {}, + "instance_pool_2": {}, + }, }, }, } @@ -151,12 +155,16 @@ func TestApplyBundlePermissions(t *testing.T) { require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.AppPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) - require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.VectorSearchEndpointPermission{Level: "CAN_USE", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, 2) + require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.InstancePools["instance_pool_1"].Permissions, resources.InstancePoolPermission{Level: "CAN_ATTACH_TO", GroupName: "TestGroup"}) } func TestWarningOnOverlapPermission(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 70cc2f66cbc..72817d13566 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -232,6 +232,26 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Instance Pools: Prefix, Tags + for _, pool := range r.InstancePools { + if pool == nil { + continue + } + pool.InstancePoolName = prefix + pool.InstancePoolName + if len(tags) > 0 { + if pool.CustomTags == nil { + pool.CustomTags = make(map[string]string, len(tags)) + } + for _, tag := range tags { + k := b.Tagging.NormalizeKey(tag.Key) + v := b.Tagging.NormalizeValue(tag.Value) + if _, ok := pool.CustomTags[k]; !ok { + pool.CustomTags[k] = v + } + } + } + } + // Dashboards: Prefix for _, dashboard := range r.Dashboards { if dashboard == nil { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index da9fcab9770..b97113028b8 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -150,6 +150,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, + InstancePools: map[string]*resources.InstancePool{ + "instance_pool1": {CreateInstancePool: compute.CreateInstancePool{InstancePoolName: "instance_pool1", NodeTypeId: "i3.xlarge"}}, + }, Dashboards: map[string]*resources.Dashboard{ "dashboard1": { DashboardConfig: resources.DashboardConfig{ @@ -376,6 +379,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Clusters assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) + // Instance pools + assert.Equal(t, "[dev lennart] instance_pool1", b.Config.Resources.InstancePools["instance_pool1"].InstancePoolName) + // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) @@ -452,6 +458,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name) assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name) assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) + assert.Equal(t, "instance_pool1", b.Config.Resources.InstancePools["instance_pool1"].InstancePoolName) assert.Equal(t, "sql_warehouse1", b.Config.Resources.SqlWarehouses["sql_warehouse1"].Name) } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 94db7ed67bb..5faed5f7b1b 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -42,6 +42,7 @@ func allResourceTypes(t *testing.T) []string { "experiments", "external_locations", "genie_spaces", + "instance_pools", "job_runs", "jobs", "model_serving_endpoints", @@ -190,6 +191,7 @@ var allowList = []string{ "registered_models", "experiments", "genie_spaces", + "instance_pools", "job_runs", "schemas", "secret_scopes", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index cfbfbe9fff5..3ab06f49918 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -43,6 +43,7 @@ type Resources struct { PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` + InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` } type ConfigResource interface { @@ -127,6 +128,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), + collectResourceMap(descriptions["instance_pools"], r.InstancePools), } } @@ -163,6 +165,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "pipelines": (&resources.Pipeline{}).ResourceDescription(), "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), + "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), diff --git a/bundle/config/resources/instance_pools.go b/bundle/config/resources/instance_pools.go new file mode 100644 index 00000000000..31c4ee45a69 --- /dev/null +++ b/bundle/config/resources/instance_pools.go @@ -0,0 +1,59 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type InstancePool struct { + BaseResource + compute.CreateInstancePool + Permissions []InstancePoolPermission `json:"permissions,omitempty"` +} + +func (s *InstancePool) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s InstancePool) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *InstancePool) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.InstancePools.GetByInstancePoolId(ctx, id) + if err != nil { + log.Debugf(ctx, "instance pool %s does not exist", id) + return false, err + } + return true, nil +} + +func (*InstancePool) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "instance_pool", + PluralName: "instance_pools", + SingularTitle: "Instance Pool", + PluralTitle: "Instance Pools", + } +} + +func (s *InstancePool) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "instance_pools", s.ID) +} + +func (s *InstancePool) GetName() string { + return s.InstancePoolName +} + +func (s *InstancePool) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/permission_types.go b/bundle/config/resources/permission_types.go index 3029ee40b8c..d067c5e3e37 100644 --- a/bundle/config/resources/permission_types.go +++ b/bundle/config/resources/permission_types.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) // Each resource defines its own permission type so that the JSON schema names them distinctly. @@ -26,10 +27,12 @@ func (p Permission) String() string { type ( AppPermission PermissionT[apps.AppPermissionLevel] ClusterPermission PermissionT[compute.ClusterPermissionLevel] + InstancePoolPermission PermissionT[compute.InstancePoolPermissionLevel] JobPermission PermissionT[jobs.JobPermissionLevel] MlflowExperimentPermission PermissionT[ml.ExperimentPermissionLevel] MlflowModelPermission PermissionT[ml.RegisteredModelPermissionLevel] ModelServingEndpointPermission PermissionT[serving.ServingEndpointPermissionLevel] PipelinePermission PermissionT[pipelines.PipelinePermissionLevel] SqlWarehousePermission PermissionT[sql.WarehousePermissionLevel] + VectorSearchEndpointPermission PermissionT[vectorsearch.VectorSearchEndpointPermissionLevel] ) diff --git a/bundle/config/resources/vector_search_endpoint.go b/bundle/config/resources/vector_search_endpoint.go index 5d7ef03ca2d..13f8d790a53 100644 --- a/bundle/config/resources/vector_search_endpoint.go +++ b/bundle/config/resources/vector_search_endpoint.go @@ -16,7 +16,7 @@ type VectorSearchEndpoint struct { BaseResource vectorsearch.CreateEndpoint - Permissions []Permission `json:"permissions,omitempty"` + Permissions []VectorSearchEndpointPermission `json:"permissions,omitempty"` } func (e *VectorSearchEndpoint) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 978d5799f45..de9caa09dc2 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -201,6 +201,9 @@ func TestResourcesBindSupport(t *testing.T) { Clusters: map[string]*resources.Cluster{ "my_cluster": {}, }, + InstancePools: map[string]*resources.InstancePool{ + "my_instance_pool": {}, + }, Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, @@ -353,6 +356,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockInstancePoolsAPI().EXPECT().GetByInstancePoolId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index a5a1b611357..26223630e77 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -144,10 +144,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return diags } -// warnForMissingGrantPrincipals warns for any grant that is missing a principal. -// grants[*].principal is optional in the SDK (json:"principal,omitempty") but the -// backend requires it. Grants exist on every securable, so match any resource type. -func warnForMissingGrantPrincipals(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +// errorForInvalidGrants errors for grants the backend rejects or that never converge: +// a missing principal is rejected, and an empty privileges list re-plans forever because +// the backend drops principals with no privileges. Erroring here (rather than warning) +// avoids a partial deploy where the securable is created before the grants call fails. +// Grants exist on every securable, so match any resource type. +func errorForInvalidGrants(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} _, err := dyn.MapByPattern( @@ -156,12 +158,20 @@ func warnForMissingGrantPrincipals(ctx context.Context, b *bundle.Bundle) diag.D func(p dyn.Path, v dyn.Value) (dyn.Value, error) { if isMissingOrEmptyString(v.Get("principal")) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, + Severity: diag.Error, Summary: "grant principal is required", Locations: v.Locations(), Paths: []dyn.Path{slices.Clone(p)}, }) } + if isMissingOrEmptySequence(v.Get("privileges")) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "grant privileges is required", + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + } return v, nil }, ) @@ -186,12 +196,24 @@ func isMissingOrEmptyString(v dyn.Value) bool { } } +// isMissingOrEmptySequence reports whether v is unset, null, or an empty sequence. +func isMissingOrEmptySequence(v dyn.Value) bool { + switch v.Kind() { + case dyn.KindInvalid, dyn.KindNil: + return true + case dyn.KindSequence: + return len(v.MustSequence()) == 0 + default: + return false + } +} + func (f *required) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := errorForMissingFields(ctx, b) + diags = diags.Extend(errorForInvalidGrants(ctx, b)) if diags.HasError() { return diags } diags = diags.Extend(warnForMissingFields(ctx, b)) - diags = diags.Extend(warnForMissingGrantPrincipals(ctx, b)) return diags } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index b688f644ba9..3deda5a8d7c 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -27,9 +27,10 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos return nil } - if _, ok := engine.Parse(string(configEngine)); !ok { - val := dyn.GetValue(b.Config.Value(), "bundle.engine") - loc := val.Location() + loc := dyn.GetValue(b.Config.Value(), "bundle.engine").Location() + + parsed, ok := engine.Parse(string(configEngine)) + if !ok { return diag.Diagnostics{{ Severity: diag.Error, Summary: fmt.Sprintf("invalid value %q for bundle.engine (expected %q or %q)", configEngine, engine.EngineTerraform, engine.EngineDirect), @@ -37,5 +38,14 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos }} } + if parsed == engine.EngineTerraform { + return diag.Diagnostics{{ + Severity: diag.Warning, + Summary: "the terraform deployment engine is deprecated and will stop working in a future version of the CLI", + Detail: "See https://docs.databricks.com/aws/en/dev-tools/bundles/direct for how to migrate to the direct deployment engine", + Locations: []dyn.Location{loc}, + }} + } + return nil } diff --git a/bundle/config/validate/validate_engine_test.go b/bundle/config/validate/validate_engine_test.go index f260b631f5c..9f18844546d 100644 --- a/bundle/config/validate/validate_engine_test.go +++ b/bundle/config/validate/validate_engine_test.go @@ -12,19 +12,34 @@ import ( "github.com/stretchr/testify/assert" ) -func TestValidateEngineValid(t *testing.T) { - for _, eng := range []engine.EngineType{engine.EngineTerraform, engine.EngineDirect} { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Engine: eng, - }, +func TestValidateEngineDirect(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Engine: engine.EngineDirect, }, - } - bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) - diags := ValidateEngine().Apply(t.Context(), b) - assert.Empty(t, diags) + }, } + bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) + diags := ValidateEngine().Apply(t.Context(), b) + assert.Empty(t, diags) +} + +func TestValidateEngineTerraformDeprecated(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Engine: engine.EngineTerraform, + }, + }, + } + loc := dyn.Location{File: "databricks.yml", Line: 5, Column: 3} + bundletest.SetLocation(b, "bundle.engine", []dyn.Location{loc}) + diags := ValidateEngine().Apply(t.Context(), b) + assert.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, "deprecated") + assert.Equal(t, []dyn.Location{loc}, diags[0].Locations) } func TestValidateEngineNotSet(t *testing.T) { diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index d61a8b9abdb..ea45903508b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -8,8 +8,10 @@ import ( "io/fs" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deployplan" @@ -153,12 +155,36 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En return deployBundle, nil } +// isPermissionsOrGrantsSubResource reports whether a plan resource key is a +// permissions or grants sub-resource ("resources...permissions" / +// ".grants"). It classifies the key structurally via config.GetNodeAndType, +// which keys on the path component (index 3), so a resource literally named +// "permissions" ("resources.jobs.permissions") is not misclassified. +func isPermissionsOrGrantsSubResource(resourceKey string) bool { + path, err := dyn.NewPathFromString(resourceKey) + if err != nil { + return false + } + _, nodeType := config.GetNodeAndType(path) + return strings.HasSuffix(nodeType, ".permissions") || strings.HasSuffix(nodeType, ".grants") +} + // ExtractChanges extracts the map of remote-vs-config changes from a deploy // plan. engine selects the LocalEdit comparison below. func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) (Changes, error) { changes := make(Changes) for resourceKey, entry := range plan.Plan { + // permissions and grants are emitted as their own plan keys + // ("resources...permissions" / ".grants"; see splitResourcePath + // in bundle/direct/bundle_plan.go). config-remote-sync cannot write them back + // to YAML: their fields (e.g. object_id) are server-populated with no source + // location, and bundle-level permissions have no per-resource YAML node at all, + // so resolving them would fail the whole sync. Skip them instead. + if isPermissionsOrGrantsSubResource(resourceKey) { + continue + } + resourceChanges := make(ResourceChanges) if entry.Changes != nil { diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index 317ac1f32ba..bb1acc682ba 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -3,6 +3,8 @@ package configsync import ( "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deployplan" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -247,3 +249,57 @@ func TestMatchPattern(t *testing.T) { }) } } + +// ExtractChanges must skip permissions/grants sub-resources: they are emitted as +// their own plan keys but cannot be written back to YAML, and resolving them +// (e.g. the server-populated object_id field, or a bundle-level permissions entry +// with no per-resource YAML node) otherwise fails the whole sync. +func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { + ctx := t.Context() + + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.my_job": { + Changes: deployplan.Changes{ + "description": {Action: deployplan.Update, New: nil, Remote: "remote-desc"}, + }, + }, + "resources.jobs.my_job.permissions": { + Changes: deployplan.Changes{ + "object_id": {Action: deployplan.Update, New: nil, Remote: "/jobs/123"}, + }, + }, + "resources.schemas.my_schema.grants": { + Changes: deployplan.Changes{ + "[0].privileges": {Action: deployplan.Update, New: nil, Remote: []any{"SELECT"}}, + }, + }, + // A resource literally named "permissions" is a regular resource, not a + // permissions sub-resource, so its changes must be kept (structural + // classification via config.GetNodeAndType, not a suffix match). + "resources.jobs.permissions": { + Changes: deployplan.Changes{ + "description": {Action: deployplan.Update, New: nil, Remote: "kept"}, + }, + }, + }} + + for _, eng := range []engine.EngineType{engine.EngineDirect, engine.EngineTerraform} { + t.Run(string(eng), func(t *testing.T) { + changes, err := ExtractChanges(ctx, &bundle.Bundle{}, plan, eng) + require.NoError(t, err) + + _, hasPerms := changes["resources.jobs.my_job.permissions"] + assert.False(t, hasPerms, "permissions sub-resource must be skipped") + _, hasGrants := changes["resources.schemas.my_schema.grants"] + assert.False(t, hasGrants, "grants sub-resource must be skipped") + + job, hasJob := changes["resources.jobs.my_job"] + require.True(t, hasJob, "regular resource changes must be kept") + assert.Contains(t, job, "description") + + namedPerms, hasNamedPerms := changes["resources.jobs.permissions"] + require.True(t, hasNamedPerms, "a resource named permissions must not be skipped") + assert.Contains(t, namedPerms, "description") + }) + } +} diff --git a/bundle/configsync/select.go b/bundle/configsync/select.go index fcd8549d0be..443c94423ae 100644 --- a/bundle/configsync/select.go +++ b/bundle/configsync/select.go @@ -1,11 +1,13 @@ package configsync import ( + "context" "fmt" "slices" "strings" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/log" ) // ResolveResourceSelectors maps ":" selectors to their plan keys @@ -21,11 +23,18 @@ import ( // is also why selection is independent from `bundle deploy --select`, which // matches "type.name" keys. // -// A selector that matches no deployed resource is an error: only deployed -// resources have an id, so a selector matching nothing is a caller mistake. +// A selector that matches no deployed resource is skipped rather than failing +// the run — but only when at least one other selector did match. The workspace +// UI batches every edited resource into one sync, so a single stale selector +// (a resource deleted remotely, or whose deploy state has drifted) must not +// drop the valid resources' edits. When NO selector matches, the error is +// returned instead: silently reporting "no changes" would let the UI show a +// success while nothing synced, hiding a real state problem. A malformed +// selector (missing ":" shape) is always an error, because that is a +// caller mistake rather than drift. // Duplicate selectors are deduplicated; the returned keys preserve the order in // which their selectors first appear. -func ResolveResourceSelectors(state *dstate.DeploymentState, selectors []string) ([]string, error) { +func ResolveResourceSelectors(ctx context.Context, state *dstate.DeploymentState, selectors []string) ([]string, error) { // Index deployed resources by ":". State keys have the form // "resources.."; indexing by the component means a // selector can only ever match a resource of that exact type, never an id @@ -48,6 +57,7 @@ func ResolveResourceSelectors(state *dstate.DeploymentState, selectors []string) } keys := make([]string, 0, len(selectors)) + var missing []string for _, selector := range selectors { resourceType, id, ok := strings.Cut(selector, ":") if !ok || resourceType == "" || id == "" { @@ -55,12 +65,26 @@ func ResolveResourceSelectors(state *dstate.DeploymentState, selectors []string) } key, ok := byTypeID[selector] if !ok { - return nil, fmt.Errorf("no deployed %s resource with id %s", resourceType, id) + missing = append(missing, selector) + continue } if !slices.Contains(keys, key) { keys = append(keys, key) } } + + // No selector matched a deployed resource: fail loudly instead of syncing + // nothing, so the caller does not report a spurious success. + if len(keys) == 0 { + resourceType, id, _ := strings.Cut(missing[0], ":") + return nil, fmt.Errorf("no deployed %s resource with id %s", resourceType, id) + } + + // Some selectors matched: skip the stale ones so the matched resources still + // sync (the UI batches several resources into one run). + for _, selector := range missing { + log.Debugf(ctx, "config-remote-sync: skipping selector %q, no deployed resource with that id", selector) + } return keys, nil } diff --git a/bundle/deploy/lock/acquire.go b/bundle/deploy/lock/acquire.go index 6e4844ca5ff..bf8b91212dd 100644 --- a/bundle/deploy/lock/acquire.go +++ b/bundle/deploy/lock/acquire.go @@ -10,12 +10,20 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/locker" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" ) -type acquire struct{} +// maxChildNodeSizeExceeded is the error_code the Workspace API returns (as a 403) +// when a directory is at its child-node limit and cannot accept a new child. +// The SDK has no sentinel for it, so we match on the code directly. +const maxChildNodeSizeExceeded = "MAX_CHILD_NODE_SIZE_EXCEEDED" -func Acquire() bundle.Mutator { - return &acquire{} +type acquire struct { + goal Goal +} + +func Acquire(goal Goal) bundle.Mutator { + return &acquire{goal} } func (m *acquire) Name() string { @@ -50,6 +58,19 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics log.Infof(ctx, "Acquiring deployment lock (force: %v)", force) err = b.Locker.Lock(ctx, force) if err != nil { + // When destroying with --force-lock, tolerate a full workspace directory + // that cannot accept the lock file: proceeding lock-less carries the same + // caveat --force-lock already documents (no guaranteed exclusive access), + // which is acceptable when the goal is to tear the deployment down. + // This check must precede the fs.ErrPermission branch below because the API + // reports the child-node limit as a 403, which the filer maps to that error. + if m.goal == GoalDestroy && force { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.ErrorCode == maxChildNodeSizeExceeded { + log.Warnf(ctx, "Proceeding with destroy without a deployment lock: %v", err) + return nil + } + } + log.Errorf(ctx, "Failed to acquire deployment lock: %v", err) if errors.Is(err, fs.ErrPermission) { diff --git a/bundle/deploy/lock/release.go b/bundle/deploy/lock/release.go index 26f95edfc95..7c8eef10e99 100644 --- a/bundle/deploy/lock/release.go +++ b/bundle/deploy/lock/release.go @@ -51,6 +51,12 @@ func (m *release) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics case GoalBind, GoalUnbind: return diag.FromErr(b.Locker.Unlock(ctx)) case GoalDestroy: + // Destroy may have proceeded without acquiring a lock (see lock.Acquire: + // --force-lock tolerates a workspace directory at its child-node limit). + // There is nothing to release in that case. + if !b.Locker.Active { + return nil + } return diag.FromErr(b.Locker.Unlock(ctx, locker.AllowLockFileNotExist)) default: return diag.Errorf("unknown goal for lock release: %s", m.goal) diff --git a/bundle/deploy/snapshot/path.go b/bundle/deploy/snapshot/path.go index 5aecc0148df..62a4b9f91ab 100644 --- a/bundle/deploy/snapshot/path.go +++ b/bundle/deploy/snapshot/path.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -24,11 +25,25 @@ import ( // the zip was built or the file's mtime. var zipEpoch = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) +// metadataFileName is the name of the metadata file embedded in every snapshot zip. +// It captures a hash of the ACL so that a change to top-level permissions always produces +// a different snapshot ID — necessary because immutable snapshots cannot be re-permissioned +// after creation. The hash avoids embedding principal names in the stored artifact. +const metadataFileName = ".databricks/snapshot-metadata.json" + +// snapshotMetadata is the structure written to metadataFileName inside the zip. +type snapshotMetadata struct { + // PermissionsHash is the SHA-256 hex digest of the JSON-serialized ACL. + PermissionsHash string `json:"permissions_hash"` +} + // BundleZip builds the zip that is uploaded to the snapshot API. // It contains: // - all files from the bundle sync root under the "files/" prefix, // selected with the same git-aware + include/exclude logic as files.Upload // - all built artifact files under the "artifacts/.internal/" prefix +// - a metadata file at metadataFileName that embeds the ACL so that any +// change to top-level permissions forces a new snapshot ID // // The snapshot ID is always IDFromContent(BundleZip(b)), ensuring the // pre-calculated path and the uploaded path are derived from the same content. @@ -44,6 +59,9 @@ func BundleZip(ctx context.Context, b *bundle.Bundle) ([]byte, int, error) { if err := addArtifactsToZip(zw, b); err != nil { return nil, 0, err } + if err := addMetadataToZip(zw, BuildACL(b)); err != nil { + return nil, 0, err + } if err := zw.Close(); err != nil { return nil, 0, err @@ -51,6 +69,30 @@ func BundleZip(ctx context.Context, b *bundle.Bundle) ([]byte, int, error) { return buf.Bytes(), fileCount, nil } +// addMetadataToZip writes the snapshot metadata file into the zip so that +// any change to the ACL changes the snapshot hash and forces a new snapshot. +func addMetadataToZip(zw *zip.Writer, acl []ACLEntry) error { + aclJSON, err := json.Marshal(acl) + if err != nil { + return fmt.Errorf("marshal ACL for permissions hash: %w", err) + } + data, err := json.Marshal(snapshotMetadata{PermissionsHash: IDFromContent(aclJSON)}) + if err != nil { + return fmt.Errorf("marshal snapshot metadata: %w", err) + } + h := &zip.FileHeader{ + Name: metadataFileName, + Method: zip.Deflate, + Modified: zipEpoch, + } + w, err := zw.CreateHeader(h) + if err != nil { + return fmt.Errorf("zip entry for %s: %w", metadataFileName, err) + } + _, err = w.Write(data) + return err +} + // IDFromContent returns the SHA-256 hex digest of content. func IDFromContent(content []byte) string { h := sha256.Sum256(content) diff --git a/bundle/deploy/snapshot/path_test.go b/bundle/deploy/snapshot/path_test.go index dd88a30e9ef..5f567d9f377 100644 --- a/bundle/deploy/snapshot/path_test.go +++ b/bundle/deploy/snapshot/path_test.go @@ -10,8 +10,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,7 +27,7 @@ func makeBundleWithFiles(t *testing.T, files map[string]string) *bundle.Bundle { require.NoError(t, os.WriteFile(p, []byte(content), 0o644)) } root := vfs.MustNew(dir) - return &bundle.Bundle{ + b := &bundle.Bundle{ BundleRootPath: dir, SyncRoot: root, // WorktreeRoot = SyncRoot is the fallback used by LoadGitDetails when @@ -36,9 +38,12 @@ func makeBundleWithFiles(t *testing.T, files map[string]string) *bundle.Bundle { // The SyncDefaultPath mutator sets this to ["."] during initialize; // set it here since these tests bypass the mutator pipeline. Empty // sync paths select no files. - Sync: config.Sync{Paths: []string{"."}}, + Sync: config.Sync{Paths: []string{"."}}, + Workspace: config.Workspace{CurrentUser: &config.User{}}, }, } + b.Config.Workspace.CurrentUser.User = &iam.User{UserName: "tester@example.com"} + return b } func TestBundleZipIsDeterministic(t *testing.T) { @@ -134,3 +139,33 @@ func TestBundleZipDoNotStripNotebookExtensions(t *testing.T) { assert.True(t, slices.Contains(names, "files/src/my_notebook.ipynb"), "notebook should keep its extension") assert.True(t, slices.Contains(names, "files/src/script.py"), "regular Python file should keep its extension") } + +func TestBundleZipIncludesMetadataFile(t *testing.T) { + b := makeBundleWithFiles(t, map[string]string{"task.py": "x = 1"}) + + zipContent, _, err := snapshot.BundleZip(t.Context(), b) + require.NoError(t, err) + + names := zipEntryNames(t, zipContent) + assert.True(t, slices.Contains(names, ".databricks/snapshot-metadata.json"), "zip must contain the snapshot metadata file") +} + +func TestBundleZipChangesWithPermissions(t *testing.T) { + files := map[string]string{"task.py": "x = 1"} + + bNoPerms := makeBundleWithFiles(t, files) + + bWithPerms := makeBundleWithFiles(t, files) + bWithPerms.Config.Permissions = []resources.Permission{ + {Level: "CAN_VIEW", GroupName: "data-team"}, + } + + zipNoPerms, _, err := snapshot.BundleZip(t.Context(), bNoPerms) + require.NoError(t, err) + zipWithPerms, _, err := snapshot.BundleZip(t.Context(), bWithPerms) + require.NoError(t, err) + + assert.NotEqual(t, zipNoPerms, zipWithPerms, "adding top-level permissions must produce a different snapshot zip") + assert.NotEqual(t, snapshot.IDFromContent(zipNoPerms), snapshot.IDFromContent(zipWithPerms), + "snapshot IDs must differ when top-level permissions change") +} diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 248a66c21a9..9fb59329ffd 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "genie_spaces", + "instance_pools", "job_runs", "vector_search_endpoints", "vector_search_indexes", diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index d227d7f564c..b3c46036c53 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -107,7 +107,10 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // place, matching the Terraform provider's recreate behaviour. err = retryOnTransientErr(ctx, func() error { return d.Adapter.DoDelete(ctx, oldID, oldState) }) if err != nil && !apierr.IsMissing(err) && !isManagedByParent(err) { - return fmt.Errorf("deleting old id=%s: %w", oldID, err) + if !d.deleteConfirmedGone(ctx, oldID) { + return fmt.Errorf("deleting old id=%s: %w", oldID, err) + } + log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } // Drop the state entry so a subsequent failure of Create or WaitAfterDelete @@ -225,6 +228,8 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, // mean configuration error that user is trying to fix by removing resource from their bundle. if errors.Is(err, apierr.ErrPermissionDenied) { log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) + } else if d.deleteConfirmedGone(ctx, oldID) { + log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } else { return fmt.Errorf("deleting id=%s: %w", oldID, err) } @@ -246,6 +251,25 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, return nil } +// deleteConfirmedGone reports whether a failed DoDelete can be treated as +// complete: the backend already removed the resource, or it is in a transient +// terminal-teardown state (e.g. an app in DELETING) that IsGone recognises and a +// retried delete would keep rejecting. This closes the plan/apply gap for saved +// plans: IsGone is consulted at plan time, but with `deploy --plan` the resource +// may enter that state only after the plan is saved. We re-read rather than match +// the delete error because the backend returns a generic 400 BAD_REQUEST (see +// apps/src/utils/AppsStatusUtils.scala) that carries no distinct SDK sentinel. +func (d *DeploymentUnit) deleteConfirmedGone(ctx context.Context, id string) bool { + remote, err := d.Adapter.DoRead(ctx, id) + if apierr.IsMissing(err) { + return true + } + if err != nil { + return false + } + return d.Adapter.IsGone(remote) +} + func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, id string, newState any, entry *deployplan.PlanEntry) error { err := retryOnTransientErr(ctx, func() error { return d.Adapter.DoResize(ctx, id, newState, entry) }) if err != nil { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 139012423ea..fda8180cfc4 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -191,6 +191,11 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks log.Warnf(ctx, "reading %s id=%q: %s", resourceKey, id, err) // This is not an error during deletion, so don't return false here } + } else if adapter.IsGone(remoteState) { + // The resource is in a transient terminal-teardown state (e.g. an app in + // DELETING) that a GET still returns but a second delete would reject. + // Treat it as gone: apply cleans up state without re-issuing the delete. + entry.Gone = true } entry.RemoteState = remoteState diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 70a9f1f5e3d..fdaa15bfcea 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -83,6 +83,13 @@ type IResource interface { // [Optional] KeyedSlices returns a map from path patterns to KeyFunc for comparing slices by key instead of by index. // Example: func (*ResourcePermissions) KeyedSlices(state *PermissionsState) map[string]any KeyedSlices() map[string]any + + // [Optional] IsGone reports whether a remote resource should be treated as + // already-deleted when planning a delete. Use for backends whose DELETE is + // asynchronous and leaves the resource in a transient terminal-teardown state + // (returned by GET, not 404) that rejects a second DELETE. + // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool + IsGone(remoteState any) bool } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -103,6 +110,7 @@ type Adapter struct { waitAfterDelete *calladapt.BoundCaller overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller + isGone *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -135,6 +143,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterUpdate: nil, waitAfterDelete: nil, overrideChangeDesc: nil, + isGone: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -231,6 +240,11 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.isGone, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "IsGone") + if err != nil { + return err + } + keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -312,6 +326,10 @@ func (a *Adapter) validate() error { validations = append(validations, "DoResize newState", a.doResize.InTypes[2], stateType) } + if a.isGone != nil { + validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -564,6 +582,19 @@ func (a *Adapter) KeyedSlices() map[string]any { return a.keyedSlices } +// IsGone reports whether the remote state represents an already-deleted resource +// for planning purposes. Resources that don't implement IsGone are never gone. +func (a *Adapter) IsGone(remoteState any) bool { + if a.isGone == nil { + return false + } + outs, err := a.isGone.Call(remoteState) + if err != nil { + return false + } + return outs[0].(bool) +} + // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 9856e6f979d..d6bcd03f73c 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -38,6 +38,7 @@ var SupportedResources = map[string]any{ "quality_monitors": (*ResourceQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), + "instance_pools": (*ResourceInstancePool)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -55,6 +56,7 @@ var SupportedResources = map[string]any{ "dashboards.permissions": (*ResourcePermissions)(nil), "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), + "instance_pools.permissions": (*ResourcePermissions)(nil), // Grants "catalogs.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index d79c80c61b2..54541d94ba9 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/database" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -79,6 +80,13 @@ var testConfig map[string]any = map[string]any{ }, }, + "instance_pools": &resources.InstancePool{ + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "my-instance-pool", + NodeTypeId: "i3.xlarge", + }, + }, + "synced_database_tables": &resources.SyncedDatabaseTable{ SyncedDatabaseTable: database.SyncedDatabaseTable{ Name: "main.myschema.my_synced_table", @@ -434,6 +442,16 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "instance_pools.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &PermissionsState{ + ObjectID: "/instance-pools/pool-permissions", + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "apps.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.Apps.Create(ctx, apps.CreateAppRequest{ App: apps.App{ @@ -1058,11 +1076,28 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + // Apps DoDelete is fire-and-forget: the API returns success while the app + // sits in DELETING state for up to ~20 minutes before the record is removed. + // A GET on the DELETING app returns the app, not 404 -- the testserver + // mirrors that in libs/testserver/apps.go. DoRead therefore still succeeds + // here; the planner treats this transient state as gone via ResourceApp.IsGone + // so delete/destroy stay idempotent (acceptance/bundle/invariant/{delete,destroy}_idempotent). + deleteLeavesDeleting := group == "apps" remoteAfterDelete, err := adapter.DoRead(ctx, createdID) - if deleteIsNoop { + switch { + case deleteIsNoop: require.NoError(t, err) - } else { + // The resource genuinely still exists, so it must not report as gone. + assert.False(t, adapter.IsGone(remoteAfterDelete)) + case deleteLeavesDeleting: + require.NoError(t, err) + require.NotNil(t, remoteAfterDelete) + // IsGone lets the planner short-circuit the second delete on this + // transient DELETING state; this is what keeps the delete/destroy + // invariant tests idempotent, so assert the contract directly. + assert.True(t, adapter.IsGone(remoteAfterDelete)) + default: require.Error(t, err) require.Nil(t, remoteAfterDelete) } diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8fc2eec34a0..6da6f3555ec 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -20,6 +20,8 @@ external_locations: catalog.CreateExternalLocation genie_spaces: dashboards.GenieUpdateSpaceRequest +instance_pools: compute.CreateInstancePool + job_runs: jobs.RunNow jobs: jobs.JobSettings diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index e629027b24b..29e1110513c 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -246,12 +246,18 @@ func hasAppChanges(entry *PlanEntry) bool { return entry.Changes.HasChangeExcept("source_code_path", "config", "git_source", "lifecycle", "lifecycle.started") } -// OverrideChangeDesc skips source_code_path drift when the remote value is empty. -// This happens when an app has no deployment yet (DefaultSourceCodePath is unset). +// OverrideChangeDesc skips drift on the deploy-only fields (source_code_path, config, +// git_source) while the app has no active deployment. DoRead reads them only from the +// active deployment, so before the first deploy (or once a stop clears it) the remote +// side is empty and the diff is spurious; it applies on the next start (manageLifecycle). func (*ResourceApp) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *AppRemote) error { - if path.String() == "source_code_path" && (remote.SourceCodePath == "" || remote.SourceCodePath == "null") { - change.Action = deployplan.Skip - change.Reason = "no deployment" + // Prefix(1) so a nested diff (e.g. config.command) matches its top-level field. + switch path.Prefix(1).String() { + case "source_code_path", "config", "git_source": + if remote.ActiveDeployment == nil { + change.Action = deployplan.Skip + change.Reason = "no active deployment" + } } return nil } @@ -302,6 +308,21 @@ func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) erro return err } +// IsGone treats a DELETING app as already-deleted. The Apps DELETE is +// fire-and-forget: it returns success while the app sits in +// ComputeState=DELETING for up to ~20 minutes, and a GET during that window +// returns the app (not 404), so callers cannot rely on IsMissing. A second +// DELETE while DELETING is rejected with 400, so without this the delete/destroy +// path is not idempotent (acceptance/bundle/invariant/{delete,destroy}_idempotent). +// Consulted both at plan time (bundle_plan.go) and after a failed apply-time +// delete (apply.go deleteConfirmedGone), which covers saved-plan deploys where +// the app enters DELETING only after the plan was computed. +// Still uncovered: a DELETING app hit during a normal deploy/update (not a +// delete) produces a spurious update/skip because plan/apply do not special-case it there. +func (*ResourceApp) IsGone(remote *AppRemote) bool { + return remote.ComputeStatus != nil && remote.ComputeStatus.State == apps.ComputeStateDeleting +} + func (r *ResourceApp) WaitAfterCreate(ctx context.Context, id string, config *AppState) (*AppRemote, error) { remote, err := r.waitForApp(ctx, r.client, config.Name) if err != nil { diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 6ded283c9f9..d253c1129ad 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -18,6 +18,13 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +// clusterWaitTimeout bounds how long we poll for a cluster to reach its target +// state (RUNNING/TERMINATED) after create, edit, or start/stop. Provisioning can +// legitimately take longer than 15 minutes on capacity-constrained workspaces +// (a cluster stays PENDING with "Finding instances for new nodes"), so we allow +// 30 minutes before giving up. Terminal states still halt immediately. +const clusterWaitTimeout = 30 * time.Minute + // ClusterState is the state type for Cluster resources. It extends compute.ClusterSpec with // lifecycle settings and the cluster ID. // ClusterId is written to state by DoCreate/DoUpdate for informational purposes; it is not @@ -90,6 +97,7 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { ClusterName: input.ClusterName, CustomTags: input.CustomTags, DataSecurityMode: input.DataSecurityMode, + DependencyMode: input.DependencyMode, DockerImage: input.DockerImage, DriverInstancePoolId: input.DriverInstancePoolId, DriverNodeTypeId: input.DriverNodeTypeId, @@ -166,8 +174,7 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust if hasClusterChanges(entry) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 - timeout := 15 * time.Minute - _, err := retries.Poll(ctx, timeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { + _, err := retries.Poll(ctx, clusterWaitTimeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, &config.ClusterSpec)) if err == nil { return wait, nil @@ -197,7 +204,7 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust return nil, err } else if !desiredStarted && alreadyRunning { // lifecycle.started=false: fire Delete; WaitAfterUpdate polls for TERMINATED. - // Delete does not remove the cluster, it just sets the state to TERMINATED. + // Note: Delete terminates the cluster; permanent removal is a separate API (permanent-delete). _, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: id}) return nil, err } @@ -212,11 +219,11 @@ func (r *ResourceCluster) WaitAfterUpdate(ctx context.Context, id string, config } if *config.Lifecycle.Started { - _, err := r.client.Clusters.WaitGetClusterRunning(ctx, id, 15*time.Minute, nil) + _, err := r.client.Clusters.WaitGetClusterRunning(ctx, id, clusterWaitTimeout, nil) return nil, err } - _, err := r.client.Clusters.WaitGetClusterTerminated(ctx, id, 15*time.Minute, nil) + _, err := r.client.Clusters.WaitGetClusterTerminated(ctx, id, clusterWaitTimeout, nil) return nil, err } @@ -224,18 +231,19 @@ func (r *ResourceCluster) WaitAfterUpdate(ctx context.Context, id string, config // When lifecycle.started=false, it then terminates the cluster. func (r *ResourceCluster) WaitAfterCreate(ctx context.Context, id string, config *ClusterState) (*ClusterRemote, error) { // Always wait for RUNNING first: clusters start in PENDING state and must be polled. - _, err := r.client.Clusters.WaitGetClusterRunning(ctx, id, 15*time.Minute, nil) + _, err := r.client.Clusters.WaitGetClusterRunning(ctx, id, clusterWaitTimeout, nil) if err != nil { return nil, err } if config.Lifecycle != nil && config.Lifecycle.Started != nil && !*config.Lifecycle.Started { // started=false: terminate the cluster after it reaches RUNNING. + // Note: Delete terminates the cluster; permanent removal is a separate API (permanent-delete). deleteWaiter, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: id}) if err != nil { return nil, err } - _, err = deleteWaiter.Get() + _, err = deleteWaiter.GetWithTimeout(clusterWaitTimeout) return nil, err } @@ -323,6 +331,7 @@ func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { CloneFrom: nil, // Not supported by DABs CustomTags: config.CustomTags, DataSecurityMode: config.DataSecurityMode, + DependencyMode: config.DependencyMode, DockerImage: config.DockerImage, DriverInstancePoolId: config.DriverInstancePoolId, DriverNodeTypeId: config.DriverNodeTypeId, @@ -372,6 +381,7 @@ func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster ClusterName: config.ClusterName, CustomTags: config.CustomTags, DataSecurityMode: config.DataSecurityMode, + DependencyMode: config.DependencyMode, DockerImage: config.DockerImage, DriverInstancePoolId: config.DriverInstancePoolId, DriverNodeTypeId: config.DriverNodeTypeId, diff --git a/bundle/direct/dresources/experiment.go b/bundle/direct/dresources/experiment.go index bedabc81365..e4f2e8ebbd7 100644 --- a/bundle/direct/dresources/experiment.go +++ b/bundle/direct/dresources/experiment.go @@ -27,6 +27,7 @@ func (*ResourceExperiment) PrepareState(input *resources.MlflowExperiment) *ml.C Name: input.Name, ArtifactLocation: input.ArtifactLocation, Tags: input.Tags, + TraceLocation: input.TraceLocation, ForceSendFields: utils.FilterFields[ml.CreateExperiment](input.ForceSendFields), } } @@ -36,6 +37,7 @@ func (*ResourceExperiment) RemapState(experiment *ml.Experiment) *ml.CreateExper Name: experiment.Name, ArtifactLocation: experiment.ArtifactLocation, Tags: experiment.Tags, + TraceLocation: experiment.TraceLocation, ForceSendFields: utils.FilterFields[ml.CreateExperiment](experiment.ForceSendFields), } } diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index bac70bc5615..f0a9423838c 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -48,10 +48,9 @@ func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVa return nil, fmt.Errorf("expected *[]catalog.PrivilegeAssignment, got %T", inputConfig) } - // Backend sorts privileges, so we sort here as well. - for i := range *grantsPtr { - slices.Sort((*grantsPtr)[i].Privileges) - } + // Normalize the same way as DoRead (sort, collapse ALL_PRIVILEGES) so the + // config and the value read back compare equal. + normalizeAssignments(*grantsPtr) return &structvar.StructVar{ Value: &GrantsState{ @@ -222,9 +221,28 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName } pageToken = resp.NextPageToken } + // Normalize the same way as the config side (sort, collapse ALL_PRIVILEGES) + // so the two compare equal and we don't report false drift. + normalizeAssignments(assignments) return assignments, nil } +// normalizeAssignments sorts each assignment's privileges (the backend sorts +// them, so we match that) and collapses a principal holding ALL_PRIVILEGES down +// to just ALL_PRIVILEGES. The collapse is applied to both the config and read +// sides, so config granting only ALL_PRIVILEGES matches a backend that reports +// ALL_PRIVILEGES plus the concrete privileges it implies, instead of reporting a +// perpetual update. +func normalizeAssignments(assignments []catalog.PrivilegeAssignment) { + for i := range assignments { + if slices.Contains(assignments[i].Privileges, catalog.PrivilegeAllPrivileges) { + assignments[i].Privileges = []catalog.Privilege{catalog.PrivilegeAllPrivileges} + continue + } + slices.Sort(assignments[i].Privileges) + } +} + func extractGrantResourceType(node string) (string, error) { rest, ok := strings.CutPrefix(node, "resources.") if !ok { diff --git a/bundle/direct/dresources/grants_test.go b/bundle/direct/dresources/grants_test.go index 1724eed40dc..a1c5895a33d 100644 --- a/bundle/direct/dresources/grants_test.go +++ b/bundle/direct/dresources/grants_test.go @@ -74,3 +74,41 @@ func TestBuildGrantChanges(t *testing.T) { }) } } + +func TestNormalizeAssignments(t *testing.T) { + tests := []struct { + name string + input []catalog.PrivilegeAssignment + expected []catalog.PrivilegeAssignment + }{ + { + name: "sorts privileges", + input: []catalog.PrivilegeAssignment{ + {Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeUseSchema, catalog.PrivilegeApplyTag}}, + }, + expected: []catalog.PrivilegeAssignment{ + {Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeApplyTag, catalog.PrivilegeUseSchema}}, + }, + }, + { + // Regression test for #6030: ALL_PRIVILEGES implies every concrete + // privilege, so a principal holding it collapses to just + // ALL_PRIVILEGES. Applied to both config and remote, this stops the + // backend's extra concrete privileges from showing as drift. + name: "collapses ALL_PRIVILEGES with concrete privileges", + input: []catalog.PrivilegeAssignment{ + {Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeUseCatalog, catalog.PrivilegeAllPrivileges, catalog.PrivilegeCreateSchema}}, + }, + expected: []catalog.PrivilegeAssignment{ + {Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeAllPrivileges}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + normalizeAssignments(tt.input) + assert.Equal(t, tt.expected, tt.input) + }) + } +} diff --git a/bundle/direct/dresources/instance_pool.go b/bundle/direct/dresources/instance_pool.go new file mode 100644 index 00000000000..37a1aeab447 --- /dev/null +++ b/bundle/direct/dresources/instance_pool.go @@ -0,0 +1,77 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceInstancePool struct { + client *databricks.WorkspaceClient +} + +func (*ResourceInstancePool) New(client *databricks.WorkspaceClient) *ResourceInstancePool { + return &ResourceInstancePool{client: client} +} + +func (*ResourceInstancePool) PrepareState(input *resources.InstancePool) *compute.CreateInstancePool { + return &input.CreateInstancePool +} + +// RemapState copies the config fields shared by GetInstancePool and CreateInstancePool; +// output-only fields (state, stats, default_tags, instance_pool_id) are not in the state. +func (*ResourceInstancePool) RemapState(remote *compute.GetInstancePool) *compute.CreateInstancePool { + return &compute.CreateInstancePool{ + AwsAttributes: remote.AwsAttributes, + AzureAttributes: remote.AzureAttributes, + CustomTags: remote.CustomTags, + DiskSpec: remote.DiskSpec, + EnableElasticDisk: remote.EnableElasticDisk, + GcpAttributes: remote.GcpAttributes, + IdleInstanceAutoterminationMinutes: remote.IdleInstanceAutoterminationMinutes, + InstancePoolName: remote.InstancePoolName, + MaxCapacity: remote.MaxCapacity, + MinIdleInstances: remote.MinIdleInstances, + NodeTypeFlexibility: remote.NodeTypeFlexibility, + NodeTypeId: remote.NodeTypeId, + PreloadedDockerImages: remote.PreloadedDockerImages, + PreloadedSparkVersions: remote.PreloadedSparkVersions, + RemoteDiskThroughput: remote.RemoteDiskThroughput, + TotalInitialRemoteDiskSize: remote.TotalInitialRemoteDiskSize, + ForceSendFields: utils.FilterFields[compute.CreateInstancePool](remote.ForceSendFields), + } +} + +func (r *ResourceInstancePool) DoRead(ctx context.Context, id string) (*compute.GetInstancePool, error) { + return r.client.InstancePools.GetByInstancePoolId(ctx, id) +} + +func (r *ResourceInstancePool) DoCreate(ctx context.Context, config *compute.CreateInstancePool) (string, *compute.GetInstancePool, error) { + resp, err := r.client.InstancePools.Create(ctx, *config) + if err != nil { + return "", nil, err + } + return resp.InstancePoolId, nil, nil +} + +func (r *ResourceInstancePool) DoUpdate(ctx context.Context, id string, config *compute.CreateInstancePool, _ *PlanEntry) (*compute.GetInstancePool, error) { + return nil, r.client.InstancePools.Edit(ctx, compute.EditInstancePool{ + InstancePoolId: id, + InstancePoolName: config.InstancePoolName, + NodeTypeId: config.NodeTypeId, + MinIdleInstances: config.MinIdleInstances, + MaxCapacity: config.MaxCapacity, + IdleInstanceAutoterminationMinutes: config.IdleInstanceAutoterminationMinutes, + CustomTags: config.CustomTags, + RemoteDiskThroughput: config.RemoteDiskThroughput, + TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, + ForceSendFields: utils.FilterFields[compute.EditInstancePool](config.ForceSendFields), + }) +} + +func (r *ResourceInstancePool) DoDelete(ctx context.Context, id string, _ *compute.CreateInstancePool) error { + return r.client.InstancePools.DeleteByInstancePoolId(ctx, id) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 6e1fa79d811..e99311757a2 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -17,6 +17,7 @@ var permissionResourceToObjectType = map[string]string{ "alerts": "/alertsv2/", "apps": "/apps/", "clusters": "/clusters/", + "instance_pools": "/instance-pools/", "dashboards": "/dashboards/", "genie_spaces": "/genie/", "database_instances": "/database-instances/", diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index e4154180970..61a32eaacf5 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -160,7 +160,15 @@ resources: - field: uid reason: spec:output_only - # experiments: no api field behaviors + experiments: + + recreate_on_changes: + - field: trace_location + reason: spec:immutable + + ignore_remote_changes: + - field: trace_location.uc_trace_location.effective_table_prefix + reason: spec:output_only external_locations: @@ -176,6 +184,8 @@ resources: - field: etag reason: spec:output_only + # instance_pools: no api field behaviors + # job_runs: no api field behaviors # jobs: no api field behaviors @@ -315,6 +325,8 @@ resources: reason: spec:input_only - field: existing_pipeline_id reason: spec:input_only + - field: extra_columns + reason: spec:input_only - field: new_pipeline_spec reason: spec:input_only - field: postgres_database @@ -332,7 +344,11 @@ resources: # quality_monitors: no api field behaviors - # registered_models: no api field behaviors + registered_models: + + ignore_remote_changes: + - field: browse_only + reason: spec:output_only # schemas: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c0f640f9c46..c9df091551f 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -343,9 +343,6 @@ resources: reason: output_only - field: updated_by reason: output_only - # Backend-computed; the user never sets it. Not annotated output_only in the spec. - - field: browse_only - reason: output_only # Aliases are managed on model versions through a separate API, and DoRead # passes IncludeAliases=false, so GET never echoes them back. Without this a # config that sets aliases reports a perpetual update (remote stays empty). @@ -598,6 +595,32 @@ resources: # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" #- field: data_security_mode + instance_pools: + # Field behaviors follow the TF provider tags cross-referenced with the edit API (compute.EditInstancePool): + # https://github.com/databricks/terraform-provider-databricks/blob/main/pools/resource_instance_pool.go + ignore_remote_changes: + # Backend fills cloud defaults the user omits; treated as managed like clusters above. + - field: aws_attributes + reason: managed + - field: azure_attributes + reason: managed + - field: gcp_attributes + reason: managed + recreate_on_changes: + # force_new and not accepted by /instance-pools/edit. + - field: disk_spec + - field: node_type_flexibility + - field: preloaded_spark_versions + - field: preloaded_docker_images + backend_defaults: + # Defaults to true server-side. + - field: enable_elastic_disk + # Backend applies a default of 60 minutes when the field is omitted. + - field: idle_instance_autotermination_minutes + values: [60] + # GCP seeds max_capacity (1000) when omitted; unconstrained as the default is cloud-dependent. + - field: max_capacity + sql_warehouses: ignore_remote_changes: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/sql/resource_sql_endpoint.go#L62 diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 70188663da1..f6c8fc8ba3c 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -16,7 +16,9 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/google/uuid" ) @@ -127,8 +129,10 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d db.Data.State = make(map[string]ResourceEntry) } - // don't indent so that every WAL entry remains on a single line - jsonMessage, err := json.Marshal(state) + // Redact sensitive fields before persisting: secrets must not appear on disk + // in plaintext. The original struct is not modified; the plan uses the + // unredacted in-memory value for API calls. + jsonMessage, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return err } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 247c1684977..4db1d40524d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -912,6 +912,59 @@ resources: "warehouse_id": "description": |- ID of the SQL warehouse used to run queries for this Genie space. + "instance_pools": + "description": |- + The instance pool definitions for the bundle, where each key is the name of the instance pool. + "markdown_description": |- + The instance pool definitions for the bundle, where each key is the name of the instance pool. See [\_](/dev-tools/bundles/resources.md#instance_pools). + "$type": + "markdown_description": |- + The instance pool resource defines an [instance pool](/api/workspace/instancepools/create), a set of idle, ready-to-use cloud instances that reduce cluster start and auto-scaling times. + "markdown_examples": |- + The following example creates an instance pool named `my_instance_pool`: + + ```yaml + resources: + instance_pools: + my_instance_pool: + instance_pool_name: my_instance_pool + node_type_id: i3.xlarge + min_idle_instances: 2 + max_capacity: 10 + idle_instance_autotermination_minutes: 60 + ``` + "$fields": + "disk_spec": + "$fields": + "disk_iops": + "description": |- + The number of IOPS to provision for each attached disk. + "disk_throughput": + "description": |- + The disk throughput to provision for each attached disk, in MB per second. + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "permissions": + "description": |- + The permissions to apply to this resource. + "markdown_description": |- + A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping. + + See [\_](/dev-tools/bundles/settings.md#permissions) and [\_](/dev-tools/bundles/permissions.md). + "$fields": + "group_name": + "description": |- + The name of the group granted the permission level. + "level": + "description": |- + The permission level to apply. The allowed levels depend on the resource type. + "service_principal_name": + "description": |- + The name of the service principal granted the permission level. + "user_name": + "description": |- + The name of the user granted the permission level. "job_runs": "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. @@ -1572,6 +1625,9 @@ resources: "existing_pipeline_id": "description": |- PLACEHOLDER + "extra_columns": + "description": |- + PLACEHOLDER "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. @@ -1908,6 +1964,19 @@ resources: - level: CAN_RUN service_principal_name: 123456-abcdef ``` + "$fields": + "group_name": + "description": |- + PLACEHOLDER + "level": + "description": |- + PLACEHOLDER + "service_principal_name": + "description": |- + PLACEHOLDER + "user_name": + "description": |- + PLACEHOLDER "vector_search_indexes": "description": |- PLACEHOLDER diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index ca8b1b5980a..653bc8a40d5 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,4 +1,4 @@ package schema // ProviderVersion is the version of the Databricks Terraform provider used for codegen. -const ProviderVersion = "1.121.0" +const ProviderVersion = "1.123.0" diff --git a/bundle/internal/tf/schema/data_source_alert_v2.go b/bundle/internal/tf/schema/data_source_alert_v2.go index c609a8f9189..bb45eb55914 100644 --- a/bundle/internal/tf/schema/data_source_alert_v2.go +++ b/bundle/internal/tf/schema/data_source_alert_v2.go @@ -53,6 +53,12 @@ type DataSourceAlertV2Evaluation struct { Threshold *DataSourceAlertV2EvaluationThreshold `json:"threshold,omitempty"` } +type DataSourceAlertV2Parameters struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Value string `json:"value,omitempty"` +} + type DataSourceAlertV2ProviderConfig struct { WorkspaceId string `json:"workspace_id,omitempty"` } @@ -78,6 +84,7 @@ type DataSourceAlertV2 struct { Id string `json:"id"` LifecycleState string `json:"lifecycle_state,omitempty"` OwnerUserName string `json:"owner_user_name,omitempty"` + Parameters []DataSourceAlertV2Parameters `json:"parameters,omitempty"` ParentPath string `json:"parent_path,omitempty"` ProviderConfig *DataSourceAlertV2ProviderConfig `json:"provider_config,omitempty"` QueryText string `json:"query_text,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_alerts_v2.go b/bundle/internal/tf/schema/data_source_alerts_v2.go index 5675e0aa56c..d61adb2929d 100644 --- a/bundle/internal/tf/schema/data_source_alerts_v2.go +++ b/bundle/internal/tf/schema/data_source_alerts_v2.go @@ -53,6 +53,12 @@ type DataSourceAlertsV2AlertsEvaluation struct { Threshold *DataSourceAlertsV2AlertsEvaluationThreshold `json:"threshold,omitempty"` } +type DataSourceAlertsV2AlertsParameters struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Value string `json:"value,omitempty"` +} + type DataSourceAlertsV2AlertsProviderConfig struct { WorkspaceId string `json:"workspace_id,omitempty"` } @@ -78,6 +84,7 @@ type DataSourceAlertsV2Alerts struct { Id string `json:"id"` LifecycleState string `json:"lifecycle_state,omitempty"` OwnerUserName string `json:"owner_user_name,omitempty"` + Parameters []DataSourceAlertsV2AlertsParameters `json:"parameters,omitempty"` ParentPath string `json:"parent_path,omitempty"` ProviderConfig *DataSourceAlertsV2AlertsProviderConfig `json:"provider_config,omitempty"` QueryText string `json:"query_text,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index 5d65b2f989b..0623526dcf1 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -348,6 +348,7 @@ type DataSourceClusterClusterInfoSpec struct { ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` @@ -413,6 +414,7 @@ type DataSourceClusterClusterInfo struct { CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` DefaultTags map[string]string `json:"default_tags,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_data_classification_catalog_config.go b/bundle/internal/tf/schema/data_source_data_classification_catalog_config.go index 9ad6965e041..5b2b55193af 100644 --- a/bundle/internal/tf/schema/data_source_data_classification_catalog_config.go +++ b/bundle/internal/tf/schema/data_source_data_classification_catalog_config.go @@ -7,6 +7,10 @@ type DataSourceDataClassificationCatalogConfigAutoTagConfigs struct { ClassificationTag string `json:"classification_tag"` } +type DataSourceDataClassificationCatalogConfigExcludedSchemas struct { + Names []string `json:"names"` +} + type DataSourceDataClassificationCatalogConfigIncludedSchemas struct { Names []string `json:"names"` } @@ -17,6 +21,7 @@ type DataSourceDataClassificationCatalogConfigProviderConfig struct { type DataSourceDataClassificationCatalogConfig struct { AutoTagConfigs []DataSourceDataClassificationCatalogConfigAutoTagConfigs `json:"auto_tag_configs,omitempty"` + ExcludedSchemas *DataSourceDataClassificationCatalogConfigExcludedSchemas `json:"excluded_schemas,omitempty"` IncludedSchemas *DataSourceDataClassificationCatalogConfigIncludedSchemas `json:"included_schemas,omitempty"` Name string `json:"name"` ProviderConfig *DataSourceDataClassificationCatalogConfigProviderConfig `json:"provider_config,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_disaster_recovery_stable_url.go b/bundle/internal/tf/schema/data_source_disaster_recovery_stable_url.go index f9dd842b9cc..b74a5f1bfc9 100644 --- a/bundle/internal/tf/schema/data_source_disaster_recovery_stable_url.go +++ b/bundle/internal/tf/schema/data_source_disaster_recovery_stable_url.go @@ -3,8 +3,10 @@ package schema type DataSourceDisasterRecoveryStableUrl struct { - FailoverGroupName string `json:"failover_group_name,omitempty"` - InitialWorkspaceId string `json:"initial_workspace_id,omitempty"` - Name string `json:"name"` - Url string `json:"url,omitempty"` + EffectiveWorkspaceId string `json:"effective_workspace_id,omitempty"` + FailoverGroupName string `json:"failover_group_name,omitempty"` + InitialWorkspaceId string `json:"initial_workspace_id,omitempty"` + Name string `json:"name"` + StableWorkspaceId string `json:"stable_workspace_id,omitempty"` + Url string `json:"url,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_disaster_recovery_stable_urls.go b/bundle/internal/tf/schema/data_source_disaster_recovery_stable_urls.go index 8103667696e..86ceb56f6b8 100644 --- a/bundle/internal/tf/schema/data_source_disaster_recovery_stable_urls.go +++ b/bundle/internal/tf/schema/data_source_disaster_recovery_stable_urls.go @@ -3,10 +3,12 @@ package schema type DataSourceDisasterRecoveryStableUrlsStableUrls struct { - FailoverGroupName string `json:"failover_group_name,omitempty"` - InitialWorkspaceId string `json:"initial_workspace_id,omitempty"` - Name string `json:"name"` - Url string `json:"url,omitempty"` + EffectiveWorkspaceId string `json:"effective_workspace_id,omitempty"` + FailoverGroupName string `json:"failover_group_name,omitempty"` + InitialWorkspaceId string `json:"initial_workspace_id,omitempty"` + Name string `json:"name"` + StableWorkspaceId string `json:"stable_workspace_id,omitempty"` + Url string `json:"url,omitempty"` } type DataSourceDisasterRecoveryStableUrls struct { diff --git a/bundle/internal/tf/schema/data_source_endpoint.go b/bundle/internal/tf/schema/data_source_endpoint.go index 39e726b8b1f..1fd4ac10f57 100644 --- a/bundle/internal/tf/schema/data_source_endpoint.go +++ b/bundle/internal/tf/schema/data_source_endpoint.go @@ -2,6 +2,12 @@ package schema +type DataSourceEndpointAwsVpcEndpointInfo struct { + AwsAccountId string `json:"aws_account_id,omitempty"` + AwsEndpointServiceId string `json:"aws_endpoint_service_id,omitempty"` + AwsVpcEndpointId string `json:"aws_vpc_endpoint_id"` +} + type DataSourceEndpointAzurePrivateEndpointInfo struct { PrivateEndpointName string `json:"private_endpoint_name"` PrivateEndpointResourceGuid string `json:"private_endpoint_resource_guid"` @@ -9,12 +15,22 @@ type DataSourceEndpointAzurePrivateEndpointInfo struct { PrivateLinkServiceId string `json:"private_link_service_id,omitempty"` } +type DataSourceEndpointGcpPscEndpointInfo struct { + EndpointRegion string `json:"endpoint_region"` + ProjectId string `json:"project_id"` + PscConnectionId string `json:"psc_connection_id,omitempty"` + PscEndpoint string `json:"psc_endpoint"` + ServiceAttachmentId string `json:"service_attachment_id,omitempty"` +} + type DataSourceEndpoint struct { AccountId string `json:"account_id,omitempty"` + AwsVpcEndpointInfo *DataSourceEndpointAwsVpcEndpointInfo `json:"aws_vpc_endpoint_info,omitempty"` AzurePrivateEndpointInfo *DataSourceEndpointAzurePrivateEndpointInfo `json:"azure_private_endpoint_info,omitempty"` CreateTime string `json:"create_time,omitempty"` DisplayName string `json:"display_name,omitempty"` EndpointId string `json:"endpoint_id,omitempty"` + GcpPscEndpointInfo *DataSourceEndpointGcpPscEndpointInfo `json:"gcp_psc_endpoint_info,omitempty"` Name string `json:"name"` Region string `json:"region,omitempty"` State string `json:"state,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_endpoints.go b/bundle/internal/tf/schema/data_source_endpoints.go index 1cef5b123aa..56dc425760e 100644 --- a/bundle/internal/tf/schema/data_source_endpoints.go +++ b/bundle/internal/tf/schema/data_source_endpoints.go @@ -2,6 +2,12 @@ package schema +type DataSourceEndpointsItemsAwsVpcEndpointInfo struct { + AwsAccountId string `json:"aws_account_id,omitempty"` + AwsEndpointServiceId string `json:"aws_endpoint_service_id,omitempty"` + AwsVpcEndpointId string `json:"aws_vpc_endpoint_id"` +} + type DataSourceEndpointsItemsAzurePrivateEndpointInfo struct { PrivateEndpointName string `json:"private_endpoint_name"` PrivateEndpointResourceGuid string `json:"private_endpoint_resource_guid"` @@ -9,12 +15,22 @@ type DataSourceEndpointsItemsAzurePrivateEndpointInfo struct { PrivateLinkServiceId string `json:"private_link_service_id,omitempty"` } +type DataSourceEndpointsItemsGcpPscEndpointInfo struct { + EndpointRegion string `json:"endpoint_region"` + ProjectId string `json:"project_id"` + PscConnectionId string `json:"psc_connection_id,omitempty"` + PscEndpoint string `json:"psc_endpoint"` + ServiceAttachmentId string `json:"service_attachment_id,omitempty"` +} + type DataSourceEndpointsItems struct { AccountId string `json:"account_id,omitempty"` + AwsVpcEndpointInfo *DataSourceEndpointsItemsAwsVpcEndpointInfo `json:"aws_vpc_endpoint_info,omitempty"` AzurePrivateEndpointInfo *DataSourceEndpointsItemsAzurePrivateEndpointInfo `json:"azure_private_endpoint_info,omitempty"` CreateTime string `json:"create_time,omitempty"` DisplayName string `json:"display_name,omitempty"` EndpointId string `json:"endpoint_id,omitempty"` + GcpPscEndpointInfo *DataSourceEndpointsItemsGcpPscEndpointInfo `json:"gcp_psc_endpoint_info,omitempty"` Name string `json:"name"` Region string `json:"region,omitempty"` State string `json:"state,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_feature.go b/bundle/internal/tf/schema/data_source_feature_engineering_feature.go index 103d9a9da2f..ea1ec37cee9 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_feature.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_feature.go @@ -29,10 +29,30 @@ type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirst struct Input string `json:"input"` } +type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstN struct { + Input string `json:"input"` + N int `json:"n"` +} + type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLast struct { Input string `json:"input"` } +type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLastDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLastN struct { + Input string `json:"input"` + N int `json:"n"` +} + type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionMax struct { Input string `json:"input"` } @@ -60,12 +80,17 @@ type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowCon type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTumbling struct { @@ -75,6 +100,7 @@ type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTum type DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindow struct { Continuous *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowContinuous `json:"continuous,omitempty"` Rolling *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSliding `json:"sliding,omitempty"` Tumbling *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTumbling `json:"tumbling,omitempty"` } @@ -93,7 +119,11 @@ type DataSourceFeatureEngineeringFeatureFunctionAggregationFunction struct { Avg *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionAvg `json:"avg,omitempty"` CountFunction *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionCountFunction `json:"count_function,omitempty"` First *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirst `json:"first,omitempty"` + FirstDistinct *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstDistinct `json:"first_distinct,omitempty"` + FirstN *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstN `json:"first_n,omitempty"` Last *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLast `json:"last,omitempty"` + LastDistinct *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLastDistinct `json:"last_distinct,omitempty"` + LastN *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionLastN `json:"last_n,omitempty"` Max *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionMax `json:"max,omitempty"` Min *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionMin `json:"min,omitempty"` StddevPop *DataSourceFeatureEngineeringFeatureFunctionAggregationFunctionStddevPop `json:"stddev_pop,omitempty"` @@ -172,8 +202,10 @@ type DataSourceFeatureEngineeringFeatureSourceRequestSource struct { } type DataSourceFeatureEngineeringFeatureSourceStreamSource struct { - FilterCondition string `json:"filter_condition,omitempty"` - FullName string `json:"full_name"` + DataframeSchema string `json:"dataframe_schema,omitempty"` + FilterCondition string `json:"filter_condition,omitempty"` + FullName string `json:"full_name"` + TransformationSql string `json:"transformation_sql,omitempty"` } type DataSourceFeatureEngineeringFeatureSource struct { @@ -190,12 +222,17 @@ type DataSourceFeatureEngineeringFeatureTimeWindowContinuous struct { type DataSourceFeatureEngineeringFeatureTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type DataSourceFeatureEngineeringFeatureTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeatureTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeatureTimeWindowTumbling struct { @@ -205,6 +242,7 @@ type DataSourceFeatureEngineeringFeatureTimeWindowTumbling struct { type DataSourceFeatureEngineeringFeatureTimeWindow struct { Continuous *DataSourceFeatureEngineeringFeatureTimeWindowContinuous `json:"continuous,omitempty"` Rolling *DataSourceFeatureEngineeringFeatureTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *DataSourceFeatureEngineeringFeatureTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *DataSourceFeatureEngineeringFeatureTimeWindowSliding `json:"sliding,omitempty"` Tumbling *DataSourceFeatureEngineeringFeatureTimeWindowTumbling `json:"tumbling,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_features.go b/bundle/internal/tf/schema/data_source_feature_engineering_features.go index 9da09724ce8..a153b7237cb 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_features.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_features.go @@ -29,10 +29,30 @@ type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirs Input string `json:"input"` } +type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirstDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirstN struct { + Input string `json:"input"` + N int `json:"n"` +} + type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLast struct { Input string `json:"input"` } +type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLastDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLastN struct { + Input string `json:"input"` + N int `json:"n"` +} + type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionMax struct { Input string `json:"input"` } @@ -60,12 +80,17 @@ type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTime type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowTumbling struct { @@ -75,6 +100,7 @@ type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTime type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindow struct { Continuous *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowContinuous `json:"continuous,omitempty"` Rolling *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowSliding `json:"sliding,omitempty"` Tumbling *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionTimeWindowTumbling `json:"tumbling,omitempty"` } @@ -93,7 +119,11 @@ type DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunction str Avg *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionAvg `json:"avg,omitempty"` CountFunction *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionCountFunction `json:"count_function,omitempty"` First *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirst `json:"first,omitempty"` + FirstDistinct *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirstDistinct `json:"first_distinct,omitempty"` + FirstN *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionFirstN `json:"first_n,omitempty"` Last *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLast `json:"last,omitempty"` + LastDistinct *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLastDistinct `json:"last_distinct,omitempty"` + LastN *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionLastN `json:"last_n,omitempty"` Max *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionMax `json:"max,omitempty"` Min *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionMin `json:"min,omitempty"` StddevPop *DataSourceFeatureEngineeringFeaturesFeaturesFunctionAggregationFunctionStddevPop `json:"stddev_pop,omitempty"` @@ -172,8 +202,10 @@ type DataSourceFeatureEngineeringFeaturesFeaturesSourceRequestSource struct { } type DataSourceFeatureEngineeringFeaturesFeaturesSourceStreamSource struct { - FilterCondition string `json:"filter_condition,omitempty"` - FullName string `json:"full_name"` + DataframeSchema string `json:"dataframe_schema,omitempty"` + FilterCondition string `json:"filter_condition,omitempty"` + FullName string `json:"full_name"` + TransformationSql string `json:"transformation_sql,omitempty"` } type DataSourceFeatureEngineeringFeaturesFeaturesSource struct { @@ -190,12 +222,17 @@ type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowContinuous struct { type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowTumbling struct { @@ -205,6 +242,7 @@ type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowTumbling struct { type DataSourceFeatureEngineeringFeaturesFeaturesTimeWindow struct { Continuous *DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowContinuous `json:"continuous,omitempty"` Rolling *DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowSliding `json:"sliding,omitempty"` Tumbling *DataSourceFeatureEngineeringFeaturesFeaturesTimeWindowTumbling `json:"tumbling,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_kafka_config.go b/bundle/internal/tf/schema/data_source_feature_engineering_kafka_config.go index eb7cbb47c01..4823fbc0f33 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_kafka_config.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_kafka_config.go @@ -72,8 +72,15 @@ type DataSourceFeatureEngineeringKafkaConfigIngestionConfig struct { IngestionPipelineId string `json:"ingestion_pipeline_id,omitempty"` } +type DataSourceFeatureEngineeringKafkaConfigKeySchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type DataSourceFeatureEngineeringKafkaConfigKeySchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *DataSourceFeatureEngineeringKafkaConfigKeySchemaProtoSchema `json:"proto_schema,omitempty"` } type DataSourceFeatureEngineeringKafkaConfigProviderConfig struct { @@ -86,8 +93,15 @@ type DataSourceFeatureEngineeringKafkaConfigSubscriptionMode struct { SubscribePattern string `json:"subscribe_pattern,omitempty"` } +type DataSourceFeatureEngineeringKafkaConfigValueSchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type DataSourceFeatureEngineeringKafkaConfigValueSchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *DataSourceFeatureEngineeringKafkaConfigValueSchemaProtoSchema `json:"proto_schema,omitempty"` } type DataSourceFeatureEngineeringKafkaConfig struct { diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_kafka_configs.go b/bundle/internal/tf/schema/data_source_feature_engineering_kafka_configs.go index 06fd36a903b..eff88ba43a6 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_kafka_configs.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_kafka_configs.go @@ -72,8 +72,15 @@ type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsIngestionConfig struct IngestionPipelineId string `json:"ingestion_pipeline_id,omitempty"` } +type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsKeySchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsKeySchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsKeySchemaProtoSchema `json:"proto_schema,omitempty"` } type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsProviderConfig struct { @@ -86,8 +93,15 @@ type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsSubscriptionMode struct SubscribePattern string `json:"subscribe_pattern,omitempty"` } +type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsValueSchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsValueSchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *DataSourceFeatureEngineeringKafkaConfigsKafkaConfigsValueSchemaProtoSchema `json:"proto_schema,omitempty"` } type DataSourceFeatureEngineeringKafkaConfigsKafkaConfigs struct { diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_materialized_feature.go b/bundle/internal/tf/schema/data_source_feature_engineering_materialized_feature.go index eeaf8d096d1..fedd8da0f7e 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_materialized_feature.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_materialized_feature.go @@ -24,7 +24,8 @@ type DataSourceFeatureEngineeringMaterializedFeatureProviderConfig struct { } type DataSourceFeatureEngineeringMaterializedFeatureStreamingMode struct { - Mode string `json:"mode,omitempty"` + FreshnessTarget string `json:"freshness_target,omitempty"` + Mode string `json:"mode,omitempty"` } type DataSourceFeatureEngineeringMaterializedFeatureTableTrigger struct{} diff --git a/bundle/internal/tf/schema/data_source_feature_engineering_materialized_features.go b/bundle/internal/tf/schema/data_source_feature_engineering_materialized_features.go index cf95ba9f289..dbedd725190 100644 --- a/bundle/internal/tf/schema/data_source_feature_engineering_materialized_features.go +++ b/bundle/internal/tf/schema/data_source_feature_engineering_materialized_features.go @@ -24,7 +24,8 @@ type DataSourceFeatureEngineeringMaterializedFeaturesMaterializedFeaturesProvide } type DataSourceFeatureEngineeringMaterializedFeaturesMaterializedFeaturesStreamingMode struct { - Mode string `json:"mode,omitempty"` + FreshnessTarget string `json:"freshness_target,omitempty"` + Mode string `json:"mode,omitempty"` } type DataSourceFeatureEngineeringMaterializedFeaturesMaterializedFeaturesTableTrigger struct{} diff --git a/bundle/internal/tf/schema/data_source_group.go b/bundle/internal/tf/schema/data_source_group.go index b15819d3824..1cb5f1c498f 100644 --- a/bundle/internal/tf/schema/data_source_group.go +++ b/bundle/internal/tf/schema/data_source_group.go @@ -20,6 +20,7 @@ type DataSourceGroup struct { InstanceProfiles []string `json:"instance_profiles,omitempty"` Members []string `json:"members,omitempty"` Recursive bool `json:"recursive,omitempty"` + Roles []string `json:"roles,omitempty"` ServicePrincipals []string `json:"service_principals,omitempty"` Users []string `json:"users,omitempty"` WorkspaceAccess bool `json:"workspace_access,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_mlflow_experiment.go b/bundle/internal/tf/schema/data_source_mlflow_experiment.go index 792600443d1..2b01a2f2557 100644 --- a/bundle/internal/tf/schema/data_source_mlflow_experiment.go +++ b/bundle/internal/tf/schema/data_source_mlflow_experiment.go @@ -11,6 +11,17 @@ type DataSourceMlflowExperimentTags struct { Value string `json:"value,omitempty"` } +type DataSourceMlflowExperimentTraceLocationUcTraceLocation struct { + Catalog string `json:"catalog"` + EffectiveTablePrefix string `json:"effective_table_prefix,omitempty"` + Schema string `json:"schema"` + TablePrefix string `json:"table_prefix,omitempty"` +} + +type DataSourceMlflowExperimentTraceLocation struct { + UcTraceLocation *DataSourceMlflowExperimentTraceLocationUcTraceLocation `json:"uc_trace_location,omitempty"` +} + type DataSourceMlflowExperiment struct { ArtifactLocation string `json:"artifact_location,omitempty"` CreationTime int `json:"creation_time,omitempty"` @@ -21,4 +32,5 @@ type DataSourceMlflowExperiment struct { Name string `json:"name,omitempty"` ProviderConfig *DataSourceMlflowExperimentProviderConfig `json:"provider_config,omitempty"` Tags []DataSourceMlflowExperimentTags `json:"tags,omitempty"` + TraceLocation *DataSourceMlflowExperimentTraceLocation `json:"trace_location,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_postgres_cdf_config.go b/bundle/internal/tf/schema/data_source_postgres_cdf_config.go new file mode 100644 index 00000000000..c83d566ebb1 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_postgres_cdf_config.go @@ -0,0 +1,17 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourcePostgresCdfConfigProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfConfig struct { + Catalog string `json:"catalog,omitempty"` + CdfConfigId string `json:"cdf_config_id,omitempty"` + CreateTime string `json:"create_time,omitempty"` + Name string `json:"name"` + PostgresSchema string `json:"postgres_schema,omitempty"` + ProviderConfig *DataSourcePostgresCdfConfigProviderConfig `json:"provider_config,omitempty"` + Schema string `json:"schema,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_postgres_cdf_configs.go b/bundle/internal/tf/schema/data_source_postgres_cdf_configs.go new file mode 100644 index 00000000000..3fb52aa0be8 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_postgres_cdf_configs.go @@ -0,0 +1,28 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourcePostgresCdfConfigsCdfConfigsProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfConfigsCdfConfigs struct { + Catalog string `json:"catalog,omitempty"` + CdfConfigId string `json:"cdf_config_id,omitempty"` + CreateTime string `json:"create_time,omitempty"` + Name string `json:"name"` + PostgresSchema string `json:"postgres_schema,omitempty"` + ProviderConfig *DataSourcePostgresCdfConfigsCdfConfigsProviderConfig `json:"provider_config,omitempty"` + Schema string `json:"schema,omitempty"` +} + +type DataSourcePostgresCdfConfigsProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfConfigs struct { + CdfConfigs []DataSourcePostgresCdfConfigsCdfConfigs `json:"cdf_configs,omitempty"` + PageSize int `json:"page_size,omitempty"` + Parent string `json:"parent"` + ProviderConfig *DataSourcePostgresCdfConfigsProviderConfig `json:"provider_config,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_postgres_cdf_status.go b/bundle/internal/tf/schema/data_source_postgres_cdf_status.go new file mode 100644 index 00000000000..22ec0338843 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_postgres_cdf_status.go @@ -0,0 +1,19 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourcePostgresCdfStatusProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfStatus struct { + CommittedLsn string `json:"committed_lsn,omitempty"` + CreateTime string `json:"create_time,omitempty"` + LastSyncTime string `json:"last_sync_time,omitempty"` + Name string `json:"name"` + PostgresTable string `json:"postgres_table,omitempty"` + ProviderConfig *DataSourcePostgresCdfStatusProviderConfig `json:"provider_config,omitempty"` + State string `json:"state,omitempty"` + StatusDetail string `json:"status_detail,omitempty"` + UcTable string `json:"uc_table,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_postgres_cdf_statuses.go b/bundle/internal/tf/schema/data_source_postgres_cdf_statuses.go new file mode 100644 index 00000000000..dc1c2a5cf69 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_postgres_cdf_statuses.go @@ -0,0 +1,30 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourcePostgresCdfStatusesCdfStatusesProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfStatusesCdfStatuses struct { + CommittedLsn string `json:"committed_lsn,omitempty"` + CreateTime string `json:"create_time,omitempty"` + LastSyncTime string `json:"last_sync_time,omitempty"` + Name string `json:"name"` + PostgresTable string `json:"postgres_table,omitempty"` + ProviderConfig *DataSourcePostgresCdfStatusesCdfStatusesProviderConfig `json:"provider_config,omitempty"` + State string `json:"state,omitempty"` + StatusDetail string `json:"status_detail,omitempty"` + UcTable string `json:"uc_table,omitempty"` +} + +type DataSourcePostgresCdfStatusesProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourcePostgresCdfStatuses struct { + CdfStatuses []DataSourcePostgresCdfStatusesCdfStatuses `json:"cdf_statuses,omitempty"` + PageSize int `json:"page_size,omitempty"` + Parent string `json:"parent"` + ProviderConfig *DataSourcePostgresCdfStatusesProviderConfig `json:"provider_config,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_postgres_database.go b/bundle/internal/tf/schema/data_source_postgres_database.go index c073e876bb5..8c1f8509eb2 100644 --- a/bundle/internal/tf/schema/data_source_postgres_database.go +++ b/bundle/internal/tf/schema/data_source_postgres_database.go @@ -8,7 +8,7 @@ type DataSourcePostgresDatabaseProviderConfig struct { type DataSourcePostgresDatabaseSpec struct { PostgresDatabase string `json:"postgres_database,omitempty"` - Role string `json:"role,omitempty"` + Role string `json:"role"` } type DataSourcePostgresDatabaseStatus struct { diff --git a/bundle/internal/tf/schema/data_source_postgres_databases.go b/bundle/internal/tf/schema/data_source_postgres_databases.go index 33bfa025870..dd511935ecb 100644 --- a/bundle/internal/tf/schema/data_source_postgres_databases.go +++ b/bundle/internal/tf/schema/data_source_postgres_databases.go @@ -8,7 +8,7 @@ type DataSourcePostgresDatabasesDatabasesProviderConfig struct { type DataSourcePostgresDatabasesDatabasesSpec struct { PostgresDatabase string `json:"postgres_database,omitempty"` - Role string `json:"role,omitempty"` + Role string `json:"role"` } type DataSourcePostgresDatabasesDatabasesStatus struct { diff --git a/bundle/internal/tf/schema/data_source_postgres_endpoint.go b/bundle/internal/tf/schema/data_source_postgres_endpoint.go index 0327ce656f4..640516a2545 100644 --- a/bundle/internal/tf/schema/data_source_postgres_endpoint.go +++ b/bundle/internal/tf/schema/data_source_postgres_endpoint.go @@ -34,8 +34,10 @@ type DataSourcePostgresEndpointStatusGroup struct { } type DataSourcePostgresEndpointStatusHosts struct { - Host string `json:"host,omitempty"` - ReadOnlyHost string `json:"read_only_host,omitempty"` + Host string `json:"host,omitempty"` + ReadOnlyHost string `json:"read_only_host,omitempty"` + ReadOnlyPooledHost string `json:"read_only_pooled_host,omitempty"` + ReadWritePooledHost string `json:"read_write_pooled_host,omitempty"` } type DataSourcePostgresEndpointStatusSettings struct { @@ -51,6 +53,7 @@ type DataSourcePostgresEndpointStatus struct { EndpointType string `json:"endpoint_type,omitempty"` Group *DataSourcePostgresEndpointStatusGroup `json:"group,omitempty"` Hosts *DataSourcePostgresEndpointStatusHosts `json:"hosts,omitempty"` + LastActiveTime string `json:"last_active_time,omitempty"` PendingState string `json:"pending_state,omitempty"` Settings *DataSourcePostgresEndpointStatusSettings `json:"settings,omitempty"` SuspendTimeoutDuration string `json:"suspend_timeout_duration,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_postgres_endpoints.go b/bundle/internal/tf/schema/data_source_postgres_endpoints.go index 75ad54f4b5c..274d475a08e 100644 --- a/bundle/internal/tf/schema/data_source_postgres_endpoints.go +++ b/bundle/internal/tf/schema/data_source_postgres_endpoints.go @@ -34,8 +34,10 @@ type DataSourcePostgresEndpointsEndpointsStatusGroup struct { } type DataSourcePostgresEndpointsEndpointsStatusHosts struct { - Host string `json:"host,omitempty"` - ReadOnlyHost string `json:"read_only_host,omitempty"` + Host string `json:"host,omitempty"` + ReadOnlyHost string `json:"read_only_host,omitempty"` + ReadOnlyPooledHost string `json:"read_only_pooled_host,omitempty"` + ReadWritePooledHost string `json:"read_write_pooled_host,omitempty"` } type DataSourcePostgresEndpointsEndpointsStatusSettings struct { @@ -51,6 +53,7 @@ type DataSourcePostgresEndpointsEndpointsStatus struct { EndpointType string `json:"endpoint_type,omitempty"` Group *DataSourcePostgresEndpointsEndpointsStatusGroup `json:"group,omitempty"` Hosts *DataSourcePostgresEndpointsEndpointsStatusHosts `json:"hosts,omitempty"` + LastActiveTime string `json:"last_active_time,omitempty"` PendingState string `json:"pending_state,omitempty"` Settings *DataSourcePostgresEndpointsEndpointsStatusSettings `json:"settings,omitempty"` SuspendTimeoutDuration string `json:"suspend_timeout_duration,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_postgres_synced_table.go b/bundle/internal/tf/schema/data_source_postgres_synced_table.go index 9f69a5fa449..2f901b4bb7e 100644 --- a/bundle/internal/tf/schema/data_source_postgres_synced_table.go +++ b/bundle/internal/tf/schema/data_source_postgres_synced_table.go @@ -6,6 +6,13 @@ type DataSourcePostgresSyncedTableProviderConfig struct { WorkspaceId string `json:"workspace_id,omitempty"` } +type DataSourcePostgresSyncedTableSpecExtraColumns struct { + ColumnName string `json:"column_name"` + ColumnType string `json:"column_type"` + Compute string `json:"compute,omitempty"` + Maintenance string `json:"maintenance,omitempty"` +} + type DataSourcePostgresSyncedTableSpecNewPipelineSpec struct { BudgetPolicyId string `json:"budget_policy_id,omitempty"` StorageCatalog string `json:"storage_catalog,omitempty"` @@ -23,6 +30,7 @@ type DataSourcePostgresSyncedTableSpec struct { Branch string `json:"branch,omitempty"` CreateDatabaseObjectsIfMissing bool `json:"create_database_objects_if_missing,omitempty"` ExistingPipelineId string `json:"existing_pipeline_id,omitempty"` + ExtraColumns []DataSourcePostgresSyncedTableSpecExtraColumns `json:"extra_columns,omitempty"` NewPipelineSpec *DataSourcePostgresSyncedTableSpecNewPipelineSpec `json:"new_pipeline_spec,omitempty"` PostgresDatabase string `json:"postgres_database,omitempty"` PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_recipients.go b/bundle/internal/tf/schema/data_source_recipients.go new file mode 100644 index 00000000000..c281019dbad --- /dev/null +++ b/bundle/internal/tf/schema/data_source_recipients.go @@ -0,0 +1,12 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceRecipientsProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type DataSourceRecipients struct { + ProviderConfig *DataSourceRecipientsProviderConfig `json:"provider_config,omitempty"` + Recipients []string `json:"recipients,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_secret_uc.go b/bundle/internal/tf/schema/data_source_secret_uc.go index b9a7a57fe6a..abbafcf80bd 100644 --- a/bundle/internal/tf/schema/data_source_secret_uc.go +++ b/bundle/internal/tf/schema/data_source_secret_uc.go @@ -7,22 +7,20 @@ type DataSourceSecretUcProviderConfig struct { } type DataSourceSecretUc struct { - BrowseOnly bool `json:"browse_only,omitempty"` - CatalogName string `json:"catalog_name,omitempty"` - Comment string `json:"comment,omitempty"` - CreateTime string `json:"create_time,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - EffectiveOwner string `json:"effective_owner,omitempty"` - EffectiveValue string `json:"effective_value,omitempty"` - ExpireTime string `json:"expire_time,omitempty"` - ExternalSecretId string `json:"external_secret_id,omitempty"` - FullName string `json:"full_name"` - MetastoreId string `json:"metastore_id,omitempty"` - Name string `json:"name,omitempty"` - Owner string `json:"owner,omitempty"` - ProviderConfig *DataSourceSecretUcProviderConfig `json:"provider_config,omitempty"` - SchemaName string `json:"schema_name,omitempty"` - UpdateTime string `json:"update_time,omitempty"` - UpdatedBy string `json:"updated_by,omitempty"` - Value string `json:"value,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreateTime string `json:"create_time,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EffectiveOwner string `json:"effective_owner,omitempty"` + EffectiveValue string `json:"effective_value,omitempty"` + ExpireTime string `json:"expire_time,omitempty"` + FullName string `json:"full_name"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + ProviderConfig *DataSourceSecretUcProviderConfig `json:"provider_config,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Value string `json:"value,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_secret_ucs.go b/bundle/internal/tf/schema/data_source_secret_ucs.go index e9701e92706..930620e7f9d 100644 --- a/bundle/internal/tf/schema/data_source_secret_ucs.go +++ b/bundle/internal/tf/schema/data_source_secret_ucs.go @@ -11,29 +11,26 @@ type DataSourceSecretUcsSecretsProviderConfig struct { } type DataSourceSecretUcsSecrets struct { - BrowseOnly bool `json:"browse_only,omitempty"` - CatalogName string `json:"catalog_name,omitempty"` - Comment string `json:"comment,omitempty"` - CreateTime string `json:"create_time,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - EffectiveOwner string `json:"effective_owner,omitempty"` - EffectiveValue string `json:"effective_value,omitempty"` - ExpireTime string `json:"expire_time,omitempty"` - ExternalSecretId string `json:"external_secret_id,omitempty"` - FullName string `json:"full_name"` - MetastoreId string `json:"metastore_id,omitempty"` - Name string `json:"name,omitempty"` - Owner string `json:"owner,omitempty"` - ProviderConfig *DataSourceSecretUcsSecretsProviderConfig `json:"provider_config,omitempty"` - SchemaName string `json:"schema_name,omitempty"` - UpdateTime string `json:"update_time,omitempty"` - UpdatedBy string `json:"updated_by,omitempty"` - Value string `json:"value,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreateTime string `json:"create_time,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EffectiveOwner string `json:"effective_owner,omitempty"` + EffectiveValue string `json:"effective_value,omitempty"` + ExpireTime string `json:"expire_time,omitempty"` + FullName string `json:"full_name"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + ProviderConfig *DataSourceSecretUcsSecretsProviderConfig `json:"provider_config,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Value string `json:"value,omitempty"` } type DataSourceSecretUcs struct { CatalogName string `json:"catalog_name,omitempty"` - IncludeBrowse bool `json:"include_browse,omitempty"` PageSize int `json:"page_size,omitempty"` ProviderConfig *DataSourceSecretUcsProviderConfig `json:"provider_config,omitempty"` SchemaName string `json:"schema_name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_serving_endpoints.go b/bundle/internal/tf/schema/data_source_serving_endpoints.go index 7347ee3b189..f94fc842d5f 100644 --- a/bundle/internal/tf/schema/data_source_serving_endpoints.go +++ b/bundle/internal/tf/schema/data_source_serving_endpoints.go @@ -191,8 +191,17 @@ type DataSourceServingEndpointsEndpointsTelemetryConfigInferenceTableConfig stru SamplingFraction int `json:"sampling_fraction,omitempty"` } +type DataSourceServingEndpointsEndpointsTelemetryConfigTableNames struct { + AnnotationsTable string `json:"annotations_table,omitempty"` + LogsTable string `json:"logs_table,omitempty"` + MetricsTable string `json:"metrics_table,omitempty"` + TracesTable string `json:"traces_table,omitempty"` +} + type DataSourceServingEndpointsEndpointsTelemetryConfig struct { InferenceTableConfig []DataSourceServingEndpointsEndpointsTelemetryConfigInferenceTableConfig `json:"inference_table_config,omitempty"` + TableNames []DataSourceServingEndpointsEndpointsTelemetryConfigTableNames `json:"table_names,omitempty"` + TelemetryProfileId string `json:"telemetry_profile_id,omitempty"` } type DataSourceServingEndpointsEndpoints struct { diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 9a8860591ce..16bd84c034c 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -105,6 +105,10 @@ type DataSources struct { PostgresBranch map[string]any `json:"databricks_postgres_branch,omitempty"` PostgresBranches map[string]any `json:"databricks_postgres_branches,omitempty"` PostgresCatalog map[string]any `json:"databricks_postgres_catalog,omitempty"` + PostgresCdfConfig map[string]any `json:"databricks_postgres_cdf_config,omitempty"` + PostgresCdfConfigs map[string]any `json:"databricks_postgres_cdf_configs,omitempty"` + PostgresCdfStatus map[string]any `json:"databricks_postgres_cdf_status,omitempty"` + PostgresCdfStatuses map[string]any `json:"databricks_postgres_cdf_statuses,omitempty"` PostgresDataApi map[string]any `json:"databricks_postgres_data_api,omitempty"` PostgresDatabase map[string]any `json:"databricks_postgres_database,omitempty"` PostgresDatabases map[string]any `json:"databricks_postgres_databases,omitempty"` @@ -117,6 +121,7 @@ type DataSources struct { PostgresSyncedTable map[string]any `json:"databricks_postgres_synced_table,omitempty"` QualityMonitorV2 map[string]any `json:"databricks_quality_monitor_v2,omitempty"` QualityMonitorsV2 map[string]any `json:"databricks_quality_monitors_v2,omitempty"` + Recipients map[string]any `json:"databricks_recipients,omitempty"` RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` RegisteredModelVersions map[string]any `json:"databricks_registered_model_versions,omitempty"` RfaAccessRequestDestinations map[string]any `json:"databricks_rfa_access_request_destinations,omitempty"` @@ -262,6 +267,10 @@ func NewDataSources() *DataSources { PostgresBranch: make(map[string]any), PostgresBranches: make(map[string]any), PostgresCatalog: make(map[string]any), + PostgresCdfConfig: make(map[string]any), + PostgresCdfConfigs: make(map[string]any), + PostgresCdfStatus: make(map[string]any), + PostgresCdfStatuses: make(map[string]any), PostgresDataApi: make(map[string]any), PostgresDatabase: make(map[string]any), PostgresDatabases: make(map[string]any), @@ -274,6 +283,7 @@ func NewDataSources() *DataSources { PostgresSyncedTable: make(map[string]any), QualityMonitorV2: make(map[string]any), QualityMonitorsV2: make(map[string]any), + Recipients: make(map[string]any), RegisteredModel: make(map[string]any), RegisteredModelVersions: make(map[string]any), RfaAccessRequestDestinations: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_alert_v2.go b/bundle/internal/tf/schema/resource_alert_v2.go index b9cdec84717..e44b8ba27ca 100644 --- a/bundle/internal/tf/schema/resource_alert_v2.go +++ b/bundle/internal/tf/schema/resource_alert_v2.go @@ -53,6 +53,12 @@ type ResourceAlertV2Evaluation struct { Threshold *ResourceAlertV2EvaluationThreshold `json:"threshold,omitempty"` } +type ResourceAlertV2Parameters struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Value string `json:"value,omitempty"` +} + type ResourceAlertV2ProviderConfig struct { WorkspaceId string `json:"workspace_id,omitempty"` } @@ -78,6 +84,7 @@ type ResourceAlertV2 struct { Id string `json:"id,omitempty"` LifecycleState string `json:"lifecycle_state,omitempty"` OwnerUserName string `json:"owner_user_name,omitempty"` + Parameters []ResourceAlertV2Parameters `json:"parameters,omitempty"` ParentPath string `json:"parent_path,omitempty"` ProviderConfig *ResourceAlertV2ProviderConfig `json:"provider_config,omitempty"` PurgeOnDelete bool `json:"purge_on_delete,omitempty"` diff --git a/bundle/internal/tf/schema/resource_all.go b/bundle/internal/tf/schema/resource_all.go index d2cebc72831..99ad3830027 100644 --- a/bundle/internal/tf/schema/resource_all.go +++ b/bundle/internal/tf/schema/resource_all.go @@ -113,6 +113,7 @@ type AllResources struct { PolicyInfo ResourcePolicyInfo `json:"databricks_policy_info,omitempty"` PostgresBranch ResourcePostgresBranch `json:"databricks_postgres_branch,omitempty"` PostgresCatalog ResourcePostgresCatalog `json:"databricks_postgres_catalog,omitempty"` + PostgresCdfConfig ResourcePostgresCdfConfig `json:"databricks_postgres_cdf_config,omitempty"` PostgresDataApi ResourcePostgresDataApi `json:"databricks_postgres_data_api,omitempty"` PostgresDatabase ResourcePostgresDatabase `json:"databricks_postgres_database,omitempty"` PostgresEndpoint ResourcePostgresEndpoint `json:"databricks_postgres_endpoint,omitempty"` diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go index 08a4e1c84c8..2f38f662f92 100644 --- a/bundle/internal/tf/schema/resource_cluster.go +++ b/bundle/internal/tf/schema/resource_cluster.go @@ -181,49 +181,51 @@ type ResourceClusterWorkloadType struct { } type ResourceCluster struct { - ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ClusterName string `json:"cluster_name,omitempty"` - CustomTags map[string]string `json:"custom_tags,omitempty"` - DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags,omitempty"` - DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` - DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` - EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` - EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` - Id string `json:"id,omitempty"` - IdempotencyToken string `json:"idempotency_token,omitempty"` - InstancePoolId string `json:"instance_pool_id,omitempty"` - IsPinned bool `json:"is_pinned,omitempty"` - IsSingleNode bool `json:"is_single_node,omitempty"` - Kind string `json:"kind,omitempty"` - NoWait bool `json:"no_wait,omitempty"` - NodeTypeId string `json:"node_type_id,omitempty"` - NumWorkers int `json:"num_workers,omitempty"` - PolicyId string `json:"policy_id,omitempty"` - RemoteDiskThroughput int `json:"remote_disk_throughput,omitempty"` - RuntimeEngine string `json:"runtime_engine,omitempty"` - SingleUserName string `json:"single_user_name,omitempty"` - SparkConf map[string]string `json:"spark_conf,omitempty"` - SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` - SshPublicKeys []string `json:"ssh_public_keys,omitempty"` - State string `json:"state,omitempty"` - TotalInitialRemoteDiskSize int `json:"total_initial_remote_disk_size,omitempty"` - Url string `json:"url,omitempty"` - UseMlRuntime bool `json:"use_ml_runtime,omitempty"` - Autoscale *ResourceClusterAutoscale `json:"autoscale,omitempty"` - AwsAttributes *ResourceClusterAwsAttributes `json:"aws_attributes,omitempty"` - AzureAttributes *ResourceClusterAzureAttributes `json:"azure_attributes,omitempty"` - ClusterLogConf *ResourceClusterClusterLogConf `json:"cluster_log_conf,omitempty"` - ClusterMountInfo []ResourceClusterClusterMountInfo `json:"cluster_mount_info,omitempty"` - DockerImage *ResourceClusterDockerImage `json:"docker_image,omitempty"` - DriverNodeTypeFlexibility *ResourceClusterDriverNodeTypeFlexibility `json:"driver_node_type_flexibility,omitempty"` - GcpAttributes *ResourceClusterGcpAttributes `json:"gcp_attributes,omitempty"` - InitScripts []ResourceClusterInitScripts `json:"init_scripts,omitempty"` - Library []ResourceClusterLibrary `json:"library,omitempty"` - ProviderConfig *ResourceClusterProviderConfig `json:"provider_config,omitempty"` - WorkerNodeTypeFlexibility *ResourceClusterWorkerNodeTypeFlexibility `json:"worker_node_type_flexibility,omitempty"` - WorkloadType *ResourceClusterWorkloadType `json:"workload_type,omitempty"` + ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` + ClearCloudAttributesOnRemove bool `json:"clear_cloud_attributes_on_remove,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + DataSecurityMode string `json:"data_security_mode,omitempty"` + DefaultTags map[string]string `json:"default_tags,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` + DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` + DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` + Id string `json:"id,omitempty"` + IdempotencyToken string `json:"idempotency_token,omitempty"` + InstancePoolId string `json:"instance_pool_id,omitempty"` + IsPinned bool `json:"is_pinned,omitempty"` + IsSingleNode bool `json:"is_single_node,omitempty"` + Kind string `json:"kind,omitempty"` + NoWait bool `json:"no_wait,omitempty"` + NodeTypeId string `json:"node_type_id,omitempty"` + NumWorkers int `json:"num_workers,omitempty"` + PolicyId string `json:"policy_id,omitempty"` + RemoteDiskThroughput int `json:"remote_disk_throughput,omitempty"` + RuntimeEngine string `json:"runtime_engine,omitempty"` + SingleUserName string `json:"single_user_name,omitempty"` + SparkConf map[string]string `json:"spark_conf,omitempty"` + SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` + SparkVersion string `json:"spark_version"` + SshPublicKeys []string `json:"ssh_public_keys,omitempty"` + State string `json:"state,omitempty"` + TotalInitialRemoteDiskSize int `json:"total_initial_remote_disk_size,omitempty"` + Url string `json:"url,omitempty"` + UseMlRuntime bool `json:"use_ml_runtime,omitempty"` + Autoscale *ResourceClusterAutoscale `json:"autoscale,omitempty"` + AwsAttributes *ResourceClusterAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *ResourceClusterAzureAttributes `json:"azure_attributes,omitempty"` + ClusterLogConf *ResourceClusterClusterLogConf `json:"cluster_log_conf,omitempty"` + ClusterMountInfo []ResourceClusterClusterMountInfo `json:"cluster_mount_info,omitempty"` + DockerImage *ResourceClusterDockerImage `json:"docker_image,omitempty"` + DriverNodeTypeFlexibility *ResourceClusterDriverNodeTypeFlexibility `json:"driver_node_type_flexibility,omitempty"` + GcpAttributes *ResourceClusterGcpAttributes `json:"gcp_attributes,omitempty"` + InitScripts []ResourceClusterInitScripts `json:"init_scripts,omitempty"` + Library []ResourceClusterLibrary `json:"library,omitempty"` + ProviderConfig *ResourceClusterProviderConfig `json:"provider_config,omitempty"` + WorkerNodeTypeFlexibility *ResourceClusterWorkerNodeTypeFlexibility `json:"worker_node_type_flexibility,omitempty"` + WorkloadType *ResourceClusterWorkloadType `json:"workload_type,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_data_classification_catalog_config.go b/bundle/internal/tf/schema/resource_data_classification_catalog_config.go index 4fedc74410a..873f785086e 100644 --- a/bundle/internal/tf/schema/resource_data_classification_catalog_config.go +++ b/bundle/internal/tf/schema/resource_data_classification_catalog_config.go @@ -7,6 +7,10 @@ type ResourceDataClassificationCatalogConfigAutoTagConfigs struct { ClassificationTag string `json:"classification_tag"` } +type ResourceDataClassificationCatalogConfigExcludedSchemas struct { + Names []string `json:"names"` +} + type ResourceDataClassificationCatalogConfigIncludedSchemas struct { Names []string `json:"names"` } @@ -17,6 +21,7 @@ type ResourceDataClassificationCatalogConfigProviderConfig struct { type ResourceDataClassificationCatalogConfig struct { AutoTagConfigs []ResourceDataClassificationCatalogConfigAutoTagConfigs `json:"auto_tag_configs,omitempty"` + ExcludedSchemas *ResourceDataClassificationCatalogConfigExcludedSchemas `json:"excluded_schemas,omitempty"` IncludedSchemas *ResourceDataClassificationCatalogConfigIncludedSchemas `json:"included_schemas,omitempty"` Name string `json:"name,omitempty"` Parent string `json:"parent"` diff --git a/bundle/internal/tf/schema/resource_disaster_recovery_stable_url.go b/bundle/internal/tf/schema/resource_disaster_recovery_stable_url.go index 79d325e5e57..2be41e085de 100644 --- a/bundle/internal/tf/schema/resource_disaster_recovery_stable_url.go +++ b/bundle/internal/tf/schema/resource_disaster_recovery_stable_url.go @@ -3,10 +3,12 @@ package schema type ResourceDisasterRecoveryStableUrl struct { - FailoverGroupName string `json:"failover_group_name,omitempty"` - InitialWorkspaceId string `json:"initial_workspace_id"` - Name string `json:"name,omitempty"` - Parent string `json:"parent"` - StableUrlId string `json:"stable_url_id"` - Url string `json:"url,omitempty"` + EffectiveWorkspaceId string `json:"effective_workspace_id,omitempty"` + FailoverGroupName string `json:"failover_group_name,omitempty"` + InitialWorkspaceId string `json:"initial_workspace_id"` + Name string `json:"name,omitempty"` + Parent string `json:"parent"` + StableUrlId string `json:"stable_url_id"` + StableWorkspaceId string `json:"stable_workspace_id,omitempty"` + Url string `json:"url,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_endpoint.go b/bundle/internal/tf/schema/resource_endpoint.go index 0b010ce55b0..b1be5023943 100644 --- a/bundle/internal/tf/schema/resource_endpoint.go +++ b/bundle/internal/tf/schema/resource_endpoint.go @@ -2,6 +2,12 @@ package schema +type ResourceEndpointAwsVpcEndpointInfo struct { + AwsAccountId string `json:"aws_account_id,omitempty"` + AwsEndpointServiceId string `json:"aws_endpoint_service_id,omitempty"` + AwsVpcEndpointId string `json:"aws_vpc_endpoint_id"` +} + type ResourceEndpointAzurePrivateEndpointInfo struct { PrivateEndpointName string `json:"private_endpoint_name"` PrivateEndpointResourceGuid string `json:"private_endpoint_resource_guid"` @@ -9,12 +15,22 @@ type ResourceEndpointAzurePrivateEndpointInfo struct { PrivateLinkServiceId string `json:"private_link_service_id,omitempty"` } +type ResourceEndpointGcpPscEndpointInfo struct { + EndpointRegion string `json:"endpoint_region"` + ProjectId string `json:"project_id"` + PscConnectionId string `json:"psc_connection_id,omitempty"` + PscEndpoint string `json:"psc_endpoint"` + ServiceAttachmentId string `json:"service_attachment_id,omitempty"` +} + type ResourceEndpoint struct { AccountId string `json:"account_id,omitempty"` + AwsVpcEndpointInfo *ResourceEndpointAwsVpcEndpointInfo `json:"aws_vpc_endpoint_info,omitempty"` AzurePrivateEndpointInfo *ResourceEndpointAzurePrivateEndpointInfo `json:"azure_private_endpoint_info,omitempty"` CreateTime string `json:"create_time,omitempty"` DisplayName string `json:"display_name"` EndpointId string `json:"endpoint_id,omitempty"` + GcpPscEndpointInfo *ResourceEndpointGcpPscEndpointInfo `json:"gcp_psc_endpoint_info,omitempty"` Name string `json:"name,omitempty"` Parent string `json:"parent"` Region string `json:"region"` diff --git a/bundle/internal/tf/schema/resource_feature_engineering_feature.go b/bundle/internal/tf/schema/resource_feature_engineering_feature.go index 4ff3bf5820c..d4d7f63572f 100644 --- a/bundle/internal/tf/schema/resource_feature_engineering_feature.go +++ b/bundle/internal/tf/schema/resource_feature_engineering_feature.go @@ -29,10 +29,30 @@ type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirst struct { Input string `json:"input"` } +type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstN struct { + Input string `json:"input"` + N int `json:"n"` +} + type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLast struct { Input string `json:"input"` } +type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLastDistinct struct { + Input string `json:"input"` + N int `json:"n"` +} + +type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLastN struct { + Input string `json:"input"` + N int `json:"n"` +} + type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionMax struct { Input string `json:"input"` } @@ -60,12 +80,17 @@ type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowConti type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTumbling struct { @@ -75,6 +100,7 @@ type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTumbl type ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindow struct { Continuous *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowContinuous `json:"continuous,omitempty"` Rolling *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowSliding `json:"sliding,omitempty"` Tumbling *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionTimeWindowTumbling `json:"tumbling,omitempty"` } @@ -93,7 +119,11 @@ type ResourceFeatureEngineeringFeatureFunctionAggregationFunction struct { Avg *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionAvg `json:"avg,omitempty"` CountFunction *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionCountFunction `json:"count_function,omitempty"` First *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirst `json:"first,omitempty"` + FirstDistinct *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstDistinct `json:"first_distinct,omitempty"` + FirstN *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionFirstN `json:"first_n,omitempty"` Last *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLast `json:"last,omitempty"` + LastDistinct *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLastDistinct `json:"last_distinct,omitempty"` + LastN *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionLastN `json:"last_n,omitempty"` Max *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionMax `json:"max,omitempty"` Min *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionMin `json:"min,omitempty"` StddevPop *ResourceFeatureEngineeringFeatureFunctionAggregationFunctionStddevPop `json:"stddev_pop,omitempty"` @@ -172,8 +202,10 @@ type ResourceFeatureEngineeringFeatureSourceRequestSource struct { } type ResourceFeatureEngineeringFeatureSourceStreamSource struct { - FilterCondition string `json:"filter_condition,omitempty"` - FullName string `json:"full_name"` + DataframeSchema string `json:"dataframe_schema,omitempty"` + FilterCondition string `json:"filter_condition,omitempty"` + FullName string `json:"full_name"` + TransformationSql string `json:"transformation_sql,omitempty"` } type ResourceFeatureEngineeringFeatureSource struct { @@ -190,12 +222,17 @@ type ResourceFeatureEngineeringFeatureTimeWindowContinuous struct { type ResourceFeatureEngineeringFeatureTimeWindowRolling struct { Delay string `json:"delay,omitempty"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type ResourceFeatureEngineeringFeatureTimeWindowSawtooth struct { + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` } type ResourceFeatureEngineeringFeatureTimeWindowSliding struct { SlideDuration string `json:"slide_duration"` - WindowDuration string `json:"window_duration"` + WindowDuration string `json:"window_duration,omitempty"` } type ResourceFeatureEngineeringFeatureTimeWindowTumbling struct { @@ -205,6 +242,7 @@ type ResourceFeatureEngineeringFeatureTimeWindowTumbling struct { type ResourceFeatureEngineeringFeatureTimeWindow struct { Continuous *ResourceFeatureEngineeringFeatureTimeWindowContinuous `json:"continuous,omitempty"` Rolling *ResourceFeatureEngineeringFeatureTimeWindowRolling `json:"rolling,omitempty"` + Sawtooth *ResourceFeatureEngineeringFeatureTimeWindowSawtooth `json:"sawtooth,omitempty"` Sliding *ResourceFeatureEngineeringFeatureTimeWindowSliding `json:"sliding,omitempty"` Tumbling *ResourceFeatureEngineeringFeatureTimeWindowTumbling `json:"tumbling,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_feature_engineering_kafka_config.go b/bundle/internal/tf/schema/resource_feature_engineering_kafka_config.go index 8b3f526bdae..bf5c49c6f17 100644 --- a/bundle/internal/tf/schema/resource_feature_engineering_kafka_config.go +++ b/bundle/internal/tf/schema/resource_feature_engineering_kafka_config.go @@ -72,8 +72,15 @@ type ResourceFeatureEngineeringKafkaConfigIngestionConfig struct { IngestionPipelineId string `json:"ingestion_pipeline_id,omitempty"` } +type ResourceFeatureEngineeringKafkaConfigKeySchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type ResourceFeatureEngineeringKafkaConfigKeySchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *ResourceFeatureEngineeringKafkaConfigKeySchemaProtoSchema `json:"proto_schema,omitempty"` } type ResourceFeatureEngineeringKafkaConfigProviderConfig struct { @@ -86,8 +93,15 @@ type ResourceFeatureEngineeringKafkaConfigSubscriptionMode struct { SubscribePattern string `json:"subscribe_pattern,omitempty"` } +type ResourceFeatureEngineeringKafkaConfigValueSchemaProtoSchema struct { + MessageName string `json:"message_name"` + SchemaText string `json:"schema_text"` +} + type ResourceFeatureEngineeringKafkaConfigValueSchema struct { - JsonSchema string `json:"json_schema,omitempty"` + AvroSchema string `json:"avro_schema,omitempty"` + JsonSchema string `json:"json_schema,omitempty"` + ProtoSchema *ResourceFeatureEngineeringKafkaConfigValueSchemaProtoSchema `json:"proto_schema,omitempty"` } type ResourceFeatureEngineeringKafkaConfig struct { diff --git a/bundle/internal/tf/schema/resource_feature_engineering_materialized_feature.go b/bundle/internal/tf/schema/resource_feature_engineering_materialized_feature.go index 4a753ec41ec..2111e47a5c2 100644 --- a/bundle/internal/tf/schema/resource_feature_engineering_materialized_feature.go +++ b/bundle/internal/tf/schema/resource_feature_engineering_materialized_feature.go @@ -24,7 +24,8 @@ type ResourceFeatureEngineeringMaterializedFeatureProviderConfig struct { } type ResourceFeatureEngineeringMaterializedFeatureStreamingMode struct { - Mode string `json:"mode,omitempty"` + FreshnessTarget string `json:"freshness_target,omitempty"` + Mode string `json:"mode,omitempty"` } type ResourceFeatureEngineeringMaterializedFeatureTableTrigger struct{} diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index b1627f02b4d..3a84cf10987 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -271,6 +271,7 @@ type ResourceJobJobClusterNewCluster struct { ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` @@ -307,8 +308,9 @@ type ResourceJobJobClusterNewCluster struct { } type ResourceJobJobCluster struct { - JobClusterKey string `json:"job_cluster_key"` - NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"` + JobClusterKey string `json:"job_cluster_key"` + ServerlessComputeId string `json:"serverless_compute_id,omitempty"` + NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"` } type ResourceJobLibraryCran struct { @@ -531,6 +533,7 @@ type ResourceJobNewCluster struct { ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` @@ -614,10 +617,17 @@ type ResourceJobRunJobTask struct { JobParameters map[string]string `json:"job_parameters,omitempty"` } +type ResourceJobScheduleSqlCondition struct { + SqlQueryId string `json:"sql_query_id"` + TriggerMode string `json:"trigger_mode,omitempty"` + WarehouseId string `json:"warehouse_id"` +} + type ResourceJobSchedule struct { - PauseStatus string `json:"pause_status,omitempty"` - QuartzCronExpression string `json:"quartz_cron_expression"` - TimezoneId string `json:"timezone_id"` + PauseStatus string `json:"pause_status,omitempty"` + QuartzCronExpression string `json:"quartz_cron_expression"` + TimezoneId string `json:"timezone_id"` + SqlCondition *ResourceJobScheduleSqlCondition `json:"sql_condition,omitempty"` } type ResourceJobSparkJarTask struct { @@ -1083,6 +1093,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct { ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` @@ -1584,6 +1595,7 @@ type ResourceJobTaskNewCluster struct { ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` + DependencyMode string `json:"dependency_mode,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` @@ -1851,6 +1863,12 @@ type ResourceJobTriggerPeriodic struct { Unit string `json:"unit"` } +type ResourceJobTriggerSqlCondition struct { + SqlQueryId string `json:"sql_query_id"` + TriggerMode string `json:"trigger_mode,omitempty"` + WarehouseId string `json:"warehouse_id"` +} + type ResourceJobTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1859,11 +1877,12 @@ type ResourceJobTriggerTableUpdate struct { } type ResourceJobTrigger struct { - PauseStatus string `json:"pause_status,omitempty"` - FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"` - Model *ResourceJobTriggerModel `json:"model,omitempty"` - Periodic *ResourceJobTriggerPeriodic `json:"periodic,omitempty"` - TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"` + PauseStatus string `json:"pause_status,omitempty"` + FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"` + Model *ResourceJobTriggerModel `json:"model,omitempty"` + Periodic *ResourceJobTriggerPeriodic `json:"periodic,omitempty"` + SqlCondition *ResourceJobTriggerSqlCondition `json:"sql_condition,omitempty"` + TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"` } type ResourceJobWebhookNotificationsOnDurationWarningThresholdExceeded struct { @@ -1907,6 +1926,7 @@ type ResourceJob struct { MaxRetries int `json:"max_retries,omitempty"` MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` Name string `json:"name,omitempty"` + ParentPath string `json:"parent_path,omitempty"` PerformanceTarget string `json:"performance_target,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` Tags map[string]string `json:"tags,omitempty"` diff --git a/bundle/internal/tf/schema/resource_mlflow_experiment.go b/bundle/internal/tf/schema/resource_mlflow_experiment.go index 26ca94ba632..972496bc15e 100644 --- a/bundle/internal/tf/schema/resource_mlflow_experiment.go +++ b/bundle/internal/tf/schema/resource_mlflow_experiment.go @@ -11,6 +11,17 @@ type ResourceMlflowExperimentTags struct { Value string `json:"value,omitempty"` } +type ResourceMlflowExperimentTraceLocationUcTraceLocation struct { + Catalog string `json:"catalog"` + EffectiveTablePrefix string `json:"effective_table_prefix,omitempty"` + Schema string `json:"schema"` + TablePrefix string `json:"table_prefix,omitempty"` +} + +type ResourceMlflowExperimentTraceLocation struct { + UcTraceLocation *ResourceMlflowExperimentTraceLocationUcTraceLocation `json:"uc_trace_location,omitempty"` +} + type ResourceMlflowExperiment struct { ArtifactLocation string `json:"artifact_location,omitempty"` CreationTime int `json:"creation_time,omitempty"` @@ -22,4 +33,5 @@ type ResourceMlflowExperiment struct { Name string `json:"name"` ProviderConfig *ResourceMlflowExperimentProviderConfig `json:"provider_config,omitempty"` Tags []ResourceMlflowExperimentTags `json:"tags,omitempty"` + TraceLocation *ResourceMlflowExperimentTraceLocation `json:"trace_location,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 3bc6ffebf94..4e892225e6b 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -234,8 +234,17 @@ type ResourceModelServingTelemetryConfigInferenceTableConfig struct { SamplingFraction int `json:"sampling_fraction,omitempty"` } +type ResourceModelServingTelemetryConfigTableNames struct { + AnnotationsTable string `json:"annotations_table,omitempty"` + LogsTable string `json:"logs_table,omitempty"` + MetricsTable string `json:"metrics_table,omitempty"` + TracesTable string `json:"traces_table,omitempty"` +} + type ResourceModelServingTelemetryConfig struct { + TelemetryProfileId string `json:"telemetry_profile_id,omitempty"` InferenceTableConfig *ResourceModelServingTelemetryConfigInferenceTableConfig `json:"inference_table_config,omitempty"` + TableNames *ResourceModelServingTelemetryConfigTableNames `json:"table_names,omitempty"` } type ResourceModelServing struct { diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 8d279e91902..ee4ff04fda0 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -213,6 +213,7 @@ type ResourcePipelineIngestionDefinitionObjectsReportTableConfiguration struct { SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` SequenceBy []string `json:"sequence_by,omitempty"` + SourceMetadataColumn string `json:"source_metadata_column,omitempty"` TableProperties map[string]string `json:"table_properties,omitempty"` AutoFullRefreshPolicy *ResourcePipelineIngestionDefinitionObjectsReportTableConfigurationAutoFullRefreshPolicy `json:"auto_full_refresh_policy,omitempty"` QueryBasedConnectorConfig *ResourcePipelineIngestionDefinitionObjectsReportTableConfigurationQueryBasedConnectorConfig `json:"query_based_connector_config,omitempty"` @@ -257,10 +258,18 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsGdriveOptio FileIngestionOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsGdriveOptionsFileIngestionOptions `json:"file_ingestion_options,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsGoogleAdsOptionsCustomReportOptions struct { + Metrics []string `json:"metrics,omitempty"` + Resource string `json:"resource"` + ResourceFields []string `json:"resource_fields,omitempty"` + Segments []string `json:"segments,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsGoogleAdsOptions struct { - LookbackWindowDays int `json:"lookback_window_days,omitempty"` - ManagerAccountId string `json:"manager_account_id"` - SyncStartDate string `json:"sync_start_date,omitempty"` + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + ManagerAccountId string `json:"manager_account_id"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsGoogleAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsJiraOptions struct { @@ -303,15 +312,25 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsKafkaOption ValueTransformer *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsKafkaOptionsValueTransformer `json:"value_transformer,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsMetaAdsOptionsCustomReportOptions struct { + ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` + ActionBreakdowns []string `json:"action_breakdowns,omitempty"` + ActionReportTime string `json:"action_report_time,omitempty"` + Breakdowns []string `json:"breakdowns,omitempty"` + Level string `json:"level,omitempty"` + TimeIncrement string `json:"time_increment,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsMetaAdsOptions struct { - ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` - ActionBreakdowns []string `json:"action_breakdowns,omitempty"` - ActionReportTime string `json:"action_report_time,omitempty"` - Breakdowns []string `json:"breakdowns,omitempty"` - CustomInsightsLookbackWindow int `json:"custom_insights_lookback_window,omitempty"` - Level string `json:"level,omitempty"` - StartDate string `json:"start_date,omitempty"` - TimeIncrement string `json:"time_increment,omitempty"` + ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` + ActionBreakdowns []string `json:"action_breakdowns,omitempty"` + ActionReportTime string `json:"action_report_time,omitempty"` + Breakdowns []string `json:"breakdowns,omitempty"` + CustomInsightsLookbackWindow int `json:"custom_insights_lookback_window,omitempty"` + Level string `json:"level,omitempty"` + StartDate string `json:"start_date,omitempty"` + TimeIncrement string `json:"time_increment,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsMetaAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsOutlookOptions struct { @@ -327,6 +346,17 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsOutlookOpti SubjectFilter []string `json:"subject_filter,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsRedditAdsOptionsCustomReportOptions struct { + Breakdowns []string `json:"breakdowns,omitempty"` + Fields []string `json:"fields,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsRedditAdsOptions struct { + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsRedditAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsSharepointOptionsFileIngestionOptionsFileFilters struct { ModifiedAfter string `json:"modified_after,omitempty"` ModifiedBefore string `json:"modified_before,omitempty"` @@ -357,14 +387,23 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsSmartsheetO EnforceSchema bool `json:"enforce_schema,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsTiktokAdsOptionsCustomReportOptions struct { + DataLevel string `json:"data_level,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Metrics []string `json:"metrics,omitempty"` + QueryLifetime bool `json:"query_lifetime,omitempty"` + ReportType string `json:"report_type,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsTiktokAdsOptions struct { - DataLevel string `json:"data_level,omitempty"` - Dimensions []string `json:"dimensions,omitempty"` - LookbackWindowDays int `json:"lookback_window_days,omitempty"` - Metrics []string `json:"metrics,omitempty"` - QueryLifetime bool `json:"query_lifetime,omitempty"` - ReportType string `json:"report_type,omitempty"` - SyncStartDate string `json:"sync_start_date,omitempty"` + DataLevel string `json:"data_level,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + Metrics []string `json:"metrics,omitempty"` + QueryLifetime bool `json:"query_lifetime,omitempty"` + ReportType string `json:"report_type,omitempty"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsTiktokAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsZendeskSupportOptions struct { @@ -379,12 +418,31 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptions struct { KafkaOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsKafkaOptions `json:"kafka_options,omitempty"` MetaAdsOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsMetaAdsOptions `json:"meta_ads_options,omitempty"` OutlookOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsOutlookOptions `json:"outlook_options,omitempty"` + RedditAdsOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsRedditAdsOptions `json:"reddit_ads_options,omitempty"` SharepointOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsSharepointOptions `json:"sharepoint_options,omitempty"` SmartsheetOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsSmartsheetOptions `json:"smartsheet_options,omitempty"` TiktokAdsOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsTiktokAdsOptions `json:"tiktok_ads_options,omitempty"` ZendeskSupportOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptionsZendeskSupportOptions `json:"zendesk_support_options,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptionsTransformsJsonOptions struct { + AsVariant bool `json:"as_variant,omitempty"` + Schema string `json:"schema,omitempty"` + SchemaEvolutionMode string `json:"schema_evolution_mode,omitempty"` + SchemaFilePath string `json:"schema_file_path,omitempty"` + SchemaHints string `json:"schema_hints,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptionsTransforms struct { + Format string `json:"format,omitempty"` + JsonOptions *ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptionsTransformsJsonOptions `json:"json_options,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptions struct { + FanoutBy string `json:"fanout_by,omitempty"` + Transforms []ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptionsTransforms `json:"transforms,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaTableConfigurationAutoFullRefreshPolicy struct { Enabled bool `json:"enabled"` MinIntervalHours int `json:"min_interval_hours,omitempty"` @@ -417,6 +475,7 @@ type ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration struct { SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` SequenceBy []string `json:"sequence_by,omitempty"` + SourceMetadataColumn string `json:"source_metadata_column,omitempty"` TableProperties map[string]string `json:"table_properties,omitempty"` AutoFullRefreshPolicy *ResourcePipelineIngestionDefinitionObjectsSchemaTableConfigurationAutoFullRefreshPolicy `json:"auto_full_refresh_policy,omitempty"` QueryBasedConnectorConfig *ResourcePipelineIngestionDefinitionObjectsSchemaTableConfigurationQueryBasedConnectorConfig `json:"query_based_connector_config,omitempty"` @@ -429,6 +488,7 @@ type ResourcePipelineIngestionDefinitionObjectsSchema struct { SourceCatalog string `json:"source_catalog,omitempty"` SourceSchema string `json:"source_schema"` ConnectorOptions *ResourcePipelineIngestionDefinitionObjectsSchemaConnectorOptions `json:"connector_options,omitempty"` + FanoutOptions *ResourcePipelineIngestionDefinitionObjectsSchemaFanoutOptions `json:"fanout_options,omitempty"` TableConfiguration *ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration `json:"table_configuration,omitempty"` } @@ -462,10 +522,18 @@ type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsGdriveOption FileIngestionOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsGdriveOptionsFileIngestionOptions `json:"file_ingestion_options,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsGoogleAdsOptionsCustomReportOptions struct { + Metrics []string `json:"metrics,omitempty"` + Resource string `json:"resource"` + ResourceFields []string `json:"resource_fields,omitempty"` + Segments []string `json:"segments,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsGoogleAdsOptions struct { - LookbackWindowDays int `json:"lookback_window_days,omitempty"` - ManagerAccountId string `json:"manager_account_id"` - SyncStartDate string `json:"sync_start_date,omitempty"` + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + ManagerAccountId string `json:"manager_account_id"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsGoogleAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsJiraOptions struct { @@ -508,15 +576,25 @@ type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsKafkaOptions ValueTransformer *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsKafkaOptionsValueTransformer `json:"value_transformer,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsMetaAdsOptionsCustomReportOptions struct { + ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` + ActionBreakdowns []string `json:"action_breakdowns,omitempty"` + ActionReportTime string `json:"action_report_time,omitempty"` + Breakdowns []string `json:"breakdowns,omitempty"` + Level string `json:"level,omitempty"` + TimeIncrement string `json:"time_increment,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsMetaAdsOptions struct { - ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` - ActionBreakdowns []string `json:"action_breakdowns,omitempty"` - ActionReportTime string `json:"action_report_time,omitempty"` - Breakdowns []string `json:"breakdowns,omitempty"` - CustomInsightsLookbackWindow int `json:"custom_insights_lookback_window,omitempty"` - Level string `json:"level,omitempty"` - StartDate string `json:"start_date,omitempty"` - TimeIncrement string `json:"time_increment,omitempty"` + ActionAttributionWindows []string `json:"action_attribution_windows,omitempty"` + ActionBreakdowns []string `json:"action_breakdowns,omitempty"` + ActionReportTime string `json:"action_report_time,omitempty"` + Breakdowns []string `json:"breakdowns,omitempty"` + CustomInsightsLookbackWindow int `json:"custom_insights_lookback_window,omitempty"` + Level string `json:"level,omitempty"` + StartDate string `json:"start_date,omitempty"` + TimeIncrement string `json:"time_increment,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsMetaAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsOutlookOptions struct { @@ -532,6 +610,17 @@ type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsOutlookOptio SubjectFilter []string `json:"subject_filter,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsRedditAdsOptionsCustomReportOptions struct { + Breakdowns []string `json:"breakdowns,omitempty"` + Fields []string `json:"fields,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsRedditAdsOptions struct { + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsRedditAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsSharepointOptionsFileIngestionOptionsFileFilters struct { ModifiedAfter string `json:"modified_after,omitempty"` ModifiedBefore string `json:"modified_before,omitempty"` @@ -562,14 +651,23 @@ type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsSmartsheetOp EnforceSchema bool `json:"enforce_schema,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsTiktokAdsOptionsCustomReportOptions struct { + DataLevel string `json:"data_level,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Metrics []string `json:"metrics,omitempty"` + QueryLifetime bool `json:"query_lifetime,omitempty"` + ReportType string `json:"report_type,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsTiktokAdsOptions struct { - DataLevel string `json:"data_level,omitempty"` - Dimensions []string `json:"dimensions,omitempty"` - LookbackWindowDays int `json:"lookback_window_days,omitempty"` - Metrics []string `json:"metrics,omitempty"` - QueryLifetime bool `json:"query_lifetime,omitempty"` - ReportType string `json:"report_type,omitempty"` - SyncStartDate string `json:"sync_start_date,omitempty"` + DataLevel string `json:"data_level,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + LookbackWindowDays int `json:"lookback_window_days,omitempty"` + Metrics []string `json:"metrics,omitempty"` + QueryLifetime bool `json:"query_lifetime,omitempty"` + ReportType string `json:"report_type,omitempty"` + SyncStartDate string `json:"sync_start_date,omitempty"` + CustomReportOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsTiktokAdsOptionsCustomReportOptions `json:"custom_report_options,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsZendeskSupportOptions struct { @@ -584,6 +682,7 @@ type ResourcePipelineIngestionDefinitionObjectsTableConnectorOptions struct { KafkaOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsKafkaOptions `json:"kafka_options,omitempty"` MetaAdsOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsMetaAdsOptions `json:"meta_ads_options,omitempty"` OutlookOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsOutlookOptions `json:"outlook_options,omitempty"` + RedditAdsOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsRedditAdsOptions `json:"reddit_ads_options,omitempty"` SharepointOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsSharepointOptions `json:"sharepoint_options,omitempty"` SmartsheetOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsSmartsheetOptions `json:"smartsheet_options,omitempty"` TiktokAdsOptions *ResourcePipelineIngestionDefinitionObjectsTableConnectorOptionsTiktokAdsOptions `json:"tiktok_ads_options,omitempty"` @@ -622,6 +721,7 @@ type ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration struct { SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` SequenceBy []string `json:"sequence_by,omitempty"` + SourceMetadataColumn string `json:"source_metadata_column,omitempty"` TableProperties map[string]string `json:"table_properties,omitempty"` AutoFullRefreshPolicy *ResourcePipelineIngestionDefinitionObjectsTableTableConfigurationAutoFullRefreshPolicy `json:"auto_full_refresh_policy,omitempty"` QueryBasedConnectorConfig *ResourcePipelineIngestionDefinitionObjectsTableTableConfigurationQueryBasedConnectorConfig `json:"query_based_connector_config,omitempty"` @@ -700,6 +800,7 @@ type ResourcePipelineIngestionDefinitionTableConfiguration struct { SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` SequenceBy []string `json:"sequence_by,omitempty"` + SourceMetadataColumn string `json:"source_metadata_column,omitempty"` TableProperties map[string]string `json:"table_properties,omitempty"` AutoFullRefreshPolicy *ResourcePipelineIngestionDefinitionTableConfigurationAutoFullRefreshPolicy `json:"auto_full_refresh_policy,omitempty"` QueryBasedConnectorConfig *ResourcePipelineIngestionDefinitionTableConfigurationQueryBasedConnectorConfig `json:"query_based_connector_config,omitempty"` diff --git a/bundle/internal/tf/schema/resource_postgres_cdf_config.go b/bundle/internal/tf/schema/resource_postgres_cdf_config.go new file mode 100644 index 00000000000..6b38760640e --- /dev/null +++ b/bundle/internal/tf/schema/resource_postgres_cdf_config.go @@ -0,0 +1,18 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourcePostgresCdfConfigProviderConfig struct { + WorkspaceId string `json:"workspace_id,omitempty"` +} + +type ResourcePostgresCdfConfig struct { + Catalog string `json:"catalog"` + CdfConfigId string `json:"cdf_config_id,omitempty"` + CreateTime string `json:"create_time,omitempty"` + Name string `json:"name,omitempty"` + Parent string `json:"parent"` + PostgresSchema string `json:"postgres_schema"` + ProviderConfig *ResourcePostgresCdfConfigProviderConfig `json:"provider_config,omitempty"` + Schema string `json:"schema"` +} diff --git a/bundle/internal/tf/schema/resource_postgres_database.go b/bundle/internal/tf/schema/resource_postgres_database.go index f9d1dadb469..cfaa32f7d28 100644 --- a/bundle/internal/tf/schema/resource_postgres_database.go +++ b/bundle/internal/tf/schema/resource_postgres_database.go @@ -8,7 +8,7 @@ type ResourcePostgresDatabaseProviderConfig struct { type ResourcePostgresDatabaseSpec struct { PostgresDatabase string `json:"postgres_database,omitempty"` - Role string `json:"role,omitempty"` + Role string `json:"role"` } type ResourcePostgresDatabaseStatus struct { diff --git a/bundle/internal/tf/schema/resource_postgres_endpoint.go b/bundle/internal/tf/schema/resource_postgres_endpoint.go index af17d5f73bc..485c636d278 100644 --- a/bundle/internal/tf/schema/resource_postgres_endpoint.go +++ b/bundle/internal/tf/schema/resource_postgres_endpoint.go @@ -34,8 +34,10 @@ type ResourcePostgresEndpointStatusGroup struct { } type ResourcePostgresEndpointStatusHosts struct { - Host string `json:"host,omitempty"` - ReadOnlyHost string `json:"read_only_host,omitempty"` + Host string `json:"host,omitempty"` + ReadOnlyHost string `json:"read_only_host,omitempty"` + ReadOnlyPooledHost string `json:"read_only_pooled_host,omitempty"` + ReadWritePooledHost string `json:"read_write_pooled_host,omitempty"` } type ResourcePostgresEndpointStatusSettings struct { @@ -51,6 +53,7 @@ type ResourcePostgresEndpointStatus struct { EndpointType string `json:"endpoint_type,omitempty"` Group *ResourcePostgresEndpointStatusGroup `json:"group,omitempty"` Hosts *ResourcePostgresEndpointStatusHosts `json:"hosts,omitempty"` + LastActiveTime string `json:"last_active_time,omitempty"` PendingState string `json:"pending_state,omitempty"` Settings *ResourcePostgresEndpointStatusSettings `json:"settings,omitempty"` SuspendTimeoutDuration string `json:"suspend_timeout_duration,omitempty"` diff --git a/bundle/internal/tf/schema/resource_postgres_synced_table.go b/bundle/internal/tf/schema/resource_postgres_synced_table.go index 7d55d52de52..958684a9129 100644 --- a/bundle/internal/tf/schema/resource_postgres_synced_table.go +++ b/bundle/internal/tf/schema/resource_postgres_synced_table.go @@ -6,6 +6,13 @@ type ResourcePostgresSyncedTableProviderConfig struct { WorkspaceId string `json:"workspace_id,omitempty"` } +type ResourcePostgresSyncedTableSpecExtraColumns struct { + ColumnName string `json:"column_name"` + ColumnType string `json:"column_type"` + Compute string `json:"compute,omitempty"` + Maintenance string `json:"maintenance,omitempty"` +} + type ResourcePostgresSyncedTableSpecNewPipelineSpec struct { BudgetPolicyId string `json:"budget_policy_id,omitempty"` StorageCatalog string `json:"storage_catalog,omitempty"` @@ -23,6 +30,7 @@ type ResourcePostgresSyncedTableSpec struct { Branch string `json:"branch,omitempty"` CreateDatabaseObjectsIfMissing bool `json:"create_database_objects_if_missing,omitempty"` ExistingPipelineId string `json:"existing_pipeline_id,omitempty"` + ExtraColumns []ResourcePostgresSyncedTableSpecExtraColumns `json:"extra_columns,omitempty"` NewPipelineSpec *ResourcePostgresSyncedTableSpecNewPipelineSpec `json:"new_pipeline_spec,omitempty"` PostgresDatabase string `json:"postgres_database,omitempty"` PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` diff --git a/bundle/internal/tf/schema/resource_secret_uc.go b/bundle/internal/tf/schema/resource_secret_uc.go index 84e199be6f2..a26feafc7ee 100644 --- a/bundle/internal/tf/schema/resource_secret_uc.go +++ b/bundle/internal/tf/schema/resource_secret_uc.go @@ -7,22 +7,20 @@ type ResourceSecretUcProviderConfig struct { } type ResourceSecretUc struct { - BrowseOnly bool `json:"browse_only,omitempty"` - CatalogName string `json:"catalog_name"` - Comment string `json:"comment,omitempty"` - CreateTime string `json:"create_time,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - EffectiveOwner string `json:"effective_owner,omitempty"` - EffectiveValue string `json:"effective_value,omitempty"` - ExpireTime string `json:"expire_time,omitempty"` - ExternalSecretId string `json:"external_secret_id,omitempty"` - FullName string `json:"full_name,omitempty"` - MetastoreId string `json:"metastore_id,omitempty"` - Name string `json:"name"` - Owner string `json:"owner,omitempty"` - ProviderConfig *ResourceSecretUcProviderConfig `json:"provider_config,omitempty"` - SchemaName string `json:"schema_name"` - UpdateTime string `json:"update_time,omitempty"` - UpdatedBy string `json:"updated_by,omitempty"` - Value string `json:"value"` + CatalogName string `json:"catalog_name"` + Comment string `json:"comment,omitempty"` + CreateTime string `json:"create_time,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EffectiveOwner string `json:"effective_owner,omitempty"` + EffectiveValue string `json:"effective_value,omitempty"` + ExpireTime string `json:"expire_time,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name"` + Owner string `json:"owner,omitempty"` + ProviderConfig *ResourceSecretUcProviderConfig `json:"provider_config,omitempty"` + SchemaName string `json:"schema_name"` + UpdateTime string `json:"update_time,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Value string `json:"value"` } diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index ed3f5ad9c5e..38bb586d425 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -109,6 +109,7 @@ type Resources struct { PolicyInfo map[string]any `json:"databricks_policy_info,omitempty"` PostgresBranch map[string]any `json:"databricks_postgres_branch,omitempty"` PostgresCatalog map[string]any `json:"databricks_postgres_catalog,omitempty"` + PostgresCdfConfig map[string]any `json:"databricks_postgres_cdf_config,omitempty"` PostgresDataApi map[string]any `json:"databricks_postgres_data_api,omitempty"` PostgresDatabase map[string]any `json:"databricks_postgres_database,omitempty"` PostgresEndpoint map[string]any `json:"databricks_postgres_endpoint,omitempty"` @@ -273,6 +274,7 @@ func NewResources() *Resources { PolicyInfo: make(map[string]any), PostgresBranch: make(map[string]any), PostgresCatalog: make(map[string]any), + PostgresCdfConfig: make(map[string]any), PostgresDataApi: make(map[string]any), PostgresDatabase: make(map[string]any), PostgresEndpoint: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 57c9a7b2754..83f3f59a966 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -22,9 +22,9 @@ type Root struct { const ( ProviderHost = "registry.terraform.io" ProviderSource = "databricks/databricks" - ProviderVersion = "1.121.0" - ProviderChecksumLinuxAmd64 = "1548a581a52d3d8af6dd33245a05c7e7efd4975a2f5d80dfb625a26958067fcd" - ProviderChecksumLinuxArm64 = "f52645353a5625433863e9f698285c1c5ccea8d5ab0265bf6e6ee243559c728f" + ProviderVersion = "1.123.0" + ProviderChecksumLinuxAmd64 = "e313f21a8d6181c4a239c2ac8a692adc824660cba8e069ed400782938628c0ef" + ProviderChecksumLinuxArm64 = "11da912f542246f642ff2e2169dc11ff486ec18d76e7dfc21a7930b2f8f8f8db" ) func NewRoot() *Root { diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index f223ab7ef52..af9c6c77f6e 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -45,6 +45,7 @@ var EnumFields = map[string][]string{ "resources.clusters.*.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, "resources.clusters.*.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, "resources.clusters.*.data_security_mode": {"DATA_SECURITY_MODE_AUTO", "DATA_SECURITY_MODE_DEDICATED", "DATA_SECURITY_MODE_STANDARD", "LEGACY_PASSTHROUGH", "LEGACY_SINGLE_USER", "LEGACY_SINGLE_USER_STANDARD", "LEGACY_TABLE_ACL", "NONE", "SINGLE_USER", "USER_ISOLATION"}, + "resources.clusters.*.dependency_mode": {"DEPENDENCY_MODE_AUTO", "DEPENDENCY_MODE_CLUSTER_LIBRARIES", "DEPENDENCY_MODE_ENVIRONMENTS"}, "resources.clusters.*.gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, "resources.clusters.*.gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, "resources.clusters.*.kind": {"CLASSIC_PREVIEW"}, @@ -64,6 +65,13 @@ var EnumFields = map[string][]string{ "resources.genie_spaces.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.instance_pools.*.aws_attributes.availability": {"ON_DEMAND", "SPOT"}, + "resources.instance_pools.*.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE"}, + "resources.instance_pools.*.disk_spec.disk_type.azure_disk_volume_type": {"PREMIUM_LRS", "STANDARD_LRS"}, + "resources.instance_pools.*.disk_spec.disk_type.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, + "resources.instance_pools.*.gcp_attributes.gcp_availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, + "resources.instance_pools.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_MANAGE"}, + "resources.job_runs.*.performance_target": {"PERFORMANCE_OPTIMIZED", "STANDARD"}, "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, @@ -79,6 +87,7 @@ var EnumFields = map[string][]string{ "resources.jobs.*.job_clusters[*].new_cluster.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, "resources.jobs.*.job_clusters[*].new_cluster.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, "resources.jobs.*.job_clusters[*].new_cluster.data_security_mode": {"DATA_SECURITY_MODE_AUTO", "DATA_SECURITY_MODE_DEDICATED", "DATA_SECURITY_MODE_STANDARD", "LEGACY_PASSTHROUGH", "LEGACY_SINGLE_USER", "LEGACY_SINGLE_USER_STANDARD", "LEGACY_TABLE_ACL", "NONE", "SINGLE_USER", "USER_ISOLATION"}, + "resources.jobs.*.job_clusters[*].new_cluster.dependency_mode": {"DEPENDENCY_MODE_AUTO", "DEPENDENCY_MODE_CLUSTER_LIBRARIES", "DEPENDENCY_MODE_ENVIRONMENTS"}, "resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, "resources.jobs.*.job_clusters[*].new_cluster.gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, "resources.jobs.*.job_clusters[*].new_cluster.kind": {"CLASSIC_PREVIEW"}, @@ -102,6 +111,7 @@ var EnumFields = map[string][]string{ "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.data_security_mode": {"DATA_SECURITY_MODE_AUTO", "DATA_SECURITY_MODE_DEDICATED", "DATA_SECURITY_MODE_STANDARD", "LEGACY_PASSTHROUGH", "LEGACY_SINGLE_USER", "LEGACY_SINGLE_USER_STANDARD", "LEGACY_TABLE_ACL", "NONE", "SINGLE_USER", "USER_ISOLATION"}, + "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.dependency_mode": {"DEPENDENCY_MODE_AUTO", "DEPENDENCY_MODE_CLUSTER_LIBRARIES", "DEPENDENCY_MODE_ENVIRONMENTS"}, "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, "resources.jobs.*.tasks[*].for_each_task.task.new_cluster.kind": {"CLASSIC_PREVIEW"}, @@ -120,6 +130,7 @@ var EnumFields = map[string][]string{ "resources.jobs.*.tasks[*].new_cluster.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, "resources.jobs.*.tasks[*].new_cluster.azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, "resources.jobs.*.tasks[*].new_cluster.data_security_mode": {"DATA_SECURITY_MODE_AUTO", "DATA_SECURITY_MODE_DEDICATED", "DATA_SECURITY_MODE_STANDARD", "LEGACY_PASSTHROUGH", "LEGACY_SINGLE_USER", "LEGACY_SINGLE_USER_STANDARD", "LEGACY_TABLE_ACL", "NONE", "SINGLE_USER", "USER_ISOLATION"}, + "resources.jobs.*.tasks[*].new_cluster.dependency_mode": {"DEPENDENCY_MODE_AUTO", "DEPENDENCY_MODE_CLUSTER_LIBRARIES", "DEPENDENCY_MODE_ENVIRONMENTS"}, "resources.jobs.*.tasks[*].new_cluster.gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, "resources.jobs.*.tasks[*].new_cluster.gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, "resources.jobs.*.tasks[*].new_cluster.kind": {"CLASSIC_PREVIEW"}, @@ -133,7 +144,7 @@ var EnumFields = map[string][]string{ "resources.jobs.*.tasks[*].sql_task.file.source": {"GIT", "WORKSPACE"}, "resources.jobs.*.trigger.model.condition": {"MODEL_ALIAS_SET", "MODEL_CREATED", "MODEL_VERSION_READY"}, "resources.jobs.*.trigger.pause_status": {"PAUSED", "UNPAUSED"}, - "resources.jobs.*.trigger.periodic.unit": {"DAYS", "HOURS", "WEEKS"}, + "resources.jobs.*.trigger.periodic.unit": {"DAYS", "HOURS", "MINUTES", "WEEKS"}, "resources.jobs.*.trigger.sql_condition.trigger_mode": {"QUERY_RETURNS_ROWS", "RESULT_VALUE_CHANGES"}, "resources.jobs.*.trigger.table_update.condition": {"ALL_UPDATED", "ANY_UPDATED"}, @@ -173,8 +184,12 @@ var EnumFields = map[string][]string{ "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.custom_report_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].format": {"JSON", "STRING"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.fanout_options.transforms[*].json_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.entity_type": {"FILE", "FILE_METADATA", "PERMISSION"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, @@ -188,6 +203,8 @@ var EnumFields = map[string][]string{ "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.custom_report_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, @@ -204,8 +221,9 @@ var EnumFields = map[string][]string{ "resources.postgres_roles.*.identity_type": {"GROUP", "SERVICE_PRINCIPAL", "USER"}, "resources.postgres_roles.*.membership_roles[*]": {"DATABRICKS_SUPERUSER"}, - "resources.postgres_synced_tables.*.scheduling_policy": {"CONTINUOUS", "SNAPSHOT", "TRIGGERED"}, - "resources.postgres_synced_tables.*.type_overrides[*].pg_type": {"PG_SPECIFIC_TYPE_VECTOR"}, + "resources.postgres_synced_tables.*.extra_columns[*].maintenance": {"STORED_GENERATED"}, + "resources.postgres_synced_tables.*.scheduling_policy": {"CONTINUOUS", "SNAPSHOT", "TRIGGERED"}, + "resources.postgres_synced_tables.*.type_overrides[*].pg_type": {"PG_SPECIFIC_TYPE_VECTOR"}, "resources.quality_monitors.*.custom_metrics[*].type": {"CUSTOM_METRIC_TYPE_AGGREGATE", "CUSTOM_METRIC_TYPE_DERIVED", "CUSTOM_METRIC_TYPE_DRIFT"}, "resources.quality_monitors.*.inference_log.problem_type": {"PROBLEM_TYPE_CLASSIFICATION", "PROBLEM_TYPE_REGRESSION"}, @@ -231,7 +249,7 @@ var EnumFields = map[string][]string{ "resources.synced_database_tables.*.unity_catalog_provisioning_state": {"ACTIVE", "DEGRADED", "DELETING", "FAILED", "PROVISIONING", "UPDATING"}, "resources.vector_search_endpoints.*.endpoint_type": {"STANDARD", "STORAGE_OPTIMIZED"}, - "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_CREATE", "CAN_MANAGE", "CAN_USE"}, "resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type": {"CONTINUOUS", "TRIGGERED"}, "resources.vector_search_indexes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 1eb3d7443c0..6b5fd3917c2 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -15,6 +15,7 @@ var RequiredFields = map[string][]string{ "resources.alerts.*.evaluation": {"comparison_operator", "source"}, "resources.alerts.*.evaluation.source": {"name"}, "resources.alerts.*.evaluation.threshold.column": {"name"}, + "resources.alerts.*.parameters[*]": {"name"}, "resources.alerts.*.permissions[*]": {"level"}, "resources.alerts.*.schedule": {"quartz_cron_schedule", "timezone_id"}, @@ -59,13 +60,17 @@ var RequiredFields = map[string][]string{ "resources.database_instances.*": {"name"}, "resources.database_instances.*.permissions[*]": {"level"}, - "resources.experiments.*": {"name"}, - "resources.experiments.*.permissions[*]": {"level"}, + "resources.experiments.*": {"name"}, + "resources.experiments.*.permissions[*]": {"level"}, + "resources.experiments.*.trace_location.uc_trace_location": {"catalog", "schema"}, "resources.external_locations.*": {"credential_name", "name", "url"}, "resources.genie_spaces.*.permissions[*]": {"level"}, + "resources.instance_pools.*": {"instance_pool_name", "node_type_id"}, + "resources.instance_pools.*.permissions[*]": {"level"}, + "resources.job_runs.*": {"job_id"}, "resources.job_runs.*.queue": {"enabled"}, @@ -202,39 +207,41 @@ var RequiredFields = map[string][]string{ "resources.models.*": {"name"}, "resources.models.*.permissions[*]": {"level"}, - "resources.pipelines.*.clusters[*].autoscale": {"max_workers", "min_workers"}, - "resources.pipelines.*.clusters[*].cluster_log_conf.dbfs": {"destination"}, - "resources.pipelines.*.clusters[*].cluster_log_conf.s3": {"destination"}, - "resources.pipelines.*.clusters[*].cluster_log_conf.volumes": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].abfss": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].dbfs": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].file": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].gcs": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].s3": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].volumes": {"destination"}, - "resources.pipelines.*.clusters[*].init_scripts[*].workspace": {"destination"}, - "resources.pipelines.*.deployment": {"kind"}, - "resources.pipelines.*.gateway_definition": {"connection_name", "gateway_storage_catalog", "gateway_storage_schema"}, - "resources.pipelines.*.ingestion_definition.data_staging_options": {"catalog_name", "schema_name"}, - "resources.pipelines.*.ingestion_definition.full_refresh_window": {"start_hour"}, - "resources.pipelines.*.ingestion_definition.objects[*].report": {"destination_catalog", "destination_schema", "source_url"}, - "resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy": {"enabled"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema": {"destination_catalog", "destination_schema", "source_schema"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options": {"manager_account_id"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy": {"enabled"}, - "resources.pipelines.*.ingestion_definition.objects[*].table": {"destination_catalog", "destination_schema", "source_table"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options": {"manager_account_id"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy": {"enabled"}, - "resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy": {"enabled"}, - "resources.pipelines.*.libraries[*].maven": {"coordinates"}, - "resources.pipelines.*.permissions[*]": {"level"}, - "resources.pipelines.*.restart_window": {"start_hour"}, + "resources.pipelines.*.clusters[*].autoscale": {"max_workers", "min_workers"}, + "resources.pipelines.*.clusters[*].cluster_log_conf.dbfs": {"destination"}, + "resources.pipelines.*.clusters[*].cluster_log_conf.s3": {"destination"}, + "resources.pipelines.*.clusters[*].cluster_log_conf.volumes": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].abfss": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].dbfs": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].file": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].gcs": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].s3": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].volumes": {"destination"}, + "resources.pipelines.*.clusters[*].init_scripts[*].workspace": {"destination"}, + "resources.pipelines.*.deployment": {"kind"}, + "resources.pipelines.*.gateway_definition": {"connection_name", "gateway_storage_catalog", "gateway_storage_schema"}, + "resources.pipelines.*.ingestion_definition.data_staging_options": {"catalog_name", "schema_name"}, + "resources.pipelines.*.ingestion_definition.full_refresh_window": {"start_hour"}, + "resources.pipelines.*.ingestion_definition.objects[*].report": {"destination_catalog", "destination_schema", "source_url"}, + "resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.auto_full_refresh_policy": {"enabled"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema": {"destination_catalog", "destination_schema", "source_schema"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options": {"manager_account_id"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.google_ads_options.custom_report_options": {"resource"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.auto_full_refresh_policy": {"enabled"}, + "resources.pipelines.*.ingestion_definition.objects[*].table": {"destination_catalog", "destination_schema", "source_table"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options": {"manager_account_id"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.google_ads_options.custom_report_options": {"resource"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.auto_full_refresh_policy": {"enabled"}, + "resources.pipelines.*.ingestion_definition.table_configuration.auto_full_refresh_policy": {"enabled"}, + "resources.pipelines.*.libraries[*].maven": {"coordinates"}, + "resources.pipelines.*.permissions[*]": {"level"}, + "resources.pipelines.*.restart_window": {"start_hour"}, "resources.postgres_branches.*": {"branch_id", "parent"}, "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, - "resources.postgres_databases.*": {"database_id", "parent"}, + "resources.postgres_databases.*": {"role", "database_id", "parent"}, "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, "resources.postgres_endpoints.*.group": {"max", "min"}, @@ -245,6 +252,7 @@ var RequiredFields = map[string][]string{ "resources.postgres_roles.*": {"role_id", "parent"}, "resources.postgres_synced_tables.*": {"synced_table_id"}, + "resources.postgres_synced_tables.*.extra_columns[*]": {"column_name", "column_type"}, "resources.postgres_synced_tables.*.type_overrides[*]": {"column_name", "pg_type"}, "resources.quality_monitors.*": {"assets_dir", "output_schema_name", "table_name"}, diff --git a/bundle/libraries/filer_volume.go b/bundle/libraries/filer_volume.go index f4b5f51f0c2..fdd6190e019 100644 --- a/bundle/libraries/filer_volume.go +++ b/bundle/libraries/filer_volume.go @@ -10,6 +10,6 @@ import ( func filerForVolume(ctx context.Context, b *bundle.Bundle, uploadPath string) (filer.Filer, string, diag.Diagnostics) { w := b.WorkspaceClient(ctx) - f, err := filer.NewFilesClient(w, uploadPath) + f, err := filer.NewFilesClient(ctx, w, uploadPath) return f, uploadPath, diag.FromErr(err) } diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index 002adea1ddf..f0a101b8ad0 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -42,6 +42,15 @@ const ( DirectAutoMigrateViaConfig = "direct_migrated_via_config" DirectAutoMigrateViaEnv = "direct_migrated_via_env" + // Whether the (deprecated) terraform engine was explicitly opted into, and via + // which source. Independent: both can be present when config and env both + // request terraform. Emitted only when true; absence means "not opted in via + // this source", so the population still on terraform can be sliced by source. + // - engine_terraform_config: bundle.engine = "terraform" in the bundle config. + // - engine_terraform_env: DATABRICKS_BUNDLE_ENGINE=terraform in the environment. + EngineTerraformConfig = "engine_terraform_config" + EngineTerraformEnv = "engine_terraform_env" + // Whether workspace.state_path is under /Workspace/Shared. StatePathIsShared = "state_path_is_shared" diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 48ba7755714..81f110f5095 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -23,7 +23,7 @@ import ( func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, engine engine.EngineType) { log.Info(ctx, "Phase: bind") - bundle.ApplyContext(ctx, b, lock.Acquire()) + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalBind)) if logdiag.HasError(ctx) { return } @@ -119,7 +119,7 @@ func jsonDump(ctx context.Context, v any, field string) string { func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, resourceKey string, engine engine.EngineType) { log.Info(ctx, "Phase: unbind") - bundle.ApplyContext(ctx, b, lock.Acquire()) + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalUnbind)) if logdiag.HasError(ctx) { return } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..d2275ef1c6d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -152,7 +152,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // mutators need informed consent if they are potentially destructive. bundle.ApplySeqContext(ctx, b, scripts.Execute(config.ScriptPreDeploy), - lock.Acquire(), + lock.Acquire(lock.GoalDeploy), ) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..2b52d575344 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -126,7 +126,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } - bundle.ApplyContext(ctx, b, lock.Acquire()) + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDestroy)) if logdiag.HasError(ctx) { return } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..31c37064029 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -154,6 +154,12 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { mutator.InitializeVolumePaths(), mutator.ResolveVolumePathReferencesOnlyResources(), + // Drop empty-string values on omitempty resource fields so they are not + // force-sent to the backend. Runs after variable resolution (a variable may + // resolve to "") and after all resource mutations, before validation so that + // required fields (no omitempty) still error if empty. + mutator.DropEmptyStrings(), + // Resolve --select selectors against the materialized resources: normalize // each to its "type.name" form and validate it exists. Runs after all resource // mutations so that dynamically added resources are visible. This does not diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 1a0123edd14..cbe74f467fe 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/dyn" @@ -201,6 +202,18 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { } } + // Record whether the deprecated terraform engine was explicitly opted into, + // separately per source. Only emitted when true; absence means "not opted in + // via this source". An invalid env value has already failed the command + // upstream via ResolveEngineSetting, so treating the FromEnv error as + // "not terraform" here cannot mask a real terraform opt-in. + if b.Config.Bundle.Engine == engine.EngineTerraform { + b.Metrics.SetBoolValue(metrics.EngineTerraformConfig, true) + } + if envEngine, _ := engine.FromEnv(ctx); envEngine == engine.EngineTerraform { + b.Metrics.SetBoolValue(metrics.EngineTerraformEnv, true) + } + // If the bundle UUID is not set, we use a default 0 value. bundleUuid := "00000000-0000-0000-0000-000000000000" if b.Config.Bundle.Uuid != "" { diff --git a/bundle/root.go b/bundle/root.go index 9ea9a8c13fe..5f7317a5e99 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -8,14 +8,16 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/folders" ) // getRootEnv returns the value of the bundle root environment variable -// if it set and is a directory. If the environment variable is set but -// is not a directory, it returns an error. If the environment variable is -// not set, it returns an empty string. -func getRootEnv(ctx context.Context) (string, error) { +// if it is set and points to a directory that contains a bundle configuration +// file. If the environment variable is set but does not point to such a +// directory, it returns diagnostics. If the environment variable is not set, +// it returns an empty string. +func getRootEnv(ctx context.Context) (string, diag.Diagnostics) { path, ok := env.Root(ctx) if !ok { return "", nil @@ -24,18 +26,25 @@ func getRootEnv(ctx context.Context) (string, error) { if err == nil && !stat.IsDir() { err = errors.New("not a directory") } + if err == nil { + _, err = config.FileNames.FindInPath(path) + } if err != nil { - return "", fmt.Errorf(`invalid bundle root %s="%s": %w`, env.RootVariable, path, err) + return "", diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf(`Invalid bundle root %s="%s": %s`, env.RootVariable, path, err), + Detail: fmt.Sprintf("The %s environment variable must point to an existing directory that contains a bundle configuration file.", env.RootVariable), + }} } return path, nil } // getRootWithTraversal returns the bundle root by traversing the filesystem // from the working directory to the root looking for a configuration file. -func getRootWithTraversal() (string, error) { +func getRootWithTraversal() (string, diag.Diagnostics) { wd, err := os.Getwd() if err != nil { - return "", err + return "", diag.FromErr(err) } for _, file := range config.FileNames { @@ -45,24 +54,28 @@ func getRootWithTraversal() (string, error) { } } - return "", fmt.Errorf(`unable to locate bundle root: %s not found`, config.FileNames[0]) + return "", diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("Unable to locate the bundle root: %s not found.", config.FileNames[0]), + Detail: fmt.Sprintf("Run this command from a directory that contains a bundle, or set the %s environment variable to the bundle root directory.\nAlternatively, create a new bundle with 'databricks bundle init'.", env.RootVariable), + }} } -// mustGetRoot returns a bundle root or an error if one cannot be found. -func mustGetRoot(ctx context.Context) (string, error) { - path, err := getRootEnv(ctx) - if path != "" || err != nil { - return path, err +// mustGetRoot returns a bundle root or diagnostics if one cannot be found. +func mustGetRoot(ctx context.Context) (string, diag.Diagnostics) { + path, diags := getRootEnv(ctx) + if path != "" || diags.HasError() { + return path, diags } return getRootWithTraversal() } // tryGetRoot returns a bundle root or an empty string if one cannot be found. -func tryGetRoot(ctx context.Context) (string, error) { +func tryGetRoot(ctx context.Context) (string, diag.Diagnostics) { // Note: an invalid value in the environment variable is still an error. - path, err := getRootEnv(ctx) - if path != "" || err != nil { - return path, err + path, diags := getRootEnv(ctx) + if path != "" || diags.HasError() { + return path, diags } // Note: traversal failing means the bundle root cannot be found. path, _ = getRootWithTraversal() diff --git a/bundle/root_test.go b/bundle/root_test.go index 07c9ed20a61..27109f2aeff 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -15,9 +15,14 @@ func TestRootFromEnv(t *testing.T) { dir := t.TempDir() t.Setenv(env.RootVariable, dir) - // It should pull the root from the environment variable. - root, err := mustGetRoot(ctx) + // The bundle root must contain a configuration file. + f, err := os.Create(filepath.Join(dir, config.FileNames[0])) require.NoError(t, err) + f.Close() + + // It should pull the root from the environment variable. + root, diags := mustGetRoot(ctx) + require.NoError(t, diags.Error()) require.Equal(t, root, dir) } @@ -27,8 +32,9 @@ func TestRootFromEnvDoesntExist(t *testing.T) { t.Setenv(env.RootVariable, filepath.Join(dir, "doesntexist")) // It should pull the root from the environment variable. - _, err := mustGetRoot(ctx) - require.Errorf(t, err, "invalid bundle root") + _, diags := mustGetRoot(ctx) + require.True(t, diags.HasError()) + require.Contains(t, diags[0].Summary, "Invalid bundle root") } func TestRootFromEnvIsFile(t *testing.T) { @@ -40,8 +46,9 @@ func TestRootFromEnvIsFile(t *testing.T) { t.Setenv(env.RootVariable, f.Name()) // It should pull the root from the environment variable. - _, err = mustGetRoot(ctx) - require.Errorf(t, err, "invalid bundle root") + _, diags := mustGetRoot(ctx) + require.True(t, diags.HasError()) + require.Contains(t, diags[0].Summary, "Invalid bundle root") } func TestRootIfEnvIsEmpty(t *testing.T) { @@ -50,8 +57,9 @@ func TestRootIfEnvIsEmpty(t *testing.T) { t.Setenv(env.RootVariable, dir) // It should pull the root from the environment variable. - _, err := mustGetRoot(ctx) - require.Errorf(t, err, "invalid bundle root") + _, diags := mustGetRoot(ctx) + require.True(t, diags.HasError()) + require.Contains(t, diags[0].Summary, "Invalid bundle root") } func TestRootLookup(t *testing.T) { @@ -82,8 +90,8 @@ func TestRootLookup(t *testing.T) { // It should find the project root from $PWD. t.Chdir("./a/b/c") - foundRoot, err := mustGetRoot(ctx) - require.NoError(t, err) + foundRoot, diags := mustGetRoot(ctx) + require.NoError(t, diags.Error()) foundRoot, err = filepath.EvalSymlinks(foundRoot) require.NoError(t, err) require.Equal(t, root, foundRoot) @@ -98,6 +106,7 @@ func TestRootLookupError(t *testing.T) { // It can't find a project root from a temporary directory. t.Chdir(t.TempDir()) - _, err := mustGetRoot(ctx) - require.ErrorContains(t, err, "unable to locate bundle root") + _, diags := mustGetRoot(ctx) + require.True(t, diags.HasError()) + require.Contains(t, diags[0].Summary, "Unable to locate the bundle root") } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2fe8eecb720..9c41d1a0b9b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -80,19 +80,18 @@ "type": "object", "properties": { "custom_description": { - "description": "[Public Preview] Custom description for the alert. support mustache template.", + "description": "Custom description for the alert. support mustache template.", "$ref": "#/$defs/string" }, "custom_summary": { - "description": "[Public Preview] Custom summary for the alert. support mustache template.", + "description": "Custom summary for the alert. support mustache template.", "$ref": "#/$defs/string" }, "display_name": { - "description": "[Public Preview] The display name of the alert.", + "description": "The display name of the alert.", "$ref": "#/$defs/string" }, "evaluation": { - "description": "[Public Preview]", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2Evaluation" }, "file_path": { @@ -102,8 +101,14 @@ "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, + "parameters": { + "description": "[Private Preview] Query parameters bound when executing the alert query, referenced in the\nquery text with `:name` syntax. Static values only.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/sql.AlertStatementParameter", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "parent_path": { - "description": "[Public Preview] The workspace path of the folder containing the alert. Can only be set on create, and cannot be updated.", + "description": "The workspace path of the folder containing the alert. Can only be set on create, and cannot be updated.", "$ref": "#/$defs/string" }, "permissions": { @@ -112,25 +117,24 @@ "markdownDescription": "A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." }, "query_text": { - "description": "[Public Preview] Text of the query to be run.", + "description": "Text of the query to be run.", "$ref": "#/$defs/string" }, "run_as": { - "description": "[Public Preview] Specifies the identity that will be used to run the alert.\nThis field allows you to configure alerts to run as a specific user or service principal.\n- For user identity: Set `user_name` to the email of an active workspace user. Users can only set this to their own email.\n- For service principal: Set `service_principal_name` to the application ID. Requires the `servicePrincipal/user` role.\nIf not specified, the alert will run as the request user.", + "description": "Specifies the identity that will be used to run the alert.\nThis field allows you to configure alerts to run as a specific user or service principal.\n- For user identity: Set `user_name` to the email of an active workspace user. Users can only set this to their own email.\n- For service principal: Set `service_principal_name` to the application ID. Requires the `servicePrincipal/user` role.\nIf not specified, the alert will run as the request user.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2RunAs" }, "run_as_user_name": { - "description": "[Public Preview] The run as username or application ID of service principal.\nOn Create and Update, this field can be set to application ID of an active service principal. Setting this field requires the servicePrincipal/user role.\nDeprecated: Use `run_as` field instead. This field will be removed in a future release.", + "description": "The run as username or application ID of service principal.\nOn Create and Update, this field can be set to application ID of an active service principal. Setting this field requires the servicePrincipal/user role.\nDeprecated: Use `run_as` field instead. This field will be removed in a future release.", "$ref": "#/$defs/string", "deprecationMessage": "This field is deprecated", "deprecated": true }, "schedule": { - "description": "[Public Preview]", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.CronSchedule" }, "warehouse_id": { - "description": "[Public Preview] ID of the SQL warehouse attached to the alert.", + "description": "ID of the SQL warehouse attached to the alert.", "$ref": "#/$defs/string" } }, @@ -425,6 +429,10 @@ "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n* `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n\nThe following modes are legacy aliases for the above modes:\n\n* `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`.\n* `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" }, + "dependency_mode": { + "description": "[Beta] Controls dependency configuration for the cluster.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DependencyMode" + }, "docker_image": { "description": "Custom docker image BYOC", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" @@ -879,6 +887,131 @@ } ] }, + "resources.InstancePool": { + "oneOf": [ + { + "type": "object", + "properties": { + "aws_attributes": { + "description": "Attributes related to instance pools running on Amazon Web Services.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to instance pools running on Azure.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAzureAttributes" + }, + "custom_tags": { + "description": "Additional tags for pool resources. Databricks will tag all pool resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags", + "$ref": "#/$defs/map/string" + }, + "disk_spec": { + "description": "Defines the specification of the disks that will be attached to all spark containers.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskSpec" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this instances in this pool will dynamically acquire\nadditional disk space when its Spark workers are running low on disk space. In AWS, this\nfeature requires specific AWS permissions to function correctly - refer to the User Guide for\nmore details.", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to instance pools running on Google Cloud Platform.\nIf not specified at pool creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolGcpAttributes" + }, + "idle_instance_autotermination_minutes": { + "description": "Automatically terminates the extra instances in the pool cache after they are inactive for this\ntime in minutes if min_idle_instances requirement is already met. If not set, the extra pool\ninstances will be automatically terminated after a default timeout. If specified, the\nthreshold must be between 0 and 10000 minutes.\nUsers can also set this value to 0 to instantly remove idle instances from the cache if\nmin cache size could still hold.", + "$ref": "#/$defs/int" + }, + "instance_pool_name": { + "description": "Pool name requested by the user. Pool name must be unique. Length must be between 1 and 100\ncharacters.", + "$ref": "#/$defs/string" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "max_capacity": { + "description": "Maximum number of outstanding instances to keep in the pool, including both instances used by\nclusters and idle instances. Clusters that require further instance provisioning will fail during\nupsize requests.", + "$ref": "#/$defs/int" + }, + "min_idle_instances": { + "description": "Minimum number of idle instances to keep in the instance pool", + "$ref": "#/$defs/int" + }, + "node_type_flexibility": { + "description": "Flexible node type configuration for the pool.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.NodeTypeFlexibility" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the [clusters/listNodeTypes](https://docs.databricks.com/api/workspace/clusters/listnodetypes) API call.", + "$ref": "#/$defs/string" + }, + "permissions": { + "description": "The permissions to apply to this resource.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.InstancePoolPermission", + "markdownDescription": "A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." + }, + "preloaded_docker_images": { + "description": "Custom Docker Image BYOC", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "preloaded_spark_versions": { + "description": "A list containing at most one preloaded Spark image version for the pool. Pool-backed clusters started\nwith the preloaded Spark version will start faster. A list of available Spark versions\ncan be retrieved by using the [clusters/sparkVersions](https://docs.databricks.com/api/workspace/clusters/sparkversions) API call.", + "$ref": "#/$defs/slice/string" + }, + "remote_disk_throughput": { + "description": "If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported for GCP HYPERDISK_BALANCED types.", + "$ref": "#/$defs/int" + }, + "total_initial_remote_disk_size": { + "description": "If set, what the total initial volume size (in GB) of the remote disks should be. Currently only supported for GCP HYPERDISK_BALANCED types.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "instance_pool_name", + "node_type_id" + ], + "markdownDescription": "The instance pool resource defines an [instance pool](https://docs.databricks.com/api/workspace/instancepools/create), a set of idle, ready-to-use cloud instances that reduce cluster start and auto-scaling times." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.InstancePoolPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "description": "The name of the group granted the permission level.", + "$ref": "#/$defs/string" + }, + "level": { + "description": "The permission level to apply. The allowed levels depend on the resource type.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolPermissionLevel" + }, + "service_principal_name": { + "description": "The name of the service principal granted the permission level.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The name of the user granted the permission level.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -1202,6 +1335,12 @@ "tags": { "description": "A collection of tags to set on the experiment. Maximum tag size and number of tags per request\ndepends on the storage backend. All storage backends are guaranteed to support tag keys up\nto 250 bytes in size and tag values up to 5000 bytes in size. All storage backends are also\nguaranteed to support up to 20 tags per request.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + }, + "trace_location": { + "description": "[Private Preview] The location where the experiment's traces are stored. When set, the\nunderlying storage is provisioned and the experiment's traces are routed\nto it. When unset, traces are stored in the default MLflow backend. This\nfield cannot be updated after the experiment is created.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTraceLocation", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true } }, "additionalProperties": false, @@ -1766,7 +1905,8 @@ "additionalProperties": false, "required": [ "database_id", - "parent" + "parent", + "role" ] }, { @@ -1958,6 +2098,9 @@ "existing_pipeline_id": { "$ref": "#/$defs/string" }, + "extra_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecExtraColumn" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" @@ -2091,10 +2234,6 @@ "description": "List of aliases associated with the registered model", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.RegisteredModelAlias" }, - "browse_only": { - "description": "Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.", - "$ref": "#/$defs/bool" - }, "catalog_name": { "description": "The name of the catalog where the schema and the registered model reside", "$ref": "#/$defs/string" @@ -2315,7 +2454,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.Channel" }, "cluster_size": { - "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large\n- Auto", + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on\nit. If you want to increase the number of concurrent queries, please tune\nmax_num_clusters.\n\nSupported values:\n- 2X-Small\n- X-Small\n- Small\n- Medium\n- Large\n- X-Large\n- 2X-Large\n- 3X-Large\n- 4X-Large\n- 5X-Large", "$ref": "#/$defs/string" }, "creator_name": { @@ -2471,11 +2610,11 @@ }, "permissions": { "description": "The permissions to apply to this resource.", - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission", "markdownDescription": "A Sequence of permissions to apply to this resource, where each item grants a permission `level` to a single `user_name`, `group_name`, or `service_principal_name`. A principal cannot be set in both a resource's `permissions` and the top-level `permissions` mapping.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." }, "target_qps": { - "description": "[Public Preview] Target QPS for the endpoint. Mutually exclusive with num_replicas.\nThe actual replica count is calculated at index creation/sync time based on this value.\nBest-effort target; the system does not guarantee this QPS will be achieved.", + "description": "Target QPS for the endpoint. Mutually exclusive with num_replicas.\nThe actual replica count is calculated at index creation/sync time based on this value.\nBest-effort target; the system does not guarantee this QPS will be achieved.", "$ref": "#/$defs/int64" }, "usage_policy_id": { @@ -2497,6 +2636,35 @@ } ] }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorSearchEndpointPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.VectorSearchIndex": { "oneOf": [ { @@ -3096,6 +3264,11 @@ "genie_spaces": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" }, + "instance_pools": { + "description": "The instance pool definitions for the bundle, where each key is the name of the instance pool.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.InstancePool", + "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." + }, "job_runs": { "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" @@ -4078,11 +4251,6 @@ "MEDIUM", "LARGE", "XLARGE" - ], - "enumDescriptions": [ - "", - "", - "[Private Preview]" ] }, { @@ -4753,59 +4921,6 @@ "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "READ_METADATA" - ], - "enumDescriptions": [ - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "[Public Preview]" ] }, { @@ -5055,7 +5170,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAvailability" }, "capacity_reservation_group": { - "description": "[Public Preview] The Azure capacity reservation group resource ID to use for launching VMs.\nWhen specified, VMs will be launched using the provided capacity reservation.\n\nCapacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not\nmanaged by databricks). Ensure the databricks-login-prod Enterprise Application is granted the following four permissions:\n1. Microsoft.Compute/capacityReservationGroups/read\n2. Microsoft.Compute/capacityReservationGroups/deploy/action\n3. Microsoft.Compute/capacityReservationGroups/capacityReservations/read\n4. Microsoft.Compute/capacityReservationGroups/capacityReservations/deploy/action\n\nFormat: `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/capacityReservationGroups/{capacityReservationGroupName}`", + "description": "The Azure capacity reservation group resource ID to use for launching VMs.\nWhen specified, VMs will be launched using the provided capacity reservation.\n\nCapacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not\nmanaged by databricks). Ensure the databricks-login-prod Enterprise Application is granted the following four permissions:\n1. Microsoft.Compute/capacityReservationGroups/read\n2. Microsoft.Compute/capacityReservationGroups/deploy/action\n3. Microsoft.Compute/capacityReservationGroups/capacityReservations/read\n4. Microsoft.Compute/capacityReservationGroups/capacityReservations/deploy/action\n\nFormat: `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/capacityReservationGroups/{capacityReservationGroupName}`", "$ref": "#/$defs/string" }, "first_on_demand": { @@ -5204,6 +5319,10 @@ "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n* `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n\nThe following modes are legacy aliases for the above modes:\n\n* `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`.\n* `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" }, + "dependency_mode": { + "description": "[Beta] Controls dependency configuration for the cluster.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DependencyMode" + }, "docker_image": { "description": "Custom docker image BYOC", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" @@ -5391,18 +5510,53 @@ } ] }, - "compute.DockerBasicAuth": { + "compute.DependencyMode": { + "oneOf": [ + { + "type": "string", + "description": "Controls dependency configuration for the cluster.\n\n* `DEPENDENCY_MODE_AUTO`: Databricks will choose the most appropriate dependency mode based on your compute configuration.\n* `DEPENDENCY_MODE_ENVIRONMENTS`: Enables a unified dependency management experience across classic and serverless, resulting in increased stability and performance. Supported only on DBR 19+ in Standard access mode.\n* `DEPENDENCY_MODE_CLUSTER_LIBRARIES`: Legacy mode: dependencies come from cluster libraries and init scripts.", + "enum": [ + "DEPENDENCY_MODE_ENVIRONMENTS", + "DEPENDENCY_MODE_CLUSTER_LIBRARIES", + "DEPENDENCY_MODE_AUTO" + ], + "enumDescriptions": [ + "[Beta] Enables a unified dependency management experience across classic and serverless, resulting in increased stability and performance. Supported only on DBR 19+ in Standard access mode.", + "[Beta] Legacy mode: dependencies come from cluster libraries and init scripts.", + "[Beta] \u003cDatabricks\u003e will choose the most appropriate dependency mode based on your compute configuration." + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DiskSpec": { "oneOf": [ { "type": "object", + "description": "Describes the disks that are launched for each instance in the spark cluster.\nFor example, if the cluster has 3 instances, each instance is configured to launch\n2 disks, 100 GiB each, then Databricks will launch a total of 6 disks,\n100 GiB each, for this cluster.", "properties": { - "password": { - "description": "Password of the user", - "$ref": "#/$defs/string" + "disk_count": { + "description": "The number of disks launched for each instance:\n- This feature is only enabled for supported node types.\n- Users can choose up to the limit of the disks supported by the node type.\n- For node types with no OS disk, at least one disk must be specified;\notherwise, cluster creation will fail.\n\nIf disks are attached, Databricks will configure Spark to use only the disks for\nscratch storage, because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no disks are attached, Databricks will configure Spark to use\ninstance store disks.\n\nNote: If disks are specified, then the Spark configuration\n`spark.local.dir` will be overridden.\n\nDisks will be mounted at:\n- For AWS: `/ebs0`, `/ebs1`, and etc.\n- For Azure: `/remote_volume0`, `/remote_volume1`, and etc.", + "$ref": "#/$defs/int" }, - "username": { - "description": "Name of the user", - "$ref": "#/$defs/string" + "disk_iops": { + "description": "The number of IOPS to provision for each attached disk.", + "$ref": "#/$defs/int" + }, + "disk_size": { + "description": "The size of each disk (in GiB) launched for each instance.\nValues must fall into the supported range for a particular instance type.\n\nFor AWS:\n- General Purpose SSD: 100 - 4096 GiB\n- Throughput Optimized HDD: 500 - 4096 GiB\n\nFor Azure:\n- Premium LRS (SSD): 1 - 1023 GiB\n- Standard LRS (HDD): 1- 1023 GiB", + "$ref": "#/$defs/int" + }, + "disk_throughput": { + "description": "The disk throughput to provision for each attached disk, in MB per second.", + "$ref": "#/$defs/int" + }, + "disk_type": { + "description": "The type of disks that will be launched with this cluster.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskType" } }, "additionalProperties": false @@ -5413,18 +5567,19 @@ } ] }, - "compute.DockerImage": { + "compute.DiskType": { "oneOf": [ { "type": "object", + "description": "Describes the disk type.", "properties": { - "basic_auth": { - "description": "Basic auth with username and password", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth" + "azure_disk_volume_type": { + "description": "All Azure Disk types that Databricks supports.\nSee https://docs.microsoft.com/en-us/azure/storage/storage-about-disks-and-vhds-linux#types-of-disks", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskTypeAzureDiskVolumeType" }, - "url": { - "description": "URL of the docker image.", - "$ref": "#/$defs/string" + "ebs_volume_type": { + "description": "All EBS volume types that Databricks supports.\nSee https://aws.amazon.com/ebs/details/ for details.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DiskTypeEbsVolumeType" } }, "additionalProperties": false @@ -5435,7 +5590,23 @@ } ] }, - "compute.EbsVolumeType": { + "compute.DiskTypeAzureDiskVolumeType": { + "oneOf": [ + { + "type": "string", + "description": "All Azure Disk types that Databricks supports.\nSee https://docs.microsoft.com/en-us/azure/storage/storage-about-disks-and-vhds-linux#types-of-disks", + "enum": [ + "PREMIUM_LRS", + "STANDARD_LRS" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DiskTypeEbsVolumeType": { "oneOf": [ { "type": "string", @@ -5451,23 +5622,83 @@ } ] }, - "compute.Environment": { + "compute.DockerBasicAuth": { "oneOf": [ { "type": "object", - "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip and java dependencies are supported.", "properties": { - "base_environment": { - "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided.\nFor more information about Databricks-provided base environments, see the\n[list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API.\nFor more information, see", + "password": { + "description": "Password of the user", "$ref": "#/$defs/string" }, - "client": { - "description": "Use `environment_version` instead.", - "$ref": "#/$defs/string", - "deprecationMessage": "This field is deprecated", - "deprecated": true - }, - "dependencies": { + "username": { + "description": "Name of the user", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DockerImage": { + "oneOf": [ + { + "type": "object", + "properties": { + "basic_auth": { + "description": "Basic auth with username and password", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth" + }, + "url": { + "description": "URL of the docker image.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.EbsVolumeType": { + "oneOf": [ + { + "type": "string", + "description": "All EBS volume types that Databricks supports.\nSee https://aws.amazon.com/ebs/details/ for details.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Environment": { + "oneOf": [ + { + "type": "object", + "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip and java dependencies are supported.", + "properties": { + "base_environment": { + "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided.\nFor more information about Databricks-provided base environments, see the\n[list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API.\nFor more information, see", + "$ref": "#/$defs/string" + }, + "client": { + "description": "Use `environment_version` instead.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "dependencies": { "description": "List of pip dependencies, as supported by the version of pip in this environment.", "$ref": "#/$defs/slice/string" }, @@ -5643,6 +5874,139 @@ } ] }, + "compute.InstancePoolAwsAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to Amazon Web Services.", + "properties": { + "availability": { + "description": "Availability type used for the spot nodes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAwsAttributesAvailability" + }, + "instance_profile_arn": { + "description": "[Beta] All AWS instances belonging to the instance pool will have this instance profile. If omitted, instances\nwill initially be launched with the workspace's default instance profile. If defined, clusters that use the\npool will inherit the instance profile, and must not specify their own instance profile on cluster creation or\nupdate. If the pool does not specify an instance profile, clusters using the pool may specify any instance profile.\nThe instance profile must have previously been added to the Databricks environment by an account administrator.\n\nThis feature may only be available to certain customer plans.", + "$ref": "#/$defs/string" + }, + "spot_bid_price_percent": { + "description": "Calculates the bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAwsAttributesAvailability": { + "oneOf": [ + { + "type": "string", + "description": "The set of AWS availability types supported when setting up nodes for a cluster.", + "enum": [ + "SPOT", + "ON_DEMAND" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAzureAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to Azure.", + "properties": { + "availability": { + "description": "Availability type used for the spot nodes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InstancePoolAzureAttributesAvailability" + }, + "capacity_reservation_group": { + "description": "The Azure capacity reservation group resource ID to use for launching VMs in this pool.\nWhen specified, VMs will be launched using the provided capacity reservation.\n\nNOTE: Omitting this field will clear any existing configured capacity reservation group on the pool.\n\nCapacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not\nmanaged by databricks). Ensure the databricks-login-prod Enterprise Application is granted the following four permissions:\n1. Microsoft.Compute/capacityReservationGroups/read\n2. Microsoft.Compute/capacityReservationGroups/deploy/action\n3. Microsoft.Compute/capacityReservationGroups/capacityReservations/read\n4. Microsoft.Compute/capacityReservationGroups/capacityReservations/deploy/action\n\nFormat: `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Compute/capacityReservationGroups/{capacityReservationGroupName}`", + "$ref": "#/$defs/string" + }, + "spot_bid_max_price": { + "description": "With variable pricing, you have option to set a max price, in US dollars (USD)\nFor example, the value 2 would be a max price of $2.00 USD per hour.\nIf you set the max price to be -1, the VM won't be evicted based on price.\nThe price for the VM will be the current price for spot or the price for a standard VM,\nwhich ever is less, as long as there is capacity and quota available.", + "$ref": "#/$defs/float64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolAzureAttributesAvailability": { + "oneOf": [ + { + "type": "string", + "description": "The set of Azure availability types supported when setting up nodes for a cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolGcpAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during instance pool creation which are related to GCP.", + "properties": { + "gcp_availability": { + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability" + }, + "local_ssd_count": { + "description": "If provided, each node in the instance pool will have this number of local SSDs attached.\nEach local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds)\nfor the supported number of local SSDs for each instance type.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west1-a\". The provided availability\nzone must be in the same region as the Databricks workspace. For example, \"us-west1-a\"\nis not a valid zone id if the Databricks workspace resides in the \"us-east1\" region.\nThis is an optional field at instance pool creation, and if not specified, a default zone will be used.\n\nThis field can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones (e.g. \"us-west1-a\").\n\nIf empty, Databricks picks an availability zone to schedule the cluster on.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InstancePoolPermissionLevel": { + "oneOf": [ + { + "type": "string", + "description": "Permission level", + "enum": [ + "CAN_MANAGE", + "CAN_ATTACH_TO" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "compute.Kind": { "oneOf": [ { @@ -6326,7 +6690,7 @@ "doNotSuggest": true }, "size": { - "description": "[Private Preview] Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR\nor PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).", + "description": "[Private Preview] Size parameter for the target type, for types that take one (e.g. vector\ndimension, varchar length). Required when the chosen pg_type needs a size.", "$ref": "#/$defs/int", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -6489,7 +6853,7 @@ "description": "AiRuntimeTask: multi-node GPU compute task definition for Databricks AI\nRuntime workloads.\n\nJobs-framework-level concepts (retries, per-task timeout, idempotency\ntoken, usage/budget policy, permissions) live on the surrounding\nTaskSettings / run-submit request and are intentionally NOT duplicated\nhere. Users compose `ai_runtime_task` with the standard Jobs/DABs task\nwrapper to get those.", "properties": { "code_source_path": { - "description": "[Private Preview] Optional workspace or UC volume path of the uploaded code-source\narchive. The CLI packages the user's local code directory into an\narchive and populates this. Customers calling the Jobs API directly\nshould upload their archive to the workspace or a UC volume first and\nsupply the resulting path here.\n\nWhen set, the training node exposes the value via the `$CODE_SOURCE`\nenvironment variable.", + "description": "[Private Preview] Workspace or UC volume path of the code-source archive, unpacked on\neach node and exposed through `$CODE_SOURCE`. Set by first-party\ntooling; not for direct callers.", "$ref": "#/$defs/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -6726,7 +7090,7 @@ "oneOf": [ { "type": "string", - "description": "Customer-facing AcceleratorType: hardware accelerator type for the\nAiRuntime workload. Per-node accelerator count is encoded in the value\nname (e.g. `GPU_8xH100` means 8 H100s per node).", + "description": "Hardware accelerator type for the AiRuntime workload. Per-node\naccelerator count is encoded in the value name (e.g. `GPU_8xH100` means\n8 H100s per node).", "enum": [ "GPU_1xA10", "GPU_1xH100", @@ -7006,7 +7370,7 @@ "description": "DeploymentSpec: configuration for one deployment within an AiRuntimeTask.\nEach entry in `AiRuntimeTask.deployments` describes a group of nodes that\nshare the same command and compute. Many single-program training\nalgorithms use a single entry where every node runs the same command;\nrole-split workloads (driver + worker, parameter server, separate eval\nnode, etc.) use multiple entries.", "properties": { "command_path": { - "description": "[Private Preview] Workspace path of the bash script to execute on each node in this\ndeployment. The CLI uploads the user's script and populates this.\nCustomers calling the Jobs API directly should upload their script to\nthe workspace first and supply the resulting path here.", + "description": "[Private Preview] Workspace path of the script to run on each node in this deployment.\nUpload the script to this path and supply the path here. When the task\nruns, the file at this path is run on each node; if it fails, the task\nfails with its exit code.\n\nExample script contents:\n\n# Plain Python:\npython train.py --epochs 10\n\n# Multi-GPU via accelerate:\naccelerate launch train.py --config config.yaml\n\n# Distributed via torchrun:\ntorchrun --nproc_per_node=8 train.py", "$ref": "#/$defs/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -7270,6 +7634,12 @@ "new_cluster": { "description": "If new_cluster, a description of a cluster that is created for each task.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + }, + "serverless_compute_id": { + "description": "[Private Preview] The ID of the serverless compute object to bind this cluster to. At most one\nJobCluster per job may set this field; the rate limit defined on the referenced\nserverless compute applies across all tasks bound to this cluster.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true } }, "additionalProperties": false, @@ -7821,7 +8191,8 @@ "enum": [ "HOURS", "DAYS", - "WEEKS" + "WEEKS", + "MINUTES" ] }, { @@ -9108,6 +9479,27 @@ } ] }, + "ml.ExperimentTraceLocation": { + "oneOf": [ + { + "type": "object", + "description": "The storage location for an experiment's traces.", + "properties": { + "uc_trace_location": { + "description": "[Private Preview] A Unity Catalog schema where the experiment's traces are stored as\nDelta tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.UcTraceLocation", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.ModelTag": { "oneOf": [ { @@ -9150,6 +9542,43 @@ } ] }, + "ml.UcTraceLocation": { + "oneOf": [ + { + "type": "object", + "description": "A Unity Catalog trace storage location. Traces are stored as Delta tables\nin the specified catalog and schema.", + "properties": { + "catalog": { + "description": "[Private Preview] The name of the Unity Catalog catalog.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "schema": { + "description": "[Private Preview] The name of the Unity Catalog schema within `catalog`.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "table_prefix": { + "description": "[Private Preview] The prefix for the trace tables, which are named\n`{catalog}.{schema}.{table_prefix}_otel_*`. May only contain letters,\ndigits, and underscores, and may be at most 238 characters. When unset, a\nserver-generated prefix derived from the experiment ID is used and this\nfield stays empty on read; the resolved value is always available in\n`effective_table_prefix`.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "catalog", + "schema" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.AutoFullRefreshPolicy": { "oneOf": [ { @@ -9242,10 +9671,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions" }, "kafka_options": { - "description": "[Private Preview]", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.KafkaOptions", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta]", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.KafkaOptions" }, "meta_ads_options": { "description": "[Beta] Meta Marketing (Meta Ads) specific options for ingestion", @@ -9257,6 +9684,12 @@ "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true }, + "reddit_ads_options": { + "description": "[Private Preview] Reddit Ads specific options for ingestion", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RedditAdsOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "sharepoint_options": { "description": "[Private Preview]", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SharepointOptions", @@ -9589,11 +10022,11 @@ "NONE" ], "enumDescriptions": [ - "[Private Preview]", - "[Private Preview]", - "[Private Preview]", - "[Private Preview]", - "[Private Preview]" + "[Beta]", + "[Beta]", + "[Beta]", + "[Beta]", + "[Beta]" ] }, { @@ -9662,31 +10095,79 @@ } ] }, - "pipelines.GoogleAdsOptions": { + "pipelines.GoogleAdsCustomReportOptions": { "oneOf": [ { "type": "object", - "description": "Google Ads specific options for ingestion (object-level).\nWhen set, these values override the corresponding fields in GoogleAdsConfig\n(source_configurations).", + "description": "User-defined custom report for the Google Ads connector. Mirrors the\nresource + fields + segments + metrics model that Google Ads GAQL exposes.\nThe customer account this report runs against is supplied by the source\nschema (namespace), not by this message. The whole message is gated by\nthe parent GoogleAdsOptions.custom_report_options stage; per-field stage\nannotations are intentionally omitted.\nOnly supported on table-type objects: a custom report requires a\ndestination table, so it cannot be specified at the schema/source level.", "properties": { - "lookback_window_days": { - "description": "[Private Preview] (Optional) Number of days to look back for report tables to capture late-arriving data.\nIf not specified, defaults to 30 days.", - "$ref": "#/$defs/int", + "metrics": { + "description": "[Private Preview] (Optional) Metric fields to select (e.g. \"metrics.clicks\",\n\"metrics.cost_micros\"). Multiple values are joined into the GAQL\nSELECT clause.", + "$ref": "#/$defs/slice/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true }, - "manager_account_id": { - "description": "[Private Preview] (Optional at this level) Manager Account ID (also called MCC Account ID) used to list\nand access customer accounts under this manager account.\nOverrides GoogleAdsConfig.manager_account_id from source_configurations when set.", + "resource": { + "description": "[Private Preview] (Required) Google Ads resource to query (e.g. \"ad_group_ad\",\n\"keyword_view\", \"search_term_view\"). Must be a resource that has\nmetrics. Values are validated against Google Ads' field-service\ncatalog at pipeline plan time.", "$ref": "#/$defs/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true }, - "sync_start_date": { - "description": "[Private Preview] (Optional) Start date for the initial sync of report tables in YYYY-MM-DD format.\nThis determines the earliest date from which to sync historical data.\nIf not specified, defaults to 2 years of historical data.", - "$ref": "#/$defs/string", + "resource_fields": { + "description": "[Private Preview] (Optional) Resource fields to select, in fully-qualified GAQL form\n(e.g. \"ad_group_ad.ad.id\", \"ad_group_ad.status\"). Multiple values are\njoined into the GAQL SELECT clause.", + "$ref": "#/$defs/slice/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true - } - }, + }, + "segments": { + "description": "[Private Preview] (Optional) Segment fields to select (e.g. \"segments.date\",\n\"segments.device\"). Must include at least one of segments.date,\nsegments.week, or segments.month — that segment is used as the\nincremental cursor for the table.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "resource" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.GoogleAdsOptions": { + "oneOf": [ + { + "type": "object", + "description": "Google Ads specific options for ingestion (object-level).\nWhen set, these values override the corresponding fields in GoogleAdsConfig\n(source_configurations).", + "properties": { + "custom_report_options": { + "description": "[Private Preview] (Optional) Custom report definition. When set, the table is treated as a\nuser-defined Google Ads custom report: the connector synthesizes a GAQL\nquery from the resource, fields, segments, and metrics specified here.\nWhen unset, the table must match one of the connector's prebuilt sources.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleAdsCustomReportOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lookback_window_days": { + "description": "[Private Preview] (Optional) Number of days to look back for report tables to capture late-arriving data.\nIf not specified, defaults to 30 days.", + "$ref": "#/$defs/int", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "manager_account_id": { + "description": "[Private Preview] (Optional at this level) Manager Account ID (also called MCC Account ID) used to list\nand access customer accounts under this manager account.\nOverrides GoogleAdsConfig.manager_account_id from source_configurations when set.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "sync_start_date": { + "description": "[Private Preview] (Optional) Start date for the initial sync of report tables in YYYY-MM-DD format.\nThis determines the earliest date from which to sync historical data.\nIf not specified, defaults to 2 years of historical data.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, "additionalProperties": false, "required": [ "manager_account_id" @@ -9890,6 +10371,33 @@ } ] }, + "pipelines.IngestionPipelineDefinitionFanoutOptions": { + "oneOf": [ + { + "type": "object", + "description": "Fanout configuration for multi-table routing from streaming sources.\nRoutes each input record to a destination table based on a routing\nkey derived from the record. The key value becomes the table name\nsuffix: {destination_catalog}.{destination_schema}.{key_value}.", + "properties": { + "fanout_by": { + "description": "[Private Preview] Column path or SQL expression whose value determines the destination table.\nSupports dotted paths (e.g. \"value.event_name\") and expressions\n(e.g. \"value:event_name::string\").", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "transforms": { + "description": "[Private Preview] Optional transforms applied to each route's DataFrame before writing\nto the destination table.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig": { "oneOf": [ { @@ -10059,34 +10567,24 @@ "type": "object", "properties": { "as_variant": { - "description": "[Private Preview] Parse the entire value as a single Variant column.", - "$ref": "#/$defs/bool", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Parse the entire value as a single Variant column.", + "$ref": "#/$defs/bool" }, "schema": { - "description": "[Private Preview] Inline schema string for JSON parsing (Spark DDL format).", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Inline schema string for JSON parsing (Spark DDL format).", + "$ref": "#/$defs/string" }, "schema_evolution_mode": { - "description": "[Private Preview] (Optional) Schema evolution mode for schema inference.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsSchemaEvolutionMode", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] (Optional) Schema evolution mode for schema inference.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsSchemaEvolutionMode" }, "schema_file_path": { - "description": "[Private Preview] Path to a schema file (.ddl).", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Path to a schema file (.ddl).", + "$ref": "#/$defs/string" }, "schema_hints": { - "description": "[Private Preview] (Optional) Schema hints as a comma-separated string of \"column_name type\" pairs.", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] (Optional) Schema hints as a comma-separated string of \"column_name type\" pairs.", + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -10109,10 +10607,8 @@ "doNotSuggest": true }, "key_transformer": { - "description": "[Private Preview] (Optional) Transformer for the message key.\nIf not specified, the key is left as raw bytes.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] (Optional) Transformer for the message key.\nIf not specified, the key is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" }, "max_offsets_per_trigger": { "description": "[Private Preview] Internal option to control the maximum number of offsets to process per trigger.", @@ -10121,28 +10617,20 @@ "doNotSuggest": true }, "starting_offset": { - "description": "[Private Preview] (Optional) Where to begin reading when no checkpoint exists.\nValid values: \"latest\" and \"earliest\". Defaults to \"latest\".", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] (Optional) Where to begin reading when no checkpoint exists.\nValid values: \"latest\" and \"earliest\". Defaults to \"latest\".", + "$ref": "#/$defs/string" }, "topic_pattern": { - "description": "[Private Preview] Java regex pattern to subscribe to matching topics.\nOnly one of topics or topic_pattern must be specified.", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Java regex pattern to subscribe to matching topics.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/string" }, "topics": { - "description": "[Private Preview] Topics to subscribe to.\nOnly one of topics or topic_pattern must be specified.", - "$ref": "#/$defs/slice/string", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Topics to subscribe to.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/slice/string" }, "value_transformer": { - "description": "[Private Preview] (Optional) Transformer for the message value.\nIf not specified, the value is left as raw bytes.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] (Optional) Transformer for the message value.\nIf not specified, the value is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" } }, "additionalProperties": false @@ -10199,6 +10687,12 @@ "description": "[Beta] (Optional) Window in days to revisit data during sync to capture\nupdated conversion data from the API, shared by prebuilt and custom reports.", "$ref": "#/$defs/int" }, + "custom_report_options": { + "description": "[Private Preview] (Optional) Per-table custom report definition. When set, defines the shape of the insights\ncall for this table (level/fields/breakdowns/action_breakdowns/etc.). Supersedes the deprecated\nflat report-shape fields above.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "level": { "description": "[Beta] (Optional, DEPRECATED — use custom_report_options.level) Granularity of data to pull\n(account, ad, adset, campaign)", "$ref": "#/$defs/string", @@ -10224,6 +10718,57 @@ } ] }, + "pipelines.MetaMarketingOptionsMetaMarketingCustomReportOptions": { + "oneOf": [ + { + "type": "object", + "description": "Defines the shape of a single Meta Ads custom report (one /insights call shape).\nstart_date, custom_insights_lookback_window live on MetaMarketingOptions, not here.\nMetrics are not customer-selectable; the connector returns a fixed standard metric set.", + "properties": { + "action_attribution_windows": { + "description": "[Private Preview] (Optional) Action attribution windows for insights reporting (e.g. \"28d_click\", \"1d_view\")", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "action_breakdowns": { + "description": "[Private Preview] (Optional) Action breakdowns to configure for data aggregation", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "action_report_time": { + "description": "[Private Preview] (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime)", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "breakdowns": { + "description": "[Private Preview] (Optional) Breakdowns to configure for data aggregation", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "level": { + "description": "[Private Preview] (Optional) Granularity of data to pull (account, ad, adset, campaign)", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "time_increment": { + "description": "[Private Preview] (Optional) Value in string by which to aggregate statistics (all_days, monthly or number of days)", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.NotebookLibrary": { "oneOf": [ { @@ -10745,6 +11290,66 @@ } ] }, + "pipelines.RedditAdsOptions": { + "oneOf": [ + { + "type": "object", + "description": "Reddit Ads specific options for ingestion", + "properties": { + "custom_report_options": { + "description": "[Private Preview] (Optional) Custom report definition. When set, the table is treated as a\nuser-defined Reddit Ads custom report. When unset, the table must match\none of the connector's prebuilt sources.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RedditAdsOptionsRedditAdsCustomReportOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "lookback_window_days": { + "description": "[Private Preview] (Optional) Number of days to look back for report tables during incremental sync\nto capture late-arriving conversions and attribution data.\nIf not specified, defaults to 30 days.", + "$ref": "#/$defs/int", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "sync_start_date": { + "description": "[Private Preview] (Optional) Start date for the initial sync of report tables in YYYY-MM-DD format.\nThis determines the earliest date from which to sync historical data.\nIf not specified, defaults to 2 years ago.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.RedditAdsOptionsRedditAdsCustomReportOptions": { + "oneOf": [ + { + "type": "object", + "description": "User-defined custom report for the Reddit Ads connector.\nApplies only to the custom_report table — prebuilt tables ignore this.", + "properties": { + "breakdowns": { + "description": "[Private Preview] (Optional) Breakdown dimensions to group report data by.\nExamples: CAMPAIGN_ID, DATE, COUNTRY, REGION, AD_ID.\nMust include at least one time dimension (DATE or HOUR).", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "fields": { + "description": "[Private Preview] (Optional) Fields to include in the report (maps to the Reddit Ads API `fields` parameter).\nExamples: IMPRESSIONS, CLICKS, SPEND, CPC, CTR.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.ReportSpec": { "oneOf": [ { @@ -10859,6 +11464,12 @@ "description": "[Public Preview] Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.", "$ref": "#/$defs/string" }, + "fanout_options": { + "description": "[Private Preview] Fanout options for multi-table routing from streaming sources.\nWhen set, records are routed to destination tables based on a\nper-record routing key. The key value becomes the table name:\n{destination_catalog}.{destination_schema}.{key_value}.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinitionFanoutOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "source_catalog": { "description": "[Public Preview] The source catalog name. Might be optional depending on the type of source.", "$ref": "#/$defs/string" @@ -11110,6 +11721,12 @@ "description": "[Public Preview] The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order.", "$ref": "#/$defs/slice/string" }, + "source_metadata_column": { + "description": "[Private Preview] (Optional) Name of the struct column added to each ingested record to hold per row source\nmetadata.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "table_properties": { "description": "[Beta] Table properties to set on the destination table.\nThese are key-value pairs that configure various Delta table behaviors or any user defined properties.\nExample: {\"delta.feature.variantType\": \"supported\", \"delta.enableTypeWidening\": \"true\"}\nNote: table_properties in table specific configuration will override the table_properties of the pipeline definition.", "$ref": "#/$defs/map/string" @@ -11157,6 +11774,12 @@ "type": "object", "description": "TikTok Ads specific options for ingestion", "properties": { + "custom_report_options": { + "description": "[Private Preview] (Optional) Custom report definition. When set, the table is treated as a\nuser-defined TikTok Ads custom report: the connector synthesizes a report\nrequest from the dimensions, metrics, report type, and data level specified\nhere. Supersedes the deprecated top-level dimensions/metrics/report_type/\ndata_level/query_lifetime fields above.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "data_level": { "description": "[Private Preview] Deprecated. Use custom_report_options.data_level instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TikTokAdsOptionsTikTokDataLevel", @@ -11218,6 +11841,51 @@ } ] }, + "pipelines.TikTokAdsOptionsTikTokAdsCustomReportOptions": { + "oneOf": [ + { + "type": "object", + "description": "User-defined custom report for the TikTok Ads connector. Groups the\ndimensions + metrics + report type + data level that define a TikTok Ads\ncustom report request.", + "properties": { + "data_level": { + "description": "[Private Preview] (Optional) Data level for the report.\nIf not specified, defaults to AUCTION_CAMPAIGN.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TikTokAdsOptionsTikTokDataLevel", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "dimensions": { + "description": "[Private Preview] (Optional) Dimensions to include in the report (e.g. \"campaign_id\",\n\"adgroup_id\", \"ad_id\", \"stat_time_day\", \"stat_time_hour\").", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "metrics": { + "description": "[Private Preview] (Optional) Metrics to include in the report (e.g. \"spend\", \"impressions\",\n\"clicks\", \"conversion\", \"cpc\").", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "query_lifetime": { + "description": "[Private Preview] (Optional) Whether to request lifetime metrics (all-time aggregated data).\nWhen true, the report returns all-time data.\nIf not specified, defaults to false.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "report_type": { + "description": "[Private Preview] (Optional) Report type for the TikTok Ads API.\nIf not specified, defaults to BASIC.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TikTokAdsOptionsTikTokReportType", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.TikTokAdsOptionsTikTokDataLevel": { "oneOf": [ { @@ -11277,16 +11945,12 @@ "description": "Specifies how to transform binary data into structured data.", "properties": { "format": { - "description": "[Private Preview] Required: the wire format of the data.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TransformerFormat", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta] Required: the wire format of the data.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TransformerFormat" }, "json_options": { - "description": "[Private Preview]", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JsonTransformerOptions", - "x-databricks-launch-stage": "PRIVATE_PREVIEW", - "doNotSuggest": true + "description": "[Beta]", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JsonTransformerOptions" } }, "additionalProperties": false @@ -11306,8 +11970,8 @@ "JSON" ], "enumDescriptions": [ - "[Private Preview]", - "[Private Preview]" + "[Beta]", + "[Beta]" ] }, { @@ -11578,6 +12242,67 @@ } ] }, + "postgres.SyncedTableSyncedTableSpecExtraColumn": { + "oneOf": [ + { + "type": "object", + "description": "An extra PostgreSQL column to add to the synced table.", + "properties": { + "column_name": { + "description": "[Private Preview] Name of the column.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "column_type": { + "description": "[Private Preview] PostgreSQL type of the column, for example \"tsvector\" or \"vector(1024)\".", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "compute": { + "description": "[Private Preview] SQL expression used to compute the column's value, for example\n\"to_tsvector('english', content)\".", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "maintenance": { + "description": "[Private Preview] How the column's value is populated and kept up to date.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecExtraColumnMaintenance", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "column_name", + "column_type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "postgres.SyncedTableSyncedTableSpecExtraColumnMaintenance": { + "oneOf": [ + { + "type": "string", + "description": "How the column's value is populated and kept up to date.", + "enum": [ + "STORED_GENERATED" + ], + "enumDescriptions": [ + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.SyncedTableSyncedTableSpecPgSpecificType": { "oneOf": [ { @@ -11633,7 +12358,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecPgSpecificType" }, "size": { - "description": "[Beta] Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR\nor PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).", + "description": "[Beta] Size parameter for the target type, for types that take one (e.g. vector\ndimension, varchar length). Required when the chosen pg_type needs a size.", "$ref": "#/$defs/int" } }, @@ -12827,6 +13552,14 @@ "inference_table_config": { "description": "[Public Preview] Configuration for inference table payload logging, including sampling.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TelemetryInferenceTableConfig" + }, + "table_names": { + "description": "[Public Preview] The Unity Catalog tables to which endpoint telemetry (logs, traces, and metrics) is exported.\nProvide this to create a new telemetry profile for the endpoint from the given tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.UnityCatalogTableNames" + }, + "telemetry_profile_id": { + "description": "[Public Preview] The ID of an existing telemetry profile to apply to this endpoint. Provide this to reuse a\ntelemetry profile that has already been created, instead of specifying table_names.", + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -12874,6 +13607,36 @@ } ] }, + "serving.UnityCatalogTableNames": { + "oneOf": [ + { + "type": "object", + "properties": { + "annotations_table": { + "description": "[Public Preview] The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported annotations.", + "$ref": "#/$defs/string" + }, + "logs_table": { + "description": "[Public Preview] The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported logs.", + "$ref": "#/$defs/string" + }, + "metrics_table": { + "description": "[Public Preview] The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported metrics.", + "$ref": "#/$defs/string" + }, + "traces_table": { + "description": "[Public Preview] The full three-level Unity Catalog name (catalog.schema.table) of the table that receives\nexported traces (spans).", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "sql.Aggregation": { "oneOf": [ { @@ -12887,16 +13650,6 @@ "MIN", "MAX", "STDDEV" - ], - "enumDescriptions": [ - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]" ] }, { @@ -12915,12 +13668,6 @@ "TRIGGERED", "OK", "ERROR" - ], - "enumDescriptions": [ - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]" ] }, { @@ -12936,10 +13683,42 @@ "enum": [ "ACTIVE", "DELETED" - ], - "enumDescriptions": [ - "[Public Preview]", - "[Public Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.AlertStatementParameter": { + "oneOf": [ + { + "type": "object", + "description": "Redash-owned copy of the internal StatementParameter for the external AlertV2 API.\nThe internal `ordinal` and `args` fields are intentionally omitted: the public API\nsupports only flat, named scalar parameters; complex types (ARRAY, MAP, STRUCT) are\nnot supported. This mirrors SEA's public StatementParameter schema, see:\ncmdexec/sql-exec-api/proto/sql_exec_api_service.proto:763-779", + "properties": { + "name": { + "description": "[Private Preview] The name of the parameter, referenced in the query as `:name`.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "type": { + "description": "[Private Preview] The SQL data type of the parameter, e.g. STRING, INT, or DATE. Defaults to STRING. This is a\nstring rather than an enum because scalar subtypes such as DECIMAL(10, 4) cannot be enumerated.\nComplex types such as ARRAY, MAP, and STRUCT are not supported.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "value": { + "description": "[Private Preview] The bound value for the parameter, given as a string. If omitted, the value is interpreted as NULL.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "name" ] }, { @@ -12954,23 +13733,23 @@ "type": "object", "properties": { "comparison_operator": { - "description": "[Public Preview] Operator used for comparison in alert evaluation.", + "description": "Operator used for comparison in alert evaluation.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.ComparisonOperator" }, "empty_result_state": { - "description": "[Public Preview] Alert state if result is empty. Please avoid setting this field to be `UNKNOWN` because `UNKNOWN` state is planned to be deprecated.", + "description": "Alert state if result is empty. Please avoid setting this field to be `UNKNOWN` because `UNKNOWN` state is planned to be deprecated.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertEvaluationState" }, "notification": { - "description": "[Public Preview] User or Notification Destination to notify when alert is triggered.", + "description": "User or Notification Destination to notify when alert is triggered.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2Notification" }, "source": { - "description": "[Public Preview] Source column from result to use to evaluate alert", + "description": "Source column from result to use to evaluate alert", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2OperandColumn" }, "threshold": { - "description": "[Public Preview] Threshold to user for alert evaluation, can be a column or a value.", + "description": "Threshold to user for alert evaluation, can be a column or a value.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2Operand" } }, @@ -12992,15 +13771,14 @@ "type": "object", "properties": { "notify_on_ok": { - "description": "[Public Preview] Whether to notify alert subscribers when alert returns back to normal.", + "description": "Whether to notify alert subscribers when alert returns back to normal.", "$ref": "#/$defs/bool" }, "retrigger_seconds": { - "description": "[Public Preview] Number of seconds an alert waits after being triggered before it is allowed to send another notification.\nIf set to 0 or omitted, the alert will not send any further notifications after the first trigger\nSetting this value to 1 allows the alert to send a notification on every evaluation where the condition is met, effectively making it always retrigger for notification purposes.", + "description": "Number of seconds an alert waits after being triggered before it is allowed to send another notification.\nIf set to 0 or omitted, the alert will not send any further notifications after the first trigger\nSetting this value to 1 allows the alert to send a notification on every evaluation where the condition is met, effectively making it always retrigger for notification purposes.", "$ref": "#/$defs/int" }, "subscriptions": { - "description": "[Public Preview]", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/sql.AlertV2Subscription" } }, @@ -13018,11 +13796,9 @@ "type": "object", "properties": { "column": { - "description": "[Public Preview]", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2OperandColumn" }, "value": { - "description": "[Public Preview]", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertV2OperandValue" } }, @@ -13040,15 +13816,13 @@ "type": "object", "properties": { "aggregation": { - "description": "[Public Preview] If not set, the behavior is equivalent to using `First row` in the UI.", + "description": "If not set, the behavior is equivalent to using `First row` in the UI.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.Aggregation" }, "display": { - "description": "[Public Preview]", "$ref": "#/$defs/string" }, "name": { - "description": "[Public Preview]", "$ref": "#/$defs/string" } }, @@ -13069,15 +13843,12 @@ "type": "object", "properties": { "bool_value": { - "description": "[Public Preview]", "$ref": "#/$defs/bool" }, "double_value": { - "description": "[Public Preview]", "$ref": "#/$defs/float64" }, "string_value": { - "description": "[Public Preview]", "$ref": "#/$defs/string" } }, @@ -13095,11 +13866,11 @@ "type": "object", "properties": { "service_principal_name": { - "description": "[Public Preview] Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", "$ref": "#/$defs/string" }, "user_name": { - "description": "[Public Preview] The email of an active workspace user. Can only set this field to their own email.", + "description": "The email of an active workspace user. Can only set this field to their own email.", "$ref": "#/$defs/string" } }, @@ -13117,11 +13888,9 @@ "type": "object", "properties": { "destination_id": { - "description": "[Public Preview]", "$ref": "#/$defs/string" }, "user_email": { - "description": "[Public Preview]", "$ref": "#/$defs/string" } }, @@ -13184,16 +13953,6 @@ "LESS_THAN_OR_EQUAL", "IS_NULL", "IS_NOT_NULL" - ], - "enumDescriptions": [ - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]", - "[Public Preview]" ] }, { @@ -13224,15 +13983,15 @@ "type": "object", "properties": { "pause_status": { - "description": "[Public Preview] Indicate whether this schedule is paused or not.", + "description": "Indicate whether this schedule is paused or not.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.SchedulePauseStatus" }, "quartz_cron_schedule": { - "description": "[Public Preview] A cron expression using quartz syntax that specifies the schedule for this pipeline.\nShould use the quartz format described here: http://www.quartz-scheduler.org/documentation/quartz-2.1.7/tutorials/tutorial-lesson-06.html", + "description": "A cron expression using quartz syntax that specifies the schedule for this pipeline.\nShould use the quartz format described here: http://www.quartz-scheduler.org/documentation/quartz-2.1.7/tutorials/tutorial-lesson-06.html", "$ref": "#/$defs/string" }, "timezone_id": { - "description": "[Public Preview] A Java timezone id. The schedule will be resolved using this timezone.\nThis will be combined with the quartz_cron_schedule to determine the schedule.\nSee https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.", + "description": "A Java timezone id. The schedule will be resolved using this timezone.\nThis will be combined with the quartz_cron_schedule to determine the schedule.\nSee https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.", "$ref": "#/$defs/string" } }, @@ -13292,10 +14051,6 @@ "enum": [ "UNPAUSED", "PAUSED" - ], - "enumDescriptions": [ - "[Public Preview]", - "[Public Preview]" ] }, { @@ -13534,6 +14289,23 @@ } ] }, + "vectorsearch.VectorSearchEndpointPermissionLevel": { + "oneOf": [ + { + "type": "string", + "description": "Permission level", + "enum": [ + "CAN_CREATE", + "CAN_MANAGE", + "CAN_USE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "oneOf": [ { @@ -13768,6 +14540,20 @@ } ] }, + "resources.InstancePool": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.InstancePool" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -14228,6 +15014,20 @@ } ] }, + "resources.InstancePoolPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.InstancePoolPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.JobPermission": { "oneOf": [ { @@ -14339,6 +15139,20 @@ "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "resources.VectorSearchEndpointPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpointPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "config.ArtifactFile": { @@ -14457,6 +15271,20 @@ } ] }, + "compute.DockerImage": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "compute.InitScriptInfo": { "oneOf": [ { @@ -14849,6 +15677,20 @@ } ] }, + "pipelines.Transformer": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.ProjectCustomTag": { "oneOf": [ { @@ -14877,6 +15719,20 @@ } ] }, + "postgres.SyncedTableSyncedTableSpecExtraColumn": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecExtraColumn" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "postgres.SyncedTableSyncedTableSpecTypeOverride": { "oneOf": [ { @@ -14975,6 +15831,20 @@ } ] }, + "sql.AlertStatementParameter": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.AlertStatementParameter" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "sql.AlertV2Subscription": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 313348f30d2..1ec6dc82999 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -57,6 +57,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, + "resources.instance_pools.test_instance_pool": {ID: "1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -149,6 +150,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -378,6 +382,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + InstancePools: map[string]*resources.InstancePool{ + "test_instance_pool": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool", + }, + }, + }, }, } @@ -477,6 +488,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -856,6 +870,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + InstancePools: map[string]*resources.InstancePool{ + "test_instance_pool": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool", + }, + }, + "test_instance_pool_new": { + CreateInstancePool: compute.CreateInstancePool{ + InstancePoolName: "test_instance_pool_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -913,6 +939,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, + "resources.instance_pools.test_instance_pool": {ID: "1"}, + "resources.instance_pools.test_instance_pool_old": {ID: "2"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -1108,6 +1136,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.InstancePools["test_instance_pool_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool_old"].ModifiedStatus) + assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index e2b16bd9798..c0e72b66cc0 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -6,13 +6,13 @@ package terraform_dabs_map // alerts / databricks_alert_v2: 3 tf-only // apps / databricks_app: 16 dabs-only // apps / databricks_app: 1 tf-only -// clusters / databricks_cluster: 25 tf-only +// clusters / databricks_cluster: 26 tf-only // dashboards / databricks_dashboard: 2 tf-only // database_instances / databricks_database_instance: 1 tf-only // experiments / databricks_mlflow_experiment: 1 tf-only // jobs / databricks_job: 11 renames -// jobs / databricks_job: 16 dabs-only -// jobs / databricks_job: 258 tf-only +// jobs / databricks_job: 7 dabs-only +// jobs / databricks_job: 259 tf-only // model_serving_endpoints / databricks_model_serving: 2 tf-only // models / databricks_mlflow_model: 1 renames // pipelines / databricks_pipeline: 3 renames @@ -118,14 +118,6 @@ var DABsOnlyFields = map[string]FieldSet{ "autotermination_minutes": {}, // jobs.*.job_clusters.new_cluster.autotermination_minutes }, }, - "parent_path": {}, - "schedule": { - "sql_condition": { - "sql_query_id": {}, // jobs.*.schedule.sql_condition.sql_query_id - "trigger_mode": {}, // jobs.*.schedule.sql_condition.trigger_mode - "warehouse_id": {}, // jobs.*.schedule.sql_condition.warehouse_id - }, - }, "tasks": { "for_each_task": { "task": { @@ -143,13 +135,6 @@ var DABsOnlyFields = map[string]FieldSet{ "autotermination_minutes": {}, // jobs.*.tasks.new_cluster.autotermination_minutes }, }, - "trigger": { - "sql_condition": { - "sql_query_id": {}, // jobs.*.trigger.sql_condition.sql_query_id - "trigger_mode": {}, // jobs.*.trigger.sql_condition.trigger_mode - "warehouse_id": {}, // jobs.*.trigger.sql_condition.warehouse_id - }, - }, }, "pipelines": { "clusters": { @@ -183,6 +168,7 @@ var TerraformOnlyFields = map[string]FieldSet{ "no_compute": {}, }, "clusters": { + "clear_cloud_attributes_on_remove": {}, "cluster_mount_info": { "local_mount_dir_path": {}, // databricks_cluster.*.cluster_mount_info.local_mount_dir_path "network_filesystem_info": { @@ -352,6 +338,7 @@ var TerraformOnlyFields = map[string]FieldSet{ "*": {}, // databricks_job.*.new_cluster.custom_tags.* }, "data_security_mode": {}, // databricks_job.*.new_cluster.data_security_mode + "dependency_mode": {}, // databricks_job.*.new_cluster.dependency_mode "docker_image": { "basic_auth": { "password": {}, // databricks_job.*.new_cluster.docker_image.basic_auth.password @@ -690,6 +677,7 @@ var DABsToTerraformWrapperFields = map[string]FieldSet{ "branch": {}, "create_database_objects_if_missing": {}, "existing_pipeline_id": {}, + "extra_columns": {}, "new_pipeline_spec": {}, "postgres_database": {}, "primary_key_columns": {}, diff --git a/cmd/account/disaster-recovery/disaster-recovery.go b/cmd/account/disaster-recovery/disaster-recovery.go index f593144adb8..014927b2dba 100644 --- a/cmd/account/disaster-recovery/disaster-recovery.go +++ b/cmd/account/disaster-recovery/disaster-recovery.go @@ -22,18 +22,16 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "disaster-recovery", - Short: `*Public Preview* Manage disaster recovery configurations and execute failover operations.`, - Long: `This command is in Public Preview and may change without notice. - -Manage disaster recovery configurations and execute failover operations.`, + Use: "disaster-recovery", + Short: `Manage disaster recovery configurations and execute failover operations.`, + Long: `Manage disaster recovery configurations and execute failover operations.`, GroupID: "disasterrecovery", RunE: root.ReportUnknownSubcommand, } cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" // Add methods cmd.AddCommand(newCreateFailoverGroup()) @@ -78,10 +76,8 @@ func newCreateFailoverGroup() *cobra.Command { // TODO: complex arg: unity_catalog_assets cmd.Use = "create-failover-group PARENT FAILOVER_GROUP_ID REGIONS WORKSPACE_SETS INITIAL_PRIMARY_REGION" - cmd.Short = `*Public Preview* Create a Failover Group.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Create a Failover Group. + cmd.Short = `Create a Failover Group.` + cmd.Long = `Create a Failover Group. Create a new failover group. @@ -95,8 +91,8 @@ Create a Failover Group. primary region. Not returned in responses.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -189,10 +185,8 @@ func newCreateStableUrl() *cobra.Command { cmd.Flags().StringVar(&createStableUrlReq.StableUrl.Name, "name", createStableUrlReq.StableUrl.Name, `Fully qualified resource name.`) cmd.Use = "create-stable-url PARENT STABLE_URL_ID INITIAL_WORKSPACE_ID" - cmd.Short = `*Public Preview* Create a Stable URL.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Create a Stable URL. + cmd.Short = `Create a Stable URL.` + cmd.Long = `Create a Stable URL. Create a new stable URL. @@ -205,8 +199,8 @@ Create a Stable URL. responses.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -280,10 +274,8 @@ func newDeleteFailoverGroup() *cobra.Command { cmd.Flags().StringVar(&deleteFailoverGroupReq.Etag, "etag", deleteFailoverGroupReq.Etag, `Opaque version string for optimistic locking.`) cmd.Use = "delete-failover-group NAME" - cmd.Short = `*Public Preview* Delete a Failover Group.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Delete a Failover Group. + cmd.Short = `Delete a Failover Group.` + cmd.Long = `Delete a Failover Group. Delete a failover group. @@ -292,8 +284,8 @@ Delete a Failover Group. accounts/{account_id}/failover-groups/{failover_group_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -341,10 +333,8 @@ func newDeleteStableUrl() *cobra.Command { var deleteStableUrlReq disasterrecovery.DeleteStableUrlRequest cmd.Use = "delete-stable-url NAME" - cmd.Short = `*Public Preview* Delete a Stable URL.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Delete a Stable URL. + cmd.Short = `Delete a Stable URL.` + cmd.Long = `Delete a Stable URL. Delete a stable URL. @@ -353,8 +343,8 @@ Delete a Stable URL. accounts/{account_id}/stable-urls/{stable_url_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -407,10 +397,8 @@ func newFailoverFailoverGroup() *cobra.Command { cmd.Flags().StringVar(&failoverFailoverGroupReq.Etag, "etag", failoverFailoverGroupReq.Etag, `Opaque version string for optimistic locking.`) cmd.Use = "failover-failover-group NAME TARGET_PRIMARY_REGION FAILOVER_TYPE" - cmd.Short = `*Public Preview* Failover a Failover Group to a new primary region.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Failover a Failover Group to a new primary region. + cmd.Short = `Failover a Failover Group to a new primary region.` + cmd.Long = `Failover a Failover Group to a new primary region. Initiate a failover to a new primary region. @@ -424,8 +412,8 @@ Failover a Failover Group to a new primary region. Supported values: [FORCED]` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -503,10 +491,8 @@ func newGetFailoverGroup() *cobra.Command { var getFailoverGroupReq disasterrecovery.GetFailoverGroupRequest cmd.Use = "get-failover-group NAME" - cmd.Short = `*Public Preview* Get a Failover Group.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Get a Failover Group. + cmd.Short = `Get a Failover Group.` + cmd.Long = `Get a Failover Group. Get a failover group. @@ -515,8 +501,8 @@ Get a Failover Group. accounts/{account_id}/failover-groups/{failover_group_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -565,10 +551,8 @@ func newGetStableUrl() *cobra.Command { var getStableUrlReq disasterrecovery.GetStableUrlRequest cmd.Use = "get-stable-url NAME" - cmd.Short = `*Public Preview* Get a Stable URL.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Get a Stable URL. + cmd.Short = `Get a Stable URL.` + cmd.Long = `Get a Stable URL. Get a stable URL. @@ -577,8 +561,8 @@ Get a Stable URL. accounts/{account_id}/stable-urls/{stable_url_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -640,10 +624,8 @@ func newListFailoverGroups() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-failover-groups PARENT" - cmd.Short = `*Public Preview* List Failover Groups.` - cmd.Long = `This command is in Public Preview and may change without notice. - -List Failover Groups. + cmd.Short = `List Failover Groups.` + cmd.Long = `List Failover Groups. List failover groups. @@ -654,8 +636,8 @@ List Failover Groups. PARENT: The parent resource. Format: accounts/{account_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -720,10 +702,8 @@ func newListStableUrls() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-stable-urls PARENT" - cmd.Short = `*Public Preview* List Stable URLs.` - cmd.Long = `This command is in Public Preview and may change without notice. - -List Stable URLs. + cmd.Short = `List Stable URLs.` + cmd.Long = `List Stable URLs. List stable URLs for an account. @@ -731,8 +711,8 @@ List Stable URLs. PARENT: The parent resource. Format: accounts/{account_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -792,10 +772,8 @@ func newUpdateFailoverGroup() *cobra.Command { // TODO: complex arg: unity_catalog_assets cmd.Use = "update-failover-group NAME UPDATE_MASK REGIONS WORKSPACE_SETS INITIAL_PRIMARY_REGION" - cmd.Short = `*Public Preview* Update a Failover Group.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Update a Failover Group. + cmd.Short = `Update a Failover Group.` + cmd.Long = `Update a Failover Group. Update a failover group. @@ -809,8 +787,8 @@ Update a Failover Group. primary region. Not returned in responses.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { diff --git a/cmd/account/endpoints/endpoints.go b/cmd/account/endpoints/endpoints.go index dff9c2344bf..8cdb1b5c73f 100644 --- a/cmd/account/endpoints/endpoints.go +++ b/cmd/account/endpoints/endpoints.go @@ -63,7 +63,9 @@ func newCreateEndpoint() *cobra.Command { cmd.Flags().Var(&createEndpointJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: aws_vpc_endpoint_info // TODO: complex arg: azure_private_endpoint_info + // TODO: complex arg: gcp_psc_endpoint_info cmd.Use = "create-endpoint PARENT DISPLAY_NAME REGION" cmd.Short = `Create a network endpoint.` diff --git a/cmd/account/iam-v2/iam-v2.go b/cmd/account/iam-v2/iam-v2.go index 7de37b8ca7b..a87ef08b240 100644 --- a/cmd/account/iam-v2/iam-v2.go +++ b/cmd/account/iam-v2/iam-v2.go @@ -73,6 +73,7 @@ func newCreateWorkspaceAssignmentDetail() *cobra.Command { cmd.Flags().Var(&createWorkspaceAssignmentDetailJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: effective_entitlements // TODO: array: entitlements cmd.Use = "create-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID" @@ -738,6 +739,7 @@ func newUpdateWorkspaceAssignmentDetail() *cobra.Command { cmd.Flags().Var(&updateWorkspaceAssignmentDetailJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: effective_entitlements // TODO: array: entitlements cmd.Use = "update-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID UPDATE_MASK PRINCIPAL_ID" diff --git a/cmd/aitools/install.go b/cmd/aitools/install.go index 033611da8bc..c6618a43269 100644 --- a/cmd/aitools/install.go +++ b/cmd/aitools/install.go @@ -18,6 +18,7 @@ import ( // install_test.go. var ( promptAgentSelection = defaultPromptAgentSelection + promptProceed = defaultPromptProceed installSkillsForAgentsFn = installer.InstallSkillsForAgents installPluginForAgentFn = installer.InstallPluginForAgent recordPluginInstallsFn = installer.RecordPluginInstalls @@ -142,7 +143,7 @@ Supported agents: Claude Code, Cursor, Codex CLI, OpenCode, GitHub Copilot, Anti // In the interactive picker path, show a plan summary and confirm. if !explicit && cmdio.IsPromptSupported(ctx) { printPlanSummary(ctx, plan, scope) - proceed, err := cmdio.AskYesOrNo(ctx, "Proceed?") + proceed, err := promptProceed() if err != nil { return err } @@ -254,6 +255,18 @@ func agentStateLabel(s agents.DisplayState) string { } } +func defaultPromptProceed() (bool, error) { + proceed := true + err := huh.NewConfirm(). + Title("Proceed?"). + Value(&proceed). + Run() + if err != nil { + return false, err + } + return proceed, nil +} + func defaultPromptAgentSelection(_ context.Context, choices []agentChoice) ([]*agents.Agent, error) { options := make([]huh.Option[string], 0, len(choices)) byName := make(map[string]*agents.Agent, len(choices)) diff --git a/cmd/aitools/install_test.go b/cmd/aitools/install_test.go index ec70e36772f..825ce1ced6c 100644 --- a/cmd/aitools/install_test.go +++ b/cmd/aitools/install_test.go @@ -322,6 +322,16 @@ func TestInstallInteractivePickerAndConfirm(t *testing.T) { return nil, nil } + // The confirm is a huh widget that reads the real terminal, so drive it via + // the override rather than piped stdin; assert it was consulted. + origProceed := promptProceed + t.Cleanup(func() { promptProceed = origProceed }) + proceedCalled := false + promptProceed = func() (bool, error) { + proceedCalled = true + return true, nil + } + ctx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) defer test.Done() go drainReader(test.Stdout) @@ -330,15 +340,9 @@ func TestInstallInteractivePickerAndConfirm(t *testing.T) { cmd := NewInstallCmd() cmd.SetContext(telemetry.WithNewLogger(ctx)) - errc := make(chan error, 1) - go func() { errc <- cmd.RunE(cmd, nil) }() - - _, err := test.Stdin.WriteString("y\n") - require.NoError(t, err) - require.NoError(t, test.Stdin.Flush()) - - require.NoError(t, <-errc) + require.NoError(t, cmd.RunE(cmd, nil)) assert.True(t, pickerCalled) + assert.True(t, proceedCalled) require.Len(t, *plugins, 1) assert.Equal(t, agents.NameClaudeCode, (*plugins)[0].agent) } diff --git a/cmd/auth/profiles.go b/cmd/auth/profiles.go index 8c317293322..e65b11371b7 100644 --- a/cmd/auth/profiles.go +++ b/cmd/auth/profiles.go @@ -21,6 +21,13 @@ import ( "gopkg.in/ini.v1" ) +// profileValidationTimeout bounds each per-profile validation so a host the SDK +// retries — connection refused, connect/TLS timeout, retriable 5xx — cannot +// stall the whole listing. Without it a single such host blocks `auth profiles` +// for the SDK's default retry budget (~5 minutes). Hosts that fail DNS are not +// retriable and already fail fast, so this only bounds the retriable cases. +const profileValidationTimeout = 5 * time.Second + type profileMetadata struct { Name string `json:"name"` Host string `json:"host,omitempty"` @@ -36,12 +43,22 @@ func (c *profileMetadata) IsEmpty() bool { return c.Host == "" && c.AccountID == "" } -func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipValidate bool) { +func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipValidate bool, timeout time.Duration) { + timeoutSeconds := int(timeout / time.Second) cfg := &config.Config{ Loaders: []config.Loader{config.ConfigFile}, ConfigFile: configFilePath, Profile: c.Name, DatabricksCliPath: env.Get(ctx, "DATABRICKS_CLI_PATH"), + + // Bound the SDK's per-request and total-retry budgets to the same + // per-profile ceiling. EnsureResolved fetches host metadata via the + // SDK's retrier, which defaults to 5 minutes — and it runs on + // context.Background internally, so the context.WithTimeout below on the + // validation call cannot reach it. Without these a single unreachable + // host stalls the listing well past the validation timeout. + HTTPTimeoutSeconds: timeoutSeconds, + RetryTimeoutSeconds: timeoutSeconds, } if skipValidate { // EnsureResolved fetches /.well-known/databricks-config to enrich @@ -72,13 +89,16 @@ func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipV log.Debugf(ctx, "Profile %q: overrode config type from %s to %s (SPOG host)", c.Name, cfg.ConfigType(), configType) } + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + switch configType { case config.AccountConfig: a, err := databricks.NewAccountClient((*databricks.Config)(cfg)) if err != nil { return } - _, err = a.Workspaces.List(ctx) + _, err = a.Workspaces.List(callCtx) c.Host = cfg.Host c.AuthType = cfg.AuthType if err != nil { @@ -90,7 +110,7 @@ func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipV if err != nil { return } - _, err = w.CurrentUser.Me(ctx, iam.MeRequest{}) + _, err = w.CurrentUser.Me(callCtx, iam.MeRequest{}) c.Host = cfg.Host c.AuthType = cfg.AuthType if err != nil { @@ -148,7 +168,7 @@ func newProfilesCommand() *cobra.Command { wg.Go(func() { ctx := cmd.Context() t := time.Now() - profile.Load(ctx, iniFile.Path(), skipValidate) + profile.Load(ctx, iniFile.Path(), skipValidate, profileValidationTimeout) log.Debugf(ctx, "Profile %q took %s to load", profile.Name, time.Since(t)) }) profiles = append(profiles, profile) diff --git a/cmd/auth/profiles_test.go b/cmd/auth/profiles_test.go index 2ebbeed0e60..90efa2a99bd 100644 --- a/cmd/auth/profiles_test.go +++ b/cmd/auth/profiles_test.go @@ -9,6 +9,7 @@ import ( "runtime" "sync/atomic" "testing" + "time" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/databricks-sdk-go/config" @@ -40,7 +41,7 @@ func TestProfiles(t *testing.T) { // Load the profile profile := &profileMetadata{Name: "profile1"} - profile.Load(ctx, configFile, true) + profile.Load(ctx, configFile, true, profileValidationTimeout) // Check the profile assert.Equal(t, "profile1", profile.Name) @@ -71,7 +72,7 @@ func TestProfileLoadSkipValidateMakesNoRequests(t *testing.T) { require.NoError(t, os.WriteFile(configFile, []byte(content), 0o600)) p := &profileMetadata{Name: "offline-profile", Host: server.URL} - p.Load(t.Context(), configFile, true) + p.Load(t.Context(), configFile, true, profileValidationTimeout) assert.Zero(t, requests.Load(), "expected no network calls with skipValidate") assert.Equal(t, server.URL, p.Host) @@ -222,7 +223,7 @@ func TestProfileLoadSPOGConfigType(t *testing.T) { Host: tc.host, AccountID: tc.accountID, } - p.Load(t.Context(), configFile, false) + p.Load(t.Context(), configFile, false, profileValidationTimeout) assert.Equal(t, tc.wantValid, p.Valid, "Valid mismatch") assert.NotEmpty(t, p.Host, "Host should be set") @@ -278,9 +279,51 @@ func TestProfileLoadNoDiscoveryStaysWorkspace(t *testing.T) { Host: server.URL, AccountID: "some-acct", } - p.Load(t.Context(), configFile, false) + p.Load(t.Context(), configFile, false, profileValidationTimeout) assert.True(t, p.Valid, "should validate as workspace when discovery is unavailable") assert.NotEmpty(t, p.Host) assert.Equal(t, "pat", p.AuthType) } + +// TestProfileLoadTimesOutOnUnresponsiveHost is a regression test for a hang: +// the SDK retries transient network failures for ~5 minutes by default, so a +// host that accepts the connection but never responds would stall the whole +// `auth profiles` listing. The timeout passed to Load must bound the wait +// (including the EnsureResolved metadata fetch, which runs on context.Background +// and so is only reachable via HTTPTimeoutSeconds/RetryTimeoutSeconds). +// +// A 2s timeout is passed here — kept >=1s because Load derives the SDK's +// integer-second budgets from it, and a sub-second value would floor to 0 +// (which the SDK reads as "use the default", defeating the test). +func TestProfileLoadTimesOutOnUnresponsiveHost(t *testing.T) { + // A server that hangs every request until the client gives up. Waiting on + // the request context (rather than a private channel) means the handler + // returns as soon as our timeout cancels the call, so server.Close doesn't + // block on a leaked connection during cleanup. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + t.Cleanup(server.Close) + + dir := t.TempDir() + configFile := filepath.Join(dir, ".databrickscfg") + t.Setenv("HOME", dir) + if runtime.GOOS == "windows" { + t.Setenv("USERPROFILE", dir) + } + content := "[hung]\nhost = " + server.URL + "\ntoken = test-token\n" + require.NoError(t, os.WriteFile(configFile, []byte(content), 0o600)) + + p := &profileMetadata{Name: "hung", Host: server.URL} + start := time.Now() + p.Load(t.Context(), configFile, false, 2*time.Second) + elapsed := time.Since(start) + + assert.False(t, p.Valid, "an unresponsive host must not validate") + // Generously above the 2s bound (metadata fetch + validation call, each + // bounded) but far below the SDK's ~5m default that this timeout prevents. + // A regression that drops the bound would blow past this and the whole + // package would then hit the go test -timeout instead. + assert.Less(t, elapsed, 60*time.Second, "Load must be bounded by profileValidationTimeout, not the SDK default") +} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 93d9bbb14c3..e5231fef58a 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -104,7 +104,7 @@ Examples: // Filter after planning, never before: the plan must cover every // resource so ${resources.*} references resolve; only the emitted // changes are restricted to the selected resources. - selected, err := configsync.ResolveResourceSelectors(&deployBundle.StateDB, selectIDs) + selected, err := configsync.ResolveResourceSelectors(ctx, &deployBundle.StateDB, selectIDs) if err != nil { return err } diff --git a/cmd/cmd.go b/cmd/cmd.go index bcd81619fca..a2c3280b940 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -18,7 +18,6 @@ import ( "github.com/databricks/cli/cmd/experimental" "github.com/databricks/cli/cmd/fs" "github.com/databricks/cli/cmd/labs" - "github.com/databricks/cli/cmd/localenv" "github.com/databricks/cli/cmd/pipelines" "github.com/databricks/cli/cmd/quickstart" "github.com/databricks/cli/cmd/root" @@ -114,7 +113,6 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(cache.New()) cli.AddCommand(experimental.New()) cli.AddCommand(psql.New()) - cli.AddCommand(localenv.New()) cli.AddCommand(configure.New()) cli.AddCommand(fs.New()) cli.AddCommand(labs.New(ctx)) diff --git a/cmd/environments/compute.go b/cmd/environments/compute.go new file mode 100644 index 00000000000..65c69825072 --- /dev/null +++ b/cmd/environments/compute.go @@ -0,0 +1,202 @@ +package environments + +import ( + "context" + "fmt" + "strconv" + "strings" + + localenv "github.com/databricks/cli/libs/localenv" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// sdkCompute adapts the Databricks SDK to the localenv.ComputeClient interface. +type sdkCompute struct { + w *databricks.WorkspaceClient +} + +// GetClusterSparkVersion returns the Spark version string for a running cluster. +func (c sdkCompute) GetClusterSparkVersion(ctx context.Context, clusterID string) (string, error) { + d, err := c.w.Clusters.GetByClusterId(ctx, clusterID) + if err != nil { + return "", fmt.Errorf("get cluster %s: %w", clusterID, err) + } + return d.SparkVersion, nil +} + +// GetClusterByName resolves a cluster name to its ID and Spark version. +// +// It intentionally does not use the SDK's GetByClusterName, which lists clusters +// in every state and errors on any name collision: a terminated cluster sharing +// a name with a live one would then block resolution for a name the user +// reasonably considers unique. Instead we list only non-terminated clusters and +// match by name. A name that is still ambiguous among live clusters, or matches +// none, is a genuine error surfaced to the caller as an actionable E_RESOLVE. +// +// The clusters/list API omits clusters terminated more than 30 days ago, so a +// name that only ever belonged to such a cluster resolves as "no active cluster +// named"; this is acceptable since a long-terminated cluster is not a usable +// target and the user can always pass --cluster-id. +func (c sdkCompute) GetClusterByName(ctx context.Context, name string) (string, string, error) { + clusters, err := c.w.Clusters.ListAll(ctx, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + // TERMINATING is excluded alongside TERMINATED: a cluster on its way + // down is not a usable target, and keeping it would reintroduce the + // stale-collision problem this filter exists to avoid. + ClusterStates: []compute.State{ + compute.StatePending, + compute.StateRunning, + compute.StateRestarting, + compute.StateResizing, + }, + }, + }) + if err != nil { + return "", "", fmt.Errorf("list clusters to resolve name %q: %w", name, err) + } + var matches []compute.ClusterDetails + for _, cl := range clusters { + if cl.ClusterName == name { + matches = append(matches, cl) + } + } + switch len(matches) { + case 0: + return "", "", fmt.Errorf("no active cluster named %q", name) + case 1: + return matches[0].ClusterId, matches[0].SparkVersion, nil + default: + return "", "", fmt.Errorf("there are %d active clusters named %q; use --cluster-id to disambiguate", len(matches), name) + } +} + +// JobTaskEnvironment resolves a single job task's compute to an environment. +// +// A job can bind multiple tasks to different serverless environment versions, so +// the target is a specific task, not the whole job. taskKey selects it: +// - an empty taskKey is an enumerate request — it returns *localenv.ErrTaskKeyRequired +// carrying the job's task keys, so the caller emits an actionable E_USAGE; +// - an unknown taskKey returns an error listing the available keys. +// +// A serverless task binds an environment_key to one of the job's environments; +// its version is read directly from that environment's spec (no fallback). A +// classic task resolves from its new_cluster, its job_cluster_key (a shared +// job_clusters entry), or its existing_cluster_id (via the Clusters API). A +// for_each_task is unwrapped to its nested task, whose compute is resolved the +// same way. +func (c sdkCompute) JobTaskEnvironment(ctx context.Context, jobID, taskKey string) (sparkVersion string, isServerless bool, version string, err error) { + id, err := strconv.ParseInt(jobID, 10, 64) + if err != nil { + return "", false, "", fmt.Errorf("invalid job ID %q: must be an integer: %w", jobID, err) + } + + job, err := c.w.Jobs.GetByJobId(ctx, id) + if err != nil { + return "", false, "", fmt.Errorf("get job %d: %w", id, err) + } + if job.Settings == nil { + return "", false, "", fmt.Errorf("job %d has no settings", id) + } + + taskKeys := make([]string, 0, len(job.Settings.Tasks)) + for _, t := range job.Settings.Tasks { + taskKeys = append(taskKeys, t.TaskKey) + } + + // No task key given: ask the caller to pick one, listing what is available. + // This is a usage error, not a resolve failure — see ResolveTarget. + if taskKey == "" { + return "", false, "", &localenv.ErrTaskKeyRequired{JobID: jobID, TaskKeys: taskKeys} + } + + var task *jobs.Task + for i := range job.Settings.Tasks { + if job.Settings.Tasks[i].TaskKey == taskKey { + task = &job.Settings.Tasks[i] + break + } + } + if task == nil { + return "", false, "", fmt.Errorf("job %s has no task %q (available: %s)", jobID, taskKey, strings.Join(taskKeys, ", ")) + } + + // A for_each_task wraps the real per-iteration task: its compute + // (environment_key / new_cluster / existing_cluster_id / job_cluster_key) + // lives on the nested task, not the outer one. Resolve against that. + if task.ForEachTask != nil { + task = &task.ForEachTask.Task + } + + return c.resolveTaskCompute(ctx, job.Settings, task, jobID, taskKey) +} + +// resolveTaskCompute resolves one task's compute to an environment. It reads the +// serverless environment version directly (no fallback) or the classic cluster's +// Spark version, consulting the job-level environments and job_clusters the task +// references by key. +func (c sdkCompute) resolveTaskCompute(ctx context.Context, settings *jobs.JobSettings, task *jobs.Task, jobID, taskKey string) (sparkVersion string, isServerless bool, version string, err error) { + // Serverless task: it references one of the job's environments by key; the + // version lives on that environment's spec and is used directly. + if task.EnvironmentKey != "" { + for _, e := range settings.Environments { + if e.EnvironmentKey == task.EnvironmentKey { + v := environmentVersion(e) + if v == "" { + return "", false, "", fmt.Errorf("task %q of job %s binds environment %q, which records no environment version", taskKey, jobID, task.EnvironmentKey) + } + return "", true, v, nil + } + } + return "", false, "", fmt.Errorf("task %q of job %s references environment %q, which the job does not define", taskKey, jobID, task.EnvironmentKey) + } + + // Classic task with an inline cluster spec. + if task.NewCluster != nil && task.NewCluster.SparkVersion != "" { + return task.NewCluster.SparkVersion, false, task.NewCluster.SparkVersion, nil + } + + // Classic task referencing a shared job cluster by key: read that cluster's + // Spark version from the job's job_clusters (the common classic-task shape). + if task.JobClusterKey != "" { + for _, jc := range settings.JobClusters { + if jc.JobClusterKey == task.JobClusterKey { + if jc.NewCluster.SparkVersion == "" { + return "", false, "", fmt.Errorf("task %q of job %s uses job cluster %q, which has no spark_version", taskKey, jobID, task.JobClusterKey) + } + return jc.NewCluster.SparkVersion, false, jc.NewCluster.SparkVersion, nil + } + } + return "", false, "", fmt.Errorf("task %q of job %s references job cluster %q, which the job does not define", taskKey, jobID, task.JobClusterKey) + } + + // Classic task pinned to an existing cluster: resolve its Spark version. + if task.ExistingClusterId != "" { + sv, cerr := c.GetClusterSparkVersion(ctx, task.ExistingClusterId) + if cerr != nil { + return "", false, "", fmt.Errorf("resolving existing cluster %s for task %q of job %s: %w", task.ExistingClusterId, taskKey, jobID, cerr) + } + return sv, false, sv, nil + } + + return "", false, "", fmt.Errorf("task %q of job %s has no serverless environment or resolvable cluster; pass --cluster-id or --serverless-version explicitly", taskKey, jobID) +} + +// environmentVersion returns the serverless environment version recorded on a +// job environment, or "" when the spec or version is absent. +// +// The version can arrive in either of two fields. environment_version is the +// current one; client is its deprecated predecessor ("Use environment_version +// instead") and is still what some jobs pin. Reading both means a client-pinned +// task is not treated as unversioned. base_environment is deliberately ignored: +// it is a path/ID, not a version. +func environmentVersion(e jobs.JobEnvironment) string { + if e.Spec == nil { + return "" + } + if e.Spec.EnvironmentVersion != "" { + return e.Spec.EnvironmentVersion + } + return e.Spec.Client +} diff --git a/cmd/localenv/compute_test.go b/cmd/environments/compute_test.go similarity index 98% rename from cmd/localenv/compute_test.go rename to cmd/environments/compute_test.go index 8620821a8f3..7b4b9300b75 100644 --- a/cmd/localenv/compute_test.go +++ b/cmd/environments/compute_test.go @@ -1,4 +1,4 @@ -package localenv +package environments import ( "testing" diff --git a/cmd/localenv/output.go b/cmd/environments/output.go similarity index 99% rename from cmd/localenv/output.go rename to cmd/environments/output.go index 85152462e26..85d60d5c235 100644 --- a/cmd/localenv/output.go +++ b/cmd/environments/output.go @@ -1,4 +1,4 @@ -package localenv +package environments import ( "context" diff --git a/cmd/environments/setup_local.go b/cmd/environments/setup_local.go new file mode 100644 index 00000000000..dee8292bb5f --- /dev/null +++ b/cmd/environments/setup_local.go @@ -0,0 +1,18 @@ +package environments + +import "github.com/spf13/cobra" + +// Commands returns the hand-written subcommands to add to the auto-generated +// "environments" command group. They are attached via the group's cmdOverrides +// hook (see cmd/workspace/environments/overrides.go), mirroring how cmd/apps +// extends the generated apps group. +// +// P0 exposes a single verb, "setup-local", which provisions a local Python +// environment matched to a Databricks compute target. It is Python-only and +// takes no language selector (spec §naming); a language axis (setup-local +// python / scala) would be additive if more languages are ever supported. +func Commands() []*cobra.Command { + return []*cobra.Command{ + newSetupLocalCommand(), + } +} diff --git a/cmd/localenv/sync.go b/cmd/environments/sync.go similarity index 67% rename from cmd/localenv/sync.go rename to cmd/environments/sync.go index 28146278012..3c0bf2880fc 100644 --- a/cmd/localenv/sync.go +++ b/cmd/environments/sync.go @@ -1,4 +1,4 @@ -package localenv +package environments import ( "context" @@ -18,7 +18,7 @@ import ( // libslocalenv.RepoConstraintBaseURL (which reads its own repo env var). const envConstraintSource = "DATABRICKS_LOCALENV_CONSTRAINT_SOURCE" -func newSyncCommand() *cobra.Command { +func newSetupLocalCommand() *cobra.Command { cmd := &cobra.Command{ Use: libslocalenv.CommandVerb, Short: "Provision a local Python environment matched to a Databricks compute target", @@ -29,6 +29,10 @@ databricks-connect version, and dependency constraints published for that key, then provisions a matched .venv with uv. A project with no pyproject.toml is initialized from scratch; an existing pyproject.toml is merged in place (its env-owned sections are refreshed, user-owned content is preserved).`, + // Hidden until the environment constraints repository is publicly + // available: the command is runnable for dogfooding but stays out of + // help and completion until it is unveiled. + Hidden: true, } // The target is selected via flags; reject stray positional args rather than // silently ignoring them. @@ -43,39 +47,42 @@ env-owned sections are refreshed, user-owned content is preserved).`, // addTargetFlags adds the shared target and mode flags to a command. func addTargetFlags(cmd *cobra.Command) { - cmd.Flags().String("cluster", "", "cluster ID to use as the compute target") - cmd.Flags().String("serverless", "", "serverless version to use as the compute target (e.g. v4)") - cmd.Flags().String("job", "", "job ID to use as the compute target") + cmd.Flags().String("cluster-id", "", "cluster ID to use as the compute target") + cmd.Flags().String("cluster-name", "", "cluster name to use as the compute target (resolved to an ID via the Clusters API)") + cmd.Flags().String("serverless-version", "", "serverless version to use as the compute target (e.g. 5)") + cmd.Flags().String("job-task", "", "job task to use as the compute target, as . (the task key is required)") cmd.Flags().Bool("constraints-only", false, "apply the Python version and constraints without adding the databricks-connect dependency") - cmd.Flags().Bool("check", false, "compute the plan without writing files or provisioning") - cmd.Flags().String("constraint-source", "", "URL for the constraint source (overrides "+envConstraintSource+")") - // Hide constraint-source from casual --help output; it is a power-user escape hatch. - _ = cmd.Flags().MarkHidden("constraint-source") - cmd.MarkFlagsMutuallyExclusive("cluster", "serverless", "job") + cmd.Flags().Bool("dry-run", false, "compute the plan without writing files or provisioning") + cmd.Flags().String("constraint-source-url", "", "URL for the constraint source (overrides "+envConstraintSource+")") + // Hide constraint-source-url from casual --help output; it is a power-user escape hatch. + _ = cmd.Flags().MarkHidden("constraint-source-url") + // The mutual exclusivity of the target flags is enforced in the pipeline's + // preflight (as E_USAGE) rather than via cmd.MarkFlagsMutuallyExclusive, so + // the conflict is reported through the phase/JSON contract the --output json + // consumer relies on, instead of a bare pre-RunE Cobra error. } -// runPipeline builds and runs the local-env Pipeline. +// runPipeline builds and runs the setup-local Pipeline. func runPipeline(cmd *cobra.Command) error { ctx := cmd.Context() - cluster, _ := cmd.Flags().GetString("cluster") - serverless, _ := cmd.Flags().GetString("serverless") - job, _ := cmd.Flags().GetString("job") + cluster, _ := cmd.Flags().GetString("cluster-id") + clusterName, _ := cmd.Flags().GetString("cluster-name") + serverless, _ := cmd.Flags().GetString("serverless-version") + jobTask, _ := cmd.Flags().GetString("job-task") constraintsOnly, _ := cmd.Flags().GetBool("constraints-only") - check, _ := cmd.Flags().GetBool("check") - constraintSource, _ := cmd.Flags().GetString("constraint-source") + check, _ := cmd.Flags().GetBool("dry-run") + constraintSource, _ := cmd.Flags().GetString("constraint-source-url") targetFlags := libslocalenv.TargetFlags{ - Cluster: cluster, - Serverless: serverless, - Job: job, - } - // ValidateTargetFlags is kept despite MarkFlagsMutuallyExclusive above: - // it also validates the library path (no Cobra equivalent) and guards - // non-Cobra call paths such as tests that invoke runPipeline directly. - if err := libslocalenv.ValidateTargetFlags(targetFlags); err != nil { - return err + Cluster: cluster, + ClusterName: clusterName, + Serverless: serverless, + JobTask: jobTask, } + // Flag validation (including mutual exclusivity) happens in the pipeline's + // preflight, so a conflict is reported as E_USAGE through the phase/JSON + // contract rather than as a bare error here. mode := libslocalenv.ModeDefault if constraintsOnly { @@ -96,11 +103,11 @@ func runPipeline(cmd *cobra.Command) error { cacheDir = filepath.Join(cacheDir, "databricks", "localenv") // The bundle is only a fallback: ResolveTarget consults it solely when no - // explicit --cluster/--serverless/--job flag is set. Skip the bundle load + // explicit --cluster-id/--cluster-name/--serverless-version/--job-task flag is set. Skip the bundle load // entirely when a flag is present — it would otherwise re-run TryConfigureBundle // (a second full load) and re-print any bundle load-time diagnostics for nothing. var bt libslocalenv.BundleTarget - if cluster == "" && serverless == "" && job == "" { + if cluster == "" && clusterName == "" && serverless == "" && jobTask == "" { bt = bundleTarget(cmd) } @@ -122,7 +129,7 @@ func runPipeline(cmd *cobra.Command) error { } // resolveConstraintBaseURL returns the constraint base URL using ordered precedence: -// an explicit --constraint-source flag, then a full-URL override from +// an explicit --constraint-source-url flag, then a full-URL override from // DATABRICKS_LOCALENV_CONSTRAINT_SOURCE, then the URL derived from the hosting repo // (libslocalenv.RepoConstraintBaseURL). All three may be unset, in which case it // returns "" and the pipeline reports the missing source at the fetch phase. diff --git a/cmd/experimental/experimental.go b/cmd/experimental/experimental.go index d87c893abc5..7dec16256de 100644 --- a/cmd/experimental/experimental.go +++ b/cmd/experimental/experimental.go @@ -3,7 +3,6 @@ package experimental import ( aircmd "github.com/databricks/cli/experimental/air/cmd" aitoolscmd "github.com/databricks/cli/experimental/aitools/cmd" - geniecmd "github.com/databricks/cli/experimental/genie/cmd" postgrescmd "github.com/databricks/cli/experimental/postgres/cmd" "github.com/spf13/cobra" ) @@ -25,7 +24,7 @@ development. They may change or be removed in future versions without notice.`, cmd.AddCommand(aircmd.New()) cmd.AddCommand(aitoolscmd.NewAitoolsCmd()) - cmd.AddCommand(geniecmd.NewGenieCmd()) + cmd.AddCommand(newGenieCmd()) cmd.AddCommand(postgrescmd.New()) cmd.AddCommand(newWorkspaceOpenCommand()) diff --git a/cmd/experimental/genie.go b/cmd/experimental/genie.go new file mode 100644 index 00000000000..a9e6fe0d146 --- /dev/null +++ b/cmd/experimental/genie.go @@ -0,0 +1,23 @@ +package experimental + +import ( + geniecmd "github.com/databricks/cli/cmd/genie" + "github.com/spf13/cobra" +) + +// newGenieCmd keeps the deprecated `experimental genie ask` path working after +// the command was promoted to `databricks genie ask`. It reuses the relocated +// builder and marks the subcommand deprecated so callers get a move notice. +func newGenieCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie", + Short: "Ask data questions via Databricks Genie", + Hidden: true, + } + + ask := geniecmd.NewAskCmd() + ask.Deprecated = `use "databricks genie ask" instead; this experimental alias will be removed in a future release` + cmd.AddCommand(ask) + + return cmd +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 9589b48a126..861dab1eda4 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -116,6 +116,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_instances", "experiments", "genie_spaces", + "instance_pools", "jobs", "model_serving_endpoints", "models", @@ -146,7 +147,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index d7ae6517539..01d566cd4a9 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -18,9 +18,10 @@ import ( "golang.org/x/sync/errgroup" ) -// Default number of concurrent file copy operations. This is a conservative -// default that should be sufficient to fully utilize the available bandwidth -// in most cases. +// defaultConcurrency is the number of files copied in parallel. Each in-flight +// file drives Files API requests (a single-shot PUT, or the multipart +// control-plane calls), which allow only ~10 concurrent requests, so this stays +// conservative regardless of whether multipart upload is enabled. const defaultConcurrency = 8 // errInvalidConcurrency is returned when the value of the concurrency @@ -37,6 +38,11 @@ type copy struct { sourceScheme string targetScheme string + // showProgress renders an upload progress bar. It is set only for a single + // large-file copy to a Volume, not for recursive copies (where many files + // would each fight for the spinner line). + showProgress bool + mu sync.Mutex // protect output from concurrent writes } @@ -123,20 +129,31 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) } defer r.Close() + // For a single large-file copy, attach a progress callback to the context + // that the Files filer forwards to the upload engine, rendering an upload bar. + // The spinner is stopped before any event line is emitted so its final frame + // does not overwrite it. + closeProgress := func() {} + if c.showProgress { + fn, stop := newProgressFunc(ctx) + ctx = filer.WithUploadProgress(ctx, fn) + closeProgress = stop + } + + var writeErr error if c.overwrite { - err = c.targetFiler.Write(ctx, targetPath, r, filer.OverwriteIfExists) - if err != nil { - return err - } + writeErr = c.targetFiler.Write(ctx, targetPath, r, filer.OverwriteIfExists) } else { - err = c.targetFiler.Write(ctx, targetPath, r) - // skip if file already exists - if err != nil && errors.Is(err, fs.ErrExist) { - return c.emitFileSkippedEvent(ctx, sourcePath, targetPath) - } - if err != nil { - return err - } + writeErr = c.targetFiler.Write(ctx, targetPath, r) + } + closeProgress() + + // skip if file already exists + if !c.overwrite && writeErr != nil && errors.Is(writeErr, fs.ErrExist) { + return c.emitFileSkippedEvent(ctx, sourcePath, targetPath) + } + if writeErr != nil { + return writeErr } return c.emitFileCopiedEvent(ctx, sourcePath, targetPath) } @@ -229,7 +246,7 @@ func newCpCommand() *cobra.Command { return err } - // Get target filer and target path without scheme + // Get target filer and target path without scheme. fullTargetPath := args[1] targetFiler, targetPath, err := filerForPath(ctx, fullTargetPath) if err != nil { @@ -259,6 +276,10 @@ func newCpCommand() *cobra.Command { return c.cpDirToDir(ctx, sourcePath, targetPath) } + // A single large file copied to a Volume goes through the multipart engine, + // which reports progress; render an upload bar for it. + c.showProgress = filer.MultipartUploadEnabled(ctx) && strings.HasPrefix(targetPath, "/Volumes/") + // If target path has a trailing separator, trim it and let case 2 handle it if hasTrailingDirSeparator(fullTargetPath) { targetPath = trimTrailingDirSeparators(targetPath) diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 54d9463ed91..42b3e546fe5 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -40,7 +40,7 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er // If the specified path has the "Volumes" prefix, use the Files API. if strings.HasPrefix(path, "/Volumes/") { - f, err := filer.NewFilesClient(w, "/") + f, err := filer.NewFilesClient(ctx, w, "/") return f, path, err } diff --git a/cmd/fs/upload_progress.go b/cmd/fs/upload_progress.go new file mode 100644 index 00000000000..a186fb5fa76 --- /dev/null +++ b/cmd/fs/upload_progress.go @@ -0,0 +1,256 @@ +package fs + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/charmbracelet/lipgloss" + + "github.com/databricks/cli/libs/cmdio" + files "github.com/databricks/sdk-go/files/v2" +) + +// renderThrottle caps how often the interactive bar is redrawn so a fast upload +// does not spend its time repainting the terminal; completion always renders. +const renderThrottle = 100 * time.Millisecond + +// preparingFrame is how long each "Preparing upload" dot frame is shown before +// the upload reports its first completed part. +const preparingFrame = 400 * time.Millisecond + +// barWidth keeps the rendered bar narrow enough that the whole progress line +// stays on a single terminal row, so the spinner's in-place redraw is clean. +const barWidth = 24 + +// rateWindow is the trailing duration over which the transfer rate is averaged. +// A few seconds smooths the burstiness of concurrent part completions while +// still tracking genuine changes in throughput. +const rateWindow = 5 * time.Second + +// Bar styling. Green for the filled portion matches the cmdio spinner glyph; +// the remainder is dimmed. +var ( + barFilledStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) + barEmptyStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("240")) +) + +// barBlocks are partial block glyphs for sub-character precision, so the bar +// advances smoothly rather than in whole-cell jumps. Index 0 is a full cell +// (8/8); index i is (8-i)/8 of a cell. Matches experimental/genie/agentstream. +var barBlocks = []string{"█", "▉", "▊", "▋", "▌", "▍", "▎", "▏"} + +// humanBytes formats a byte count using binary (1024-based) units. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + units := []string{"KiB", "MiB", "GiB", "TiB", "PiB"} + div, exp := int64(unit), 0 + for n/div >= unit && exp < len(units)-1 { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %s", float64(n)/float64(div), units[exp]) +} + +// formatSpeed formats a transfer rate in bytes per second. +func formatSpeed(bytesPerSec float64) string { + if bytesPerSec < 0 { + bytesPerSec = 0 + } + return humanBytes(int64(bytesPerSec)) + "/s" +} + +// formatETA formats the estimated time remaining as MM:SS. It returns a +// placeholder when the rate is unknown so the line does not show a misleading +// estimate before any throughput has been measured. +func formatETA(remaining int64, bytesPerSec float64) string { + if bytesPerSec <= 0 || remaining < 0 { + return "--:--" + } + secs := int(float64(remaining) / bytesPerSec) + return fmt.Sprintf("%02d:%02d", secs/60, secs%60) +} + +// formatPlainProgress renders a single non-interactive progress line. +func formatPlainProgress(p files.Progress) string { + if p.Total <= 0 { + return "Uploaded " + humanBytes(p.Transferred) + } + pct := int(float64(p.Transferred) / float64(p.Total) * 100) + return fmt.Sprintf("Uploaded %s / %s (%d%%)", humanBytes(p.Transferred), humanBytes(p.Total), pct) +} + +// renderBar returns a fixed-width progress bar for the given completion ratio. +// The total visible width is always barWidth: full cells, an optional partial +// cell for the fractional remainder, then the dimmed empty track. +func renderBar(ratio float64) string { + ratio = min(max(ratio, 0), 1) + exact := ratio * barWidth + full := int(exact) + + filled := strings.Repeat("█", full) + partial := int((exact - float64(full)) * 8) + if partial > 0 && full < barWidth { + filled += barBlocks[8-partial] + full++ // the partial glyph occupies one cell of the track + } + + return barFilledStyle.Render(filled) + + barEmptyStyle.Render(strings.Repeat("░", barWidth-full)) +} + +// rateSample is a cumulative byte count observed at a point in time. +type rateSample struct { + at time.Time + bytes int64 +} + +// rateMeter computes the transfer rate as a rolling average over a trailing +// time window. Averaging over elapsed wall-clock time, rather than an +// exponential moving average over samples, keeps the reported speed and the +// ETA derived from it stable: the engine reports progress in irregular bursts +// as concurrent parts complete, and a per-sample EMA spikes on every burst. +// Callers pass the current time so the meter stays deterministic and +// unit-testable. +type rateMeter struct { + samples []rateSample +} + +// observe records a cumulative byte count at now and returns the average +// bytes/sec over the trailing rateWindow. It returns 0 until two samples span +// a positive interval (the first sample, or one taken after a stall longer +// than the window, leaves nothing to average against). +func (m *rateMeter) observe(now time.Time, transferred int64) float64 { + // Drop samples that have aged out of the window, then record this one. + // Filtering in place reuses the backing array; at the render cadence the + // window holds at most a few dozen samples, so this stays cheap. + cutoff := now.Add(-rateWindow) + kept := m.samples[:0] + for _, s := range m.samples { + if !s.at.Before(cutoff) { + kept = append(kept, s) + } + } + m.samples = append(kept, rateSample{at: now, bytes: transferred}) + + oldest := m.samples[0] + dt := now.Sub(oldest.at).Seconds() + if dt <= 0 { + return 0 + } + return float64(transferred-oldest.bytes) / dt +} + +// progressRenderer builds the rich single-line progress string shown as the +// spinner suffix in an interactive terminal. +type progressRenderer struct { + meter rateMeter +} + +// newProgressRenderer creates a renderer. +func newProgressRenderer() *progressRenderer { + return &progressRenderer{} +} + +// render returns the progress line for the given progress sample at time now. +func (r *progressRenderer) render(now time.Time, p files.Progress) string { + rate := r.meter.observe(now, p.Transferred) + if p.Total <= 0 { + return fmt.Sprintf("%s %s", humanBytes(p.Transferred), formatSpeed(rate)) + } + ratio := float64(p.Transferred) / float64(p.Total) + return fmt.Sprintf("%s %.0f%% %s/%s %s ETA %s", + renderBar(ratio), ratio*100, + humanBytes(p.Transferred), humanBytes(p.Total), + formatSpeed(rate), formatETA(p.Total-p.Transferred, rate)) +} + +// newProgressFunc returns the upload progress callback for the current terminal +// and a function that stops it. In an interactive terminal it renders a rich +// single-line bar via the spinner; otherwise it logs coarse progress so a long +// upload still shows life. The returned stop function is idempotent (it is safe +// to defer it and also call it before printing a summary line). The callback is +// serialized by the engine, so its captured state needs no locking. +func newProgressFunc(ctx context.Context) (files.ProgressFunc, func()) { + if cmdio.GetInteractiveMode(ctx) == cmdio.InteractiveModeNone { + nextPct := 10 + fn := func(p files.Progress) { + // The final summary line covers completion; only log intermediate steps. + if p.Total <= 0 || p.Transferred >= p.Total { + return + } + pct := int(float64(p.Transferred) / float64(p.Total) * 100) + if pct < nextPct { + return + } + cmdio.LogString(ctx, formatPlainProgress(p)) + for pct >= nextPct { + nextPct += 10 + } + } + return fn, func() {} + } + + sp := cmdio.NewSpinner(ctx, cmdio.WithElapsedTime()) + r := newProgressRenderer() + + // The callback first fires only when a part completes; session initiation, URL + // minting, and the first PUT take a beat, during which it never runs. Animate a + // "Preparing upload" label with cycling dots over that window so the spinner is + // not bare. mu serializes the animator's suffix updates with the callback's, and + // preparing gates the animator off once real progress arrives, so the bar + // replaces the label with no flicker. + var ( + mu sync.Mutex + preparing = true + lastRender time.Time + ) + prepCtx, stopPreparing := context.WithCancel(ctx) + prepDone := make(chan struct{}) + go func() { + defer close(prepDone) + ticker := time.NewTicker(preparingFrame) + defer ticker.Stop() + for dots := 1; ; dots = dots%3 + 1 { + mu.Lock() + if !preparing { + mu.Unlock() + return + } + sp.Update("Preparing upload" + strings.Repeat(".", dots)) + mu.Unlock() + select { + case <-prepCtx.Done(): + return + case <-ticker.C: + } + } + }() + + fn := func(p files.Progress) { + mu.Lock() + defer mu.Unlock() + if preparing { + preparing = false + stopPreparing() // first real progress: stop the animator, the bar takes over + } + now := time.Now() + if p.Total > 0 && p.Transferred < p.Total && now.Sub(lastRender) < renderThrottle { + return + } + lastRender = now + sp.Update(r.render(now, p)) + } + // Stop the animator and wait for it to exit before closing the spinner, so no + // suffix update races the spinner shutdown. + return fn, func() { + stopPreparing() + <-prepDone + sp.Close() + } +} diff --git a/cmd/fs/upload_progress_test.go b/cmd/fs/upload_progress_test.go new file mode 100644 index 00000000000..2786cf0fd40 --- /dev/null +++ b/cmd/fs/upload_progress_test.go @@ -0,0 +1,96 @@ +package fs + +import ( + "testing" + "time" + + "github.com/charmbracelet/lipgloss" +) + +func TestRenderBarWidth(t *testing.T) { + // The bar must always occupy exactly barWidth cells regardless of ratio, + // including the partial-cell and out-of-range cases. + for _, ratio := range []float64{-0.5, 0, 0.0001, 0.123, 0.5, 0.52, 0.999, 1, 1.5} { + if w := lipgloss.Width(renderBar(ratio)); w != barWidth { + t.Errorf("renderBar(%v) width = %d, want %d", ratio, w, barWidth) + } + } +} + +func TestRateMeter(t *testing.T) { + var m rateMeter + base := time.Unix(0, 0) + + if got := m.observe(base, 0); got != 0 { + t.Fatalf("first observe = %v, want 0", got) + } + // A second sample at the same instant spans no interval to average over. + if got := m.observe(base, 1<<20); got != 0 { + t.Fatalf("zero-dt observe = %v, want 0 (no positive interval)", got) + } + + // A steady 1 MiB/s stream averages to 1 MiB/s. + var rate float64 + for i := 1; i <= 50; i++ { + rate = m.observe(base.Add(time.Duration(i)*time.Second), int64(i)<<20) + } + const want = float64(1 << 20) + if rate < want*0.99 || rate > want*1.01 { + t.Errorf("steady rate = %v, want ~%v", rate, want) + } +} + +func TestRateMeterSmoothsBursts(t *testing.T) { + var m rateMeter + base := time.Unix(0, 0) + + // Prime a steady 1 MiB/s stream over the full window. + for i := range 6 { + m.observe(base.Add(time.Duration(i)*time.Second), int64(i)<<20) + } + + // A large part lands almost instantly: 5 MiB in 100ms. A per-sample EMA + // would read this as a ~50 MiB/s spike; the rolling window keeps the + // reported rate near the windowed throughput. + rate := m.observe(base.Add(5100*time.Millisecond), 10<<20) + if rate > 3<<20 { + t.Errorf("post-burst rate = %.0f B/s, want smoothed under %d B/s", rate, 3<<20) + } +} + +func TestFormatSpeed(t *testing.T) { + cases := []struct { + bps float64 + want string + }{ + {0, "0 B/s"}, + {-5, "0 B/s"}, + {512, "512 B/s"}, + {48 << 20, "48.0 MiB/s"}, + } + for _, tc := range cases { + if got := formatSpeed(tc.bps); got != tc.want { + t.Errorf("formatSpeed(%v) = %q, want %q", tc.bps, got, tc.want) + } + } +} + +func TestFormatETA(t *testing.T) { + cases := []struct { + remaining int64 + bps float64 + want string + }{ + {0, 0, "--:--"}, + {1 << 20, 0, "--:--"}, + {-1, 1 << 20, "--:--"}, + {0, 1 << 20, "00:00"}, + {1 << 20, 1 << 20, "00:01"}, + {120 << 20, 1 << 20, "02:00"}, + } + for _, tc := range cases { + if got := formatETA(tc.remaining, tc.bps); got != tc.want { + t.Errorf("formatETA(%d, %v) = %q, want %q", tc.remaining, tc.bps, got, tc.want) + } + } +} diff --git a/cmd/fuzz_panic_test.go b/cmd/fuzz_panic_test.go index 5ac344d0341..970ca17f0a8 100644 --- a/cmd/fuzz_panic_test.go +++ b/cmd/fuzz_panic_test.go @@ -198,6 +198,7 @@ func isAutoGenerated(leaf leafCommand) bool { // "bundle", "auth", "sync", "fs", etc. The heuristic: anything whose // root isn't in this block list is auto-generated. manualRoots := map[string]bool{ + "aitools": true, "bundle": true, "auth": true, "sync": true, diff --git a/experimental/genie/cmd/ask.go b/cmd/genie/ask.go similarity index 88% rename from experimental/genie/cmd/ask.go rename to cmd/genie/ask.go index a3590a3dfb6..0458024a195 100644 --- a/experimental/genie/cmd/ask.go +++ b/cmd/genie/ask.go @@ -1,4 +1,4 @@ -package geniecmd +package genie import ( "context" @@ -10,15 +10,15 @@ import ( "syscall" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/experimental/genie" - "github.com/databricks/cli/experimental/genie/agentstream" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/genie" + "github.com/databricks/cli/libs/genie/agentstream" "github.com/spf13/cobra" ) -func newAskCmd() *cobra.Command { +func NewAskCmd() *cobra.Command { var warehouseID string var raw bool var includeSQL bool @@ -30,14 +30,14 @@ func newAskCmd() *cobra.Command { Long: `Ask a data question and get an answer from Databricks Genie. Examples: - databricks experimental genie ask "What were total sales last month?" - databricks experimental genie ask "What tables exist?" --output json - databricks experimental genie ask "Revenue by region" --warehouse-id 1234567890abcdef - databricks experimental genie ask "What tables exist?" --raw + databricks genie ask "What were total sales last month?" + databricks genie ask "What tables exist?" --output json + databricks genie ask "Revenue by region" --warehouse-id 1234567890abcdef + databricks genie ask "What tables exist?" --raw # Continue a conversation across calls with a session id you choose: - databricks experimental genie ask -s sales "What were total sales by quarter?" - databricks experimental genie ask -s sales "Break that down by region"`, + databricks genie ask -s sales "What were total sales by quarter?" + databricks genie ask -s sales "Break that down by region"`, Args: root.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { cmd.SetContext(root.SkipLoadBundle(cmd.Context())) diff --git a/experimental/genie/cmd/ask_test.go b/cmd/genie/ask_test.go similarity index 99% rename from experimental/genie/cmd/ask_test.go rename to cmd/genie/ask_test.go index 23f58c3d04d..7daae23ec16 100644 --- a/experimental/genie/cmd/ask_test.go +++ b/cmd/genie/ask_test.go @@ -1,4 +1,4 @@ -package geniecmd +package genie import ( "bytes" diff --git a/experimental/genie/cmd/conversations.go b/cmd/genie/conversations.go similarity index 99% rename from experimental/genie/cmd/conversations.go rename to cmd/genie/conversations.go index e0c2aa6135c..cda7f170ec2 100644 --- a/experimental/genie/cmd/conversations.go +++ b/cmd/genie/conversations.go @@ -1,4 +1,4 @@ -package geniecmd +package genie import ( "context" diff --git a/experimental/genie/cmd/conversations_test.go b/cmd/genie/conversations_test.go similarity index 99% rename from experimental/genie/cmd/conversations_test.go rename to cmd/genie/conversations_test.go index abb1ae9ffe1..aaa8f298dd1 100644 --- a/experimental/genie/cmd/conversations_test.go +++ b/cmd/genie/conversations_test.go @@ -1,4 +1,4 @@ -package geniecmd +package genie import ( "os" diff --git a/cmd/localenv/compute.go b/cmd/localenv/compute.go deleted file mode 100644 index 8ce8c91314d..00000000000 --- a/cmd/localenv/compute.go +++ /dev/null @@ -1,114 +0,0 @@ -package localenv - -import ( - "context" - "fmt" - "strconv" - - databricks "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -// sdkCompute adapts the Databricks SDK to the localenv.ComputeClient interface. -type sdkCompute struct { - w *databricks.WorkspaceClient -} - -// GetClusterSparkVersion returns the Spark version string for a running cluster. -func (c sdkCompute) GetClusterSparkVersion(ctx context.Context, clusterID string) (string, error) { - d, err := c.w.Clusters.GetByClusterId(ctx, clusterID) - if err != nil { - return "", fmt.Errorf("get cluster %s: %w", clusterID, err) - } - return d.SparkVersion, nil -} - -// GetJobSparkVersion inspects the job's configuration to determine compute type. -// -// A job is considered serverless when it has non-empty Environments (JobEnvironment -// entries), which signals the Databricks serverless runtime. A job with classic compute -// uses JobClusters; we read SparkVersion from the first job cluster's NewCluster spec. -// -// Task-level compute (tasks[].new_cluster / tasks[].existing_cluster_id with no -// job-level job_clusters) is not resolved here: it may vary per task and an -// existing_cluster_id would need a second lookup, which is out of scope for the -// initial job support. Such a job returns an actionable error rather than a wrong -// guess; use --cluster or --serverless explicitly instead. -func (c sdkCompute) GetJobSparkVersion(ctx context.Context, jobID string) (sparkVersion string, isServerless bool, version string, err error) { - id, err := strconv.ParseInt(jobID, 10, 64) - if err != nil { - return "", false, "", fmt.Errorf("invalid job ID %q: must be an integer: %w", jobID, err) - } - - job, err := c.w.Jobs.GetByJobId(ctx, id) - if err != nil { - return "", false, "", fmt.Errorf("get job %d: %w", id, err) - } - - if job.Settings == nil { - return "", false, "", fmt.Errorf("job %d has no settings", id) - } - - // A job that declares both serverless environments and classic job clusters is - // ambiguous: its tasks can run on different compute, so there is no single - // correct local environment to provision. Refuse rather than guess serverless. - if len(job.Settings.Environments) > 0 && len(job.Settings.JobClusters) > 0 { - return "", false, "", fmt.Errorf("job %d has both serverless environments and job clusters; pass --cluster or --serverless explicitly to disambiguate", id) - } - - // Serverless jobs have Environments populated; classic compute uses JobClusters. - if len(job.Settings.Environments) > 0 { - // The serverless environment version (e.g. "4") is recorded on the job's - // environment spec, unlike the bundle path where it is unavailable. Return - // it so ResolveTarget pins the matching serverless-vN instead of defaulting - // to v4. An empty version (older jobs) falls back to v4 in ResolveTarget. - version := environmentVersion(job.Settings.Environments[0]) - // Tasks can reference any environment_key, so if the job's environments do - // not all share one version there is no single correct local environment - // (mirrors the job-cluster check below). Refuse rather than guess from the - // first. A pinned-vs-unpinned mix is also ambiguous, so compare raw values. - for _, e := range job.Settings.Environments[1:] { - if environmentVersion(e) != version { - return "", false, "", fmt.Errorf("job %d has serverless environments with differing versions; pass --serverless explicitly to disambiguate", id) - } - } - return "", true, version, nil - } - - if len(job.Settings.JobClusters) > 0 { - sv := job.Settings.JobClusters[0].NewCluster.SparkVersion - if sv == "" { - return "", false, "", fmt.Errorf("could not determine compute for job %d: first job cluster has no spark_version", id) - } - // Tasks can reference any job_cluster_key, so if the job's clusters do not - // all share one Spark version there is no single correct local environment. - // Refuse rather than silently provisioning for the first cluster. - for _, jc := range job.Settings.JobClusters[1:] { - if jc.NewCluster.SparkVersion != sv { - return "", false, "", fmt.Errorf("job %d has job clusters with differing spark_version; pass --cluster or --serverless explicitly to disambiguate", id) - } - } - return sv, false, sv, nil - } - - return "", false, "", fmt.Errorf("could not determine compute for job %d from its environments or job clusters (task-level compute is not supported); pass --cluster or --serverless explicitly", id) -} - -// environmentVersion returns the serverless environment version recorded on a -// job environment, or "" when the spec or version is absent. -// -// The version can arrive in either of two fields. environment_version is the -// current one; client is its deprecated predecessor ("Use environment_version -// instead") and is still what some jobs pin. Reading both means the v4 fallback -// and the divergence guard observe whichever field actually carries the pin, -// rather than treating a client-pinned job as unversioned. base_environment is -// deliberately ignored: it is a path/ID, not a version. -func environmentVersion(e jobs.JobEnvironment) string { - if e.Spec == nil { - return "" - } - if e.Spec.EnvironmentVersion != "" { - return e.Spec.EnvironmentVersion - } - return e.Spec.Client -} diff --git a/cmd/localenv/localenv.go b/cmd/localenv/localenv.go deleted file mode 100644 index b93c36f7102..00000000000 --- a/cmd/localenv/localenv.go +++ /dev/null @@ -1,45 +0,0 @@ -package localenv - -import ( - "github.com/databricks/cli/cmd/root" - libslocalenv "github.com/databricks/cli/libs/localenv" - "github.com/spf13/cobra" -) - -// New returns the local-env command group. The group, subgroup, and verb names -// come from the single command-name constants in libs/localenv so a rename is a -// one-location change (spec §0 / invariant 8). -// -// The command is Hidden while the feature lands across the stacked PRs: it is -// wired and runnable for dogfooding, but stays out of help and completion until -// the final PR unveils it (removes this flag, adds the help line and changelog). -func New() *cobra.Command { - cmd := &cobra.Command{ - Use: libslocalenv.CommandGroup, - Short: "Manage local development environments matched to Databricks compute", - GroupID: "development", - Hidden: true, - Long: `Manage local development environments matched to a Databricks compute target. - -Derives the Python version, databricks-connect version, and dependency -constraints from the selected compute (cluster, serverless, or job) so that -local resolution matches the Databricks runtime.`, - RunE: root.ReportUnknownSubcommand, - } - cmd.AddCommand(newPythonCommand()) - return cmd -} - -// newPythonCommand returns the "python" subgroup. It is a parent-only node: with -// no verb it reports an unknown-subcommand error (mirroring the generated command -// groups) rather than doing nothing. -func newPythonCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: libslocalenv.CommandSubgroup, - Short: "Manage the local Python environment", - Long: `Manage the local Python environment matched to a Databricks compute target.`, - RunE: root.ReportUnknownSubcommand, - } - cmd.AddCommand(newSyncCommand()) - return cmd -} diff --git a/cmd/root/root.go b/cmd/root/root.go index 6783449eebc..55e78e58633 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -82,6 +82,7 @@ func New(ctx context.Context) *cobra.Command { ctx = withUpstreamInUserAgent(ctx) ctx = withInteractiveModeInUserAgent(ctx) ctx = installer.WithAiToolsInUserAgent(ctx) + ctx = installer.WithAiDevKitInUserAgent(ctx) ctx = InjectTestPidToUserAgent(ctx) cmd.SetContext(ctx) diff --git a/cmd/workspace/ai-search/ai-search.go b/cmd/workspace/ai-search/ai-search.go index da9a7d8080e..82e664eb3e2 100644 --- a/cmd/workspace/ai-search/ai-search.go +++ b/cmd/workspace/ai-search/ai-search.go @@ -23,8 +23,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "ai-search", - Short: `*Beta* **AI Search Endpoint**: Represents the compute resources to host AI Search indexes.`, - Long: `This command is in Beta and may change without notice. + Short: `*Public Preview* **AI Search Endpoint**: Represents the compute resources to host AI Search indexes.`, + Long: `This command is in Public Preview and may change without notice. **AI Search Endpoint**: Represents the compute resources to host AI Search indexes. AIP-conformant replacement for the legacy VectorSearchEndpoints API; @@ -34,8 +34,8 @@ func New() *cobra.Command { } cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" // Add methods cmd.AddCommand(newCreateEndpoint()) @@ -91,8 +91,8 @@ func newCreateEndpoint() *cobra.Command { cmd.Flags().StringVar(&createEndpointReq.Endpoint.UsagePolicyId, "usage-policy-id", createEndpointReq.Endpoint.UsagePolicyId, `The usage policy id applied to the endpoint.`) cmd.Use = "create-endpoint PARENT ENDPOINT_TYPE" - cmd.Short = `*Beta* Create an AI Search endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Create an AI Search endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Create an AI Search endpoint. @@ -105,8 +105,8 @@ Create an AI Search endpoint. Supported values: [STANDARD, STORAGE_OPTIMIZED]` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -192,8 +192,8 @@ func newCreateIndex() *cobra.Command { // TODO: complex arg: status cmd.Use = "create-index PARENT PRIMARY_KEY INDEX_TYPE" - cmd.Short = `*Beta* Create an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Create an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Create an AI Search index. @@ -207,8 +207,8 @@ Create an AI Search index. Supported values: [DELTA_SYNC, DIRECT_ACCESS]` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -286,8 +286,8 @@ func newDeleteEndpoint() *cobra.Command { var deleteEndpointReq aisearch.DeleteEndpointRequest cmd.Use = "delete-endpoint NAME" - cmd.Short = `*Beta* Delete an AI Search endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Delete an AI Search endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Delete an AI Search endpoint. @@ -296,8 +296,8 @@ Delete an AI Search endpoint. workspaces/{workspace_id}/endpoints/{endpoint_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -345,8 +345,8 @@ func newDeleteIndex() *cobra.Command { var deleteIndexReq aisearch.DeleteIndexRequest cmd.Use = "delete-index NAME" - cmd.Short = `*Beta* Delete an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Delete an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Delete an AI Search index. @@ -355,8 +355,8 @@ Delete an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -404,8 +404,8 @@ func newGetEndpoint() *cobra.Command { var getEndpointReq aisearch.GetEndpointRequest cmd.Use = "get-endpoint NAME" - cmd.Short = `*Beta* Get an AI Search endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Get an AI Search endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Get an AI Search endpoint. @@ -416,8 +416,8 @@ Get an AI Search endpoint. workspaces/{workspace_id}/endpoints/{endpoint_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -466,8 +466,8 @@ func newGetIndex() *cobra.Command { var getIndexReq aisearch.GetIndexRequest cmd.Use = "get-index NAME" - cmd.Short = `*Beta* Get an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Get an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Get an AI Search index. @@ -478,8 +478,8 @@ Get an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -541,8 +541,8 @@ func newListEndpoints() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-endpoints PARENT" - cmd.Short = `*Beta* List AI Search endpoints.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* List AI Search endpoints.` + cmd.Long = `This command is in Public Preview and may change without notice. List AI Search endpoints. @@ -553,8 +553,8 @@ List AI Search endpoints. workspaces/{workspace_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -619,8 +619,8 @@ func newListIndexes() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-indexes PARENT" - cmd.Short = `*Beta* List AI Search indexes.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* List AI Search indexes.` + cmd.Long = `This command is in Public Preview and may change without notice. List AI Search indexes. @@ -631,8 +631,8 @@ List AI Search indexes. workspaces/{workspace_id}/endpoints/{endpoint_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -699,8 +699,8 @@ func newQueryIndex() *cobra.Command { // TODO: array: sort_columns cmd.Use = "query-index NAME" - cmd.Short = `*Beta* Query an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Query an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Query an AI Search index. @@ -712,8 +712,8 @@ Query an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -779,8 +779,8 @@ func newRemoveData() *cobra.Command { cmd.Flags().Var(&removeDataJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "remove-data NAME" - cmd.Short = `*Beta* Remove data from an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Remove data from an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Remove data from an AI Search index. @@ -791,8 +791,8 @@ Remove data from an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -861,8 +861,8 @@ func newScanIndex() *cobra.Command { cmd.Flags().StringVar(&scanIndexReq.PageToken, "page-token", scanIndexReq.PageToken, `Page token from a previous response; if unset, scanning starts from the beginning.`) cmd.Use = "scan-index NAME" - cmd.Short = `*Beta* Scan an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Scan an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Scan an AI Search index. @@ -873,8 +873,8 @@ Scan an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -935,8 +935,8 @@ func newSyncIndex() *cobra.Command { var syncIndexReq aisearch.SyncIndexRequest cmd.Use = "sync-index NAME" - cmd.Short = `*Beta* Synchronize an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Synchronize an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Synchronize an AI Search index. @@ -950,8 +950,8 @@ Synchronize an AI Search index. workspaces/{workspace_id}/endpoints/{endpoint_id}/indexes/{index_id}` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -1014,8 +1014,8 @@ func newUpdateEndpoint() *cobra.Command { cmd.Flags().StringVar(&updateEndpointReq.Endpoint.UsagePolicyId, "usage-policy-id", updateEndpointReq.Endpoint.UsagePolicyId, `The usage policy id applied to the endpoint.`) cmd.Use = "update-endpoint NAME UPDATE_MASK ENDPOINT_TYPE" - cmd.Short = `*Beta* Update an AI Search endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Update an AI Search endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Update an AI Search endpoint. @@ -1036,8 +1036,8 @@ Update an AI Search endpoint. Supported values: [STANDARD, STORAGE_OPTIMIZED]` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -1119,8 +1119,8 @@ func newUpsertData() *cobra.Command { cmd.Flags().Var(&upsertDataJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "upsert-data NAME INPUTS_JSON" - cmd.Short = `*Beta* Upsert data into an AI Search index.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Upsert data into an AI Search index.` + cmd.Long = `This command is in Public Preview and may change without notice. Upsert data into an AI Search index. @@ -1132,8 +1132,8 @@ Upsert data into an AI Search index. INPUTS_JSON: JSON document describing the rows to upsert.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { diff --git a/cmd/workspace/alerts-v2/alerts-v2.go b/cmd/workspace/alerts-v2/alerts-v2.go index 3af56df67af..48fe68451c0 100644 --- a/cmd/workspace/alerts-v2/alerts-v2.go +++ b/cmd/workspace/alerts-v2/alerts-v2.go @@ -20,18 +20,16 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "alerts-v2", - Short: `*Public Preview* New version of SQL Alerts.`, - Long: `This command is in Public Preview and may change without notice. - -New version of SQL Alerts`, + Use: "alerts-v2", + Short: `New version of SQL Alerts.`, + Long: `New version of SQL Alerts`, GroupID: "sql", RunE: root.ReportUnknownSubcommand, } cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" // Add methods cmd.AddCommand(newCreateAlert()) @@ -69,15 +67,14 @@ func newCreateAlert() *cobra.Command { cmd.Flags().StringVar(&createAlertReq.Alert.CustomDescription, "custom-description", createAlertReq.Alert.CustomDescription, `Custom description for the alert.`) cmd.Flags().StringVar(&createAlertReq.Alert.CustomSummary, "custom-summary", createAlertReq.Alert.CustomSummary, `Custom summary for the alert.`) // TODO: complex arg: effective_run_as + // TODO: array: parameters cmd.Flags().StringVar(&createAlertReq.Alert.ParentPath, "parent-path", createAlertReq.Alert.ParentPath, `The workspace path of the folder containing the alert.`) // TODO: complex arg: run_as cmd.Flags().StringVar(&createAlertReq.Alert.RunAsUserName, "run-as-user-name", createAlertReq.Alert.RunAsUserName, `The run as username or application ID of service principal.`) cmd.Use = "create-alert DISPLAY_NAME QUERY_TEXT WAREHOUSE_ID EVALUATION SCHEDULE" - cmd.Short = `*Public Preview* Create an alert.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Create an alert. + cmd.Short = `Create an alert.` + cmd.Long = `Create an alert. Create Alert @@ -89,8 +86,8 @@ Create an alert. SCHEDULE: ` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -180,16 +177,14 @@ func newGetAlert() *cobra.Command { var getAlertReq sql.GetAlertV2Request cmd.Use = "get-alert ID" - cmd.Short = `*Public Preview* Get an alert.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Get an alert. + cmd.Short = `Get an alert.` + cmd.Long = `Get an alert. Gets an alert.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { @@ -263,16 +258,14 @@ func newListAlerts() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-alerts" - cmd.Short = `*Public Preview* List alerts.` - cmd.Long = `This command is in Public Preview and may change without notice. - -List alerts. + cmd.Short = `List alerts.` + cmd.Long = `List alerts. Gets a list of alerts accessible to the user, ordered by creation time.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(0) @@ -324,18 +317,16 @@ func newTrashAlert() *cobra.Command { cmd.Flags().BoolVar(&trashAlertReq.Purge, "purge", trashAlertReq.Purge, `Whether to permanently delete the alert.`) cmd.Use = "trash-alert ID" - cmd.Short = `*Public Preview* Delete an alert (legacy TrashAlert).` - cmd.Long = `This command is in Public Preview and may change without notice. - -Delete an alert (legacy TrashAlert). + cmd.Short = `Delete an alert (legacy TrashAlert).` + cmd.Long = `Delete an alert (legacy TrashAlert). Moves an alert to the trash. Trashed alerts immediately disappear from list views, and can no longer trigger. You can restore a trashed alert through the UI. A trashed alert is permanently deleted after 30 days.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { @@ -401,15 +392,14 @@ func newUpdateAlert() *cobra.Command { cmd.Flags().StringVar(&updateAlertReq.Alert.CustomDescription, "custom-description", updateAlertReq.Alert.CustomDescription, `Custom description for the alert.`) cmd.Flags().StringVar(&updateAlertReq.Alert.CustomSummary, "custom-summary", updateAlertReq.Alert.CustomSummary, `Custom summary for the alert.`) // TODO: complex arg: effective_run_as + // TODO: array: parameters cmd.Flags().StringVar(&updateAlertReq.Alert.ParentPath, "parent-path", updateAlertReq.Alert.ParentPath, `The workspace path of the folder containing the alert.`) // TODO: complex arg: run_as cmd.Flags().StringVar(&updateAlertReq.Alert.RunAsUserName, "run-as-user-name", updateAlertReq.Alert.RunAsUserName, `The run as username or application ID of service principal.`) cmd.Use = "update-alert ID UPDATE_MASK DISPLAY_NAME QUERY_TEXT WAREHOUSE_ID EVALUATION SCHEDULE" - cmd.Short = `*Public Preview* Update an alert.` - cmd.Long = `This command is in Public Preview and may change without notice. - -Update an alert. + cmd.Short = `Update an alert.` + cmd.Long = `Update an alert. Update alert @@ -433,8 +423,8 @@ Update an alert. SCHEDULE: ` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { diff --git a/cmd/workspace/bundle-deployments/bundle-deployments.go b/cmd/workspace/bundle-deployments/bundle-deployments.go index b320cd9fba8..7dec3b0b6cc 100644 --- a/cmd/workspace/bundle-deployments/bundle-deployments.go +++ b/cmd/workspace/bundle-deployments/bundle-deployments.go @@ -177,28 +177,21 @@ func newCreateDeployment() *cobra.Command { cmd.Flags().Var(&createDeploymentJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: complex arg: git_info + cmd.Flags().StringVar(&createDeploymentReq.Deployment.InitialParentPath, "initial-parent-path", createDeploymentReq.Deployment.InitialParentPath, `The workspace path of the existing folder where the deployment is initially created.`) // TODO: complex arg: workspace_info - cmd.Use = "create-deployment DEPLOYMENT_ID" + cmd.Use = "create-deployment" cmd.Short = `Create a deployment.` cmd.Long = `Create a deployment. - Creates a new deployment in the workspace. - - The caller must provide a deployment_id which becomes the final component of - the deployment's resource name. If a deployment with the same ID already - exists, the server returns ALREADY_EXISTS. - - Arguments: - DEPLOYMENT_ID: The ID to use for the deployment, which will become the final component of - the deployment's resource name (i.e. deployments/{deployment_id}).` + Creates a new deployment in the workspace.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" cmd.Annotations["launch_stage_display"] = "Private Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) + check := root.ExactArgs(0) return check(cmd, args) } @@ -219,7 +212,6 @@ func newCreateDeployment() *cobra.Command { } } } - createDeploymentReq.DeploymentId = args[0] response, err := w.BundleDeployments.CreateDeployment(ctx, createDeploymentReq) if err != nil { @@ -471,11 +463,6 @@ func newDeleteDeployment() *cobra.Command { Deletes a deployment. - The deployment is marked as deleted. It and all its children (versions and - their operations) will be permanently deleted after the retention policy - expires. If the deployment has an in-progress version, the server returns - RESOURCE_CONFLICT. - Arguments: NAME: Resource name of the deployment to delete. Format: deployments/{deployment_id}` diff --git a/cmd/workspace/clean-room-asset-revisions/clean-room-asset-revisions.go b/cmd/workspace/clean-room-asset-revisions/clean-room-asset-revisions.go index e8188f3939c..8379f87849f 100644 --- a/cmd/workspace/clean-room-asset-revisions/clean-room-asset-revisions.go +++ b/cmd/workspace/clean-room-asset-revisions/clean-room-asset-revisions.go @@ -69,7 +69,14 @@ Get an asset revision. Arguments: CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: Asset type. Only NOTEBOOK_FILE is supported. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: Name of the asset. ETAG: Revision etag to fetch. If not provided, the latest revision will be returned.` @@ -155,7 +162,14 @@ List asset revisions. Arguments: CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: Asset type. Only NOTEBOOK_FILE is supported. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: Name of the asset.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/clean-room-assets/clean-room-assets.go b/cmd/workspace/clean-room-assets/clean-room-assets.go index f335be9f485..bb65c38d9c3 100644 --- a/cmd/workspace/clean-room-assets/clean-room-assets.go +++ b/cmd/workspace/clean-room-assets/clean-room-assets.go @@ -69,6 +69,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Asset.CleanRoomName, "clean-room-name", createReq.Asset.CleanRoomName, `The name of the clean room this asset belongs to.`) // TODO: complex arg: foreign_table // TODO: complex arg: foreign_table_local_details + // TODO: complex arg: jar_analysis // TODO: complex arg: notebook // TODO: complex arg: table // TODO: complex arg: table_local_details @@ -98,7 +99,14 @@ func newCreate() *cobra.Command { For notebooks, the name is the notebook file name. For jar analyses, the name is the jar analysis name. ASSET_TYPE: The type of the asset. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME]` + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ]` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" @@ -182,6 +190,7 @@ func newCreateCleanRoomAssetReview() *cobra.Command { cmd.Flags().Var(&createCleanRoomAssetReviewJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: jar_analysis_review // TODO: complex arg: notebook_review cmd.Use = "create-clean-room-asset-review CLEAN_ROOM_NAME ASSET_TYPE NAME" @@ -195,7 +204,14 @@ Create a review (e.g. approval) for an asset. Arguments: CLEAN_ROOM_NAME: Name of the clean room ASSET_TYPE: Asset type. Can either be NOTEBOOK_FILE or JAR_ANALYSIS. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: Name of the asset` cmd.Annotations = make(map[string]string) @@ -275,7 +291,14 @@ func newDelete() *cobra.Command { Arguments: CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: The type of the asset. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: The fully qualified name of the asset, it is same as the name field in CleanRoomAsset.` @@ -343,7 +366,14 @@ func newGet() *cobra.Command { Arguments: CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: The type of the asset. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: The fully qualified name of the asset, it is same as the name field in CleanRoomAsset.` @@ -481,6 +511,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Asset.CleanRoomName, "clean-room-name", updateReq.Asset.CleanRoomName, `The name of the clean room this asset belongs to.`) // TODO: complex arg: foreign_table // TODO: complex arg: foreign_table_local_details + // TODO: complex arg: jar_analysis // TODO: complex arg: notebook // TODO: complex arg: table // TODO: complex arg: table_local_details @@ -498,7 +529,14 @@ func newUpdate() *cobra.Command { Arguments: CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: The type of the asset. - Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] + Supported values: [ + FOREIGN_TABLE, + JAR_ANALYSIS, + NOTEBOOK_FILE, + TABLE, + VIEW, + VOLUME, + ] NAME: A fully qualified name that uniquely identifies the asset within the clean room. This is also the name displayed in the clean room UI. diff --git a/cmd/workspace/clean-room-task-runs/clean-room-task-runs.go b/cmd/workspace/clean-room-task-runs/clean-room-task-runs.go index 044912d41d1..1ab6b639bc6 100644 --- a/cmd/workspace/clean-room-task-runs/clean-room-task-runs.go +++ b/cmd/workspace/clean-room-task-runs/clean-room-task-runs.go @@ -18,9 +18,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "clean-room-task-runs", - Short: `Clean room task runs are the executions of notebooks in a clean room.`, - Long: `Clean room task runs are the executions of notebooks in a clean room.`, + Use: "clean-room-task-runs", + Short: `Clean room task runs are the executions of notebooks and JAR analyses in a clean room.`, + Long: `Clean room task runs are the executions of notebooks and JAR analyses in a + clean room.`, GroupID: "cleanrooms", RunE: root.ReportUnknownSubcommand, } @@ -31,6 +32,7 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newList()) + cmd.AddCommand(newListCleanRoomTaskRunsHandler()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -116,4 +118,81 @@ func newList() *cobra.Command { return cmd } +// start list-clean-room-task-runs-handler command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listCleanRoomTaskRunsHandlerOverrides []func( + *cobra.Command, + *cleanrooms.ListCleanRoomTaskRunsRequest, +) + +func newListCleanRoomTaskRunsHandler() *cobra.Command { + cmd := &cobra.Command{} + + var listCleanRoomTaskRunsHandlerReq cleanrooms.ListCleanRoomTaskRunsRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listCleanRoomTaskRunsHandlerLimit int + + cmd.Flags().StringVar(&listCleanRoomTaskRunsHandlerReq.Name, "name", listCleanRoomTaskRunsHandlerReq.Name, `Executable name.`) + cmd.Flags().IntVar(&listCleanRoomTaskRunsHandlerReq.PageSize, "page-size", listCleanRoomTaskRunsHandlerReq.PageSize, `The maximum number of task runs to return.`) + cmd.Flags().Var(&listCleanRoomTaskRunsHandlerReq.TaskType, "task-type", `Filter by the type of Clean Room task. Supported values: [JAR, NOTEBOOK]`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listCleanRoomTaskRunsHandlerLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listCleanRoomTaskRunsHandlerReq.PageToken, "page-token", listCleanRoomTaskRunsHandlerReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-clean-room-task-runs-handler CLEAN_ROOM_NAME" + cmd.Short = `List task runs.` + cmd.Long = `List task runs. + + List all the historical task runs in a clean room. + + Arguments: + CLEAN_ROOM_NAME: Name of the clean room.` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listCleanRoomTaskRunsHandlerReq.CleanRoomName = args[0] + + response := w.CleanRoomTaskRuns.ListCleanRoomTaskRunsHandler(ctx, listCleanRoomTaskRunsHandlerReq) + if listCleanRoomTaskRunsHandlerLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listCleanRoomTaskRunsHandlerLimit) + } + if listCleanRoomTaskRunsHandlerLimit > 0 { + ctx = cmdio.WithLimit(ctx, listCleanRoomTaskRunsHandlerLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listCleanRoomTaskRunsHandlerOverrides { + fn(cmd, &listCleanRoomTaskRunsHandlerReq) + } + + return cmd +} + // end service CleanRoomTaskRuns diff --git a/cmd/workspace/clean-rooms/clean-rooms.go b/cmd/workspace/clean-rooms/clean-rooms.go index 84443530d7f..aa904968b58 100644 --- a/cmd/workspace/clean-rooms/clean-rooms.go +++ b/cmd/workspace/clean-rooms/clean-rooms.go @@ -74,6 +74,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.CleanRoom.Comment, "comment", createReq.CleanRoom.Comment, ``) + cmd.Flags().BoolVar(&createReq.CleanRoom.EnableSharedOutput, "enable-shared-output", createReq.CleanRoom.EnableSharedOutput, `Whether allow task to write to shared output schema.`) cmd.Flags().StringVar(&createReq.CleanRoom.Name, "name", createReq.CleanRoom.Name, `The name of the clean room.`) // TODO: complex arg: output_catalog cmd.Flags().StringVar(&createReq.CleanRoom.Owner, "owner", createReq.CleanRoom.Owner, `This is the Databricks username of the owner of the local clean room securable for permission management.`) diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index 303d5346632..d0b0302b66f 100644 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -220,6 +220,7 @@ func newCreate() *cobra.Command { SINGLE_USER, USER_ISOLATION, ]`) + cmd.Flags().Var(&createReq.DependencyMode, "dependency-mode", `Controls dependency configuration for the cluster. Supported values: [DEPENDENCY_MODE_AUTO, DEPENDENCY_MODE_CLUSTER_LIBRARIES, DEPENDENCY_MODE_ENVIRONMENTS]`) // TODO: complex arg: docker_image cmd.Flags().StringVar(&createReq.DriverInstancePoolId, "driver-instance-pool-id", createReq.DriverInstancePoolId, `The optional ID of the instance pool for the driver of the cluster belongs.`) // TODO: complex arg: driver_node_type_flexibility @@ -503,6 +504,7 @@ func newEdit() *cobra.Command { SINGLE_USER, USER_ISOLATION, ]`) + cmd.Flags().Var(&editReq.DependencyMode, "dependency-mode", `Controls dependency configuration for the cluster. Supported values: [DEPENDENCY_MODE_AUTO, DEPENDENCY_MODE_CLUSTER_LIBRARIES, DEPENDENCY_MODE_ENVIRONMENTS]`) // TODO: complex arg: docker_image cmd.Flags().StringVar(&editReq.DriverInstancePoolId, "driver-instance-pool-id", editReq.DriverInstancePoolId, `The optional ID of the instance pool for the driver of the cluster belongs.`) // TODO: complex arg: driver_node_type_flexibility diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go index eaaa91ad7dc..3adb27bc530 100644 --- a/cmd/workspace/connections/connections.go +++ b/cmd/workspace/connections/connections.go @@ -73,6 +73,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `User-provided free-form text description.`) // TODO: complex arg: environment_settings + cmd.Flags().StringVar(&createReq.Parent, "parent", createReq.Parent, `Parent schema for schema-level connections, in format "schemas/{catalog}.{schema}".`) // TODO: map via StringToStringVar: properties cmd.Flags().BoolVar(&createReq.ReadOnly, "read-only", createReq.ReadOnly, `If the connection is read only.`) @@ -289,6 +290,8 @@ func newList() *cobra.Command { // branches, so the method-call path is always reached. var listLimit int + cmd.Flags().StringVar(&listReq.Parent, "parent", listReq.Parent, `Optional.`) + // Limit flag for total result capping. cmd.Flags().IntVar(&listLimit, "limit", 0, `Maximum number of results to return.`) diff --git a/cmd/workspace/environments/overrides.go b/cmd/workspace/environments/overrides.go new file mode 100644 index 00000000000..39864abceaf --- /dev/null +++ b/cmd/workspace/environments/overrides.go @@ -0,0 +1,19 @@ +package environments + +import ( + environmentsCli "github.com/databricks/cli/cmd/environments" + "github.com/spf13/cobra" +) + +func init() { + cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { + // Attach the hand-written local-provisioning commands (e.g. setup-local) + // to the auto-generated "environments" group. The group intentionally + // spans server-side environment-resource APIs and local provisioning, so + // a local-install verb belongs here (spec §naming). This mirrors how + // cmd/apps extends the generated apps group. + for _, c := range environmentsCli.Commands() { + cmd.AddCommand(c) + } + }) +} diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index c52160e1cff..92f7c54c62d 100644 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -107,6 +107,7 @@ func newCreateExperiment() *cobra.Command { cmd.Flags().StringVar(&createExperimentReq.ArtifactLocation, "artifact-location", createExperimentReq.ArtifactLocation, `Location where all artifacts for the experiment are stored.`) // TODO: array: tags + // TODO: complex arg: trace_location cmd.Use = "create-experiment NAME" cmd.Short = `Create experiment.` diff --git a/cmd/workspace/feature-engineering/feature-engineering.go b/cmd/workspace/feature-engineering/feature-engineering.go index 131bef26c49..4dca56a0c8c 100644 --- a/cmd/workspace/feature-engineering/feature-engineering.go +++ b/cmd/workspace/feature-engineering/feature-engineering.go @@ -412,9 +412,8 @@ func newCreateStream() *cobra.Command { NAME: Full three-part (catalog.schema.stream) name of the stream. SOURCE_CONFIG: Source-specific configuration. Determines the streaming platform source. CONNECTION_CONFIG: Specifies how to connect and authenticate to the stream platform. - SCHEMA_CONFIG: Schema definitions for the stream. Currently only direct schemas are - supported. In a future milestone, we will support schema registries - through a UC Connection. + SCHEMA_CONFIG: Schema definitions for the stream, provided either directly on the Stream + or resolved from an external schema registry through a UC Connection. INGESTION_CONFIG: Configuration for streaming data ingestion: the managed table storing an offline copy of forward fill data and optional historical backfill.` @@ -1616,9 +1615,8 @@ func newUpdateStream() *cobra.Command { UPDATE_MASK: The list of fields to update. SOURCE_CONFIG: Source-specific configuration. Determines the streaming platform source. CONNECTION_CONFIG: Specifies how to connect and authenticate to the stream platform. - SCHEMA_CONFIG: Schema definitions for the stream. Currently only direct schemas are - supported. In a future milestone, we will support schema registries - through a UC Connection. + SCHEMA_CONFIG: Schema definitions for the stream, provided either directly on the Stream + or resolved from an external schema registry through a UC Connection. INGESTION_CONFIG: Configuration for streaming data ingestion: the managed table storing an offline copy of forward fill data and optional historical backfill.` diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index 4b02963878c..f2af834a048 100644 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -514,7 +514,8 @@ Download message attachment visualization. Download a rendered image of a message visualization attachment. The response body is the raw PNG image, not a JSON payload. This is only available if the - attachment is a visualization and the message status is COMPLETED. + attachment is a visualization and the message status is COMPLETED. This + endpoint is not supported for Private Link workspaces. Arguments: NAME: The resource name of the attachment to render, in the format diff --git a/cmd/workspace/genie/overrides.go b/cmd/workspace/genie/overrides.go new file mode 100644 index 00000000000..750cefc96b7 --- /dev/null +++ b/cmd/workspace/genie/overrides.go @@ -0,0 +1,15 @@ +package genie + +import ( + geniecmd "github.com/databricks/cli/cmd/genie" + "github.com/spf13/cobra" +) + +func init() { + cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) { + // "ask" is a hand-written streaming data-question command, not part of the + // generated Genie spaces/conversations API surface; attach it to the same + // top-level "genie" group so it runs as `databricks genie ask`. + cmd.AddCommand(geniecmd.NewAskCmd()) + }) +} diff --git a/cmd/workspace/grants/grants.go b/cmd/workspace/grants/grants.go index 51ec366bcf9..4f8a4d10dd4 100644 --- a/cmd/workspace/grants/grants.go +++ b/cmd/workspace/grants/grants.go @@ -230,8 +230,10 @@ func newList() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list SECURABLE_TYPE FULL_NAME" - cmd.Short = `List permissions.` - cmd.Long = `List permissions. + cmd.Short = `*Public Preview* List permissions.` + cmd.Long = `This command is in Public Preview and may change without notice. + +List permissions. Lists the privilege assignments for a securable. Does not include inherited privileges. Paginated version of Get Permissions API. @@ -240,12 +242,9 @@ func newList() *cobra.Command { SECURABLE_TYPE: Type of securable. FULL_NAME: Full name of securable.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Private Preview" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(2) @@ -312,8 +311,10 @@ func newListEffective() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-effective SECURABLE_TYPE FULL_NAME" - cmd.Short = `List effective permissions.` - cmd.Long = `List effective permissions. + cmd.Short = `*Public Preview* List effective permissions.` + cmd.Long = `This command is in Public Preview and may change without notice. + +List effective permissions. Lists the effective privilege assignments for a securable. Includes inherited privileges. Paginated version of Get Effective Permissions API. @@ -322,12 +323,9 @@ func newListEffective() *cobra.Command { SECURABLE_TYPE: Type of securable. FULL_NAME: Full name of securable.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Private Preview" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(2) diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 0f9ec046da5..cb47b4c0bb0 100644 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -1647,6 +1647,7 @@ func newSubmit() *cobra.Command { // TODO: complex arg: health cmd.Flags().StringVar(&submitReq.IdempotencyToken, "idempotency-token", submitReq.IdempotencyToken, `An optional token that can be used to guarantee the idempotency of job run requests.`) // TODO: complex arg: notification_settings + cmd.Flags().Var(&submitReq.PerformanceTarget, "performance-target", `The performance mode on a serverless one-time run. Supported values: [PERFORMANCE_OPTIMIZED, STANDARD]`) // TODO: complex arg: queue // TODO: complex arg: run_as cmd.Flags().StringVar(&submitReq.RunName, "run-name", submitReq.RunName, `An optional name for the run.`) diff --git a/cmd/workspace/postgres/postgres.go b/cmd/workspace/postgres/postgres.go index 8ca99ba3ba7..fe2281b6c31 100644 --- a/cmd/workspace/postgres/postgres.go +++ b/cmd/workspace/postgres/postgres.go @@ -54,6 +54,7 @@ Use the Postgres API to create and manage Lakebase Autoscaling Postgres // Add methods cmd.AddCommand(newCreateBranch()) cmd.AddCommand(newCreateCatalog()) + cmd.AddCommand(newCreateCdfConfig()) cmd.AddCommand(newCreateDataApi()) cmd.AddCommand(newCreateDatabase()) cmd.AddCommand(newCreateEndpoint()) @@ -62,6 +63,7 @@ Use the Postgres API to create and manage Lakebase Autoscaling Postgres cmd.AddCommand(newCreateSyncedTable()) cmd.AddCommand(newDeleteBranch()) cmd.AddCommand(newDeleteCatalog()) + cmd.AddCommand(newDeleteCdfConfig()) cmd.AddCommand(newDeleteDataApi()) cmd.AddCommand(newDeleteDatabase()) cmd.AddCommand(newDeleteEndpoint()) @@ -71,6 +73,8 @@ Use the Postgres API to create and manage Lakebase Autoscaling Postgres cmd.AddCommand(newGenerateDatabaseCredential()) cmd.AddCommand(newGetBranch()) cmd.AddCommand(newGetCatalog()) + cmd.AddCommand(newGetCdfConfig()) + cmd.AddCommand(newGetCdfStatus()) cmd.AddCommand(newGetDataApi()) cmd.AddCommand(newGetDatabase()) cmd.AddCommand(newGetEndpoint()) @@ -79,6 +83,8 @@ Use the Postgres API to create and manage Lakebase Autoscaling Postgres cmd.AddCommand(newGetRole()) cmd.AddCommand(newGetSyncedTable()) cmd.AddCommand(newListBranches()) + cmd.AddCommand(newListCdfConfigs()) + cmd.AddCommand(newListCdfStatuses()) cmd.AddCommand(newListDatabases()) cmd.AddCommand(newListEndpoints()) cmd.AddCommand(newListProjects()) @@ -353,6 +359,153 @@ Register a Database in UC. return cmd } +// start create-cdf-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createCdfConfigOverrides []func( + *cobra.Command, + *postgres.CreateCdfConfigRequest, +) + +func newCreateCdfConfig() *cobra.Command { + cmd := &cobra.Command{} + + var createCdfConfigReq postgres.CreateCdfConfigRequest + createCdfConfigReq.CdfConfig = postgres.CdfConfig{} + var createCdfConfigJson flags.JsonFlag + + var createCdfConfigSkipWait bool + var createCdfConfigTimeout time.Duration + + cmd.Flags().BoolVar(&createCdfConfigSkipWait, "no-wait", createCdfConfigSkipWait, `do not wait to reach DONE state`) + cmd.Flags().DurationVar(&createCdfConfigTimeout, "timeout", 0, `maximum amount of time to reach DONE state`) + + cmd.Flags().Var(&createCdfConfigJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createCdfConfigReq.CdfConfigId, "cdf-config-id", createCdfConfigReq.CdfConfigId, `The user-specified id for the CdfConfig, forming the final segment of its resource name.`) + cmd.Flags().StringVar(&createCdfConfigReq.CdfConfig.Name, "name", createCdfConfigReq.CdfConfig.Name, `Output only.`) + + cmd.Use = "create-cdf-config PARENT CATALOG SCHEMA POSTGRES_SCHEMA" + cmd.Short = `*Beta* Create a change data feed configuration.` + cmd.Long = `This command is in Beta and may change without notice. + +Create a change data feed configuration. + + Create a CDF configuration that materializes the change data feed for all + tables in a Postgres schema as open-format Delta tables in Unity Catalog. Once + created, each table's change history is continuously written to its + corresponding Lakehouse table. + + This is a long-running operation. By default, the command waits for the + operation to complete. Use --no-wait to return immediately with the raw + operation details. The operation's 'name' field can then be used to poll for + completion using the get-operation command. + + Arguments: + PARENT: The parent database under which to create the CdfConfig. Format: + projects/{project}/branches/{branch}/databases/{database} + CATALOG: The Unity Catalog catalog that replicated tables are written into. Set at + creation; the CdfConfig is immutable. + SCHEMA: The Unity Catalog schema that replicated tables are written into. Set at + creation; the CdfConfig is immutable. + POSTGRES_SCHEMA: The Postgres schema this CdfConfig replicates from. Unique within the + parent database. Set at creation; the CdfConfig is immutable.` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return errors.New("when --json flag is specified, provide only PARENT as positional arguments. Provide 'catalog', 'schema', 'postgres_schema' in your JSON input") + } + return nil + } + check := root.ExactArgs(4) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createCdfConfigJson.Unmarshal(&createCdfConfigReq.CdfConfig) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + createCdfConfigReq.Parent = args[0] + if !cmd.Flags().Changed("json") { + createCdfConfigReq.CdfConfig.Catalog = args[1] + } + if !cmd.Flags().Changed("json") { + createCdfConfigReq.CdfConfig.Schema = args[2] + } + if !cmd.Flags().Changed("json") { + createCdfConfigReq.CdfConfig.PostgresSchema = args[3] + } + + // Determine which mode to execute based on flags. + switch { + case createCdfConfigSkipWait: + wait, err := w.Postgres.CreateCdfConfig(ctx, createCdfConfigReq) + if err != nil { + return err + } + + // Return operation immediately without waiting. + operation, err := w.Postgres.GetOperation(ctx, postgres.GetOperationRequest{ + Name: wait.Name(), + }) + if err != nil { + return err + } + return cmdio.Render(ctx, operation) + + default: + wait, err := w.Postgres.CreateCdfConfig(ctx, createCdfConfigReq) + if err != nil { + return err + } + + // Show spinner while waiting for completion. + sp := cmdio.NewSpinner(ctx) + sp.Update("Waiting for create-cdf-config to complete...") + + // Wait for completion. + opts := api.WithTimeout(createCdfConfigTimeout) + response, err := wait.Wait(ctx, opts) + if err != nil { + return err + } + sp.Close() + return cmdio.Render(ctx, response) + } + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createCdfConfigOverrides { + fn(cmd, &createCdfConfigReq) + } + + return cmd +} + // start create-data-api command // Slice with functions to override default command behavior. @@ -1328,6 +1481,114 @@ Delete a Database Catalog. return cmd } +// start delete-cdf-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteCdfConfigOverrides []func( + *cobra.Command, + *postgres.DeleteCdfConfigRequest, +) + +func newDeleteCdfConfig() *cobra.Command { + cmd := &cobra.Command{} + + var deleteCdfConfigReq postgres.DeleteCdfConfigRequest + + var deleteCdfConfigSkipWait bool + var deleteCdfConfigTimeout time.Duration + + cmd.Flags().BoolVar(&deleteCdfConfigSkipWait, "no-wait", deleteCdfConfigSkipWait, `do not wait to reach DONE state`) + cmd.Flags().DurationVar(&deleteCdfConfigTimeout, "timeout", 0, `maximum amount of time to reach DONE state`) + + cmd.Flags().BoolVar(&deleteCdfConfigReq.Force, "force", deleteCdfConfigReq.Force, `When true, also drops the replicated Delta tables in Unity Catalog.`) + + cmd.Use = "delete-cdf-config NAME" + cmd.Short = `*Beta* Delete a change data feed configuration.` + cmd.Long = `This command is in Beta and may change without notice. + +Delete a change data feed configuration. + + Delete a CDF configuration and stop materializing the change data feed. When + force=true, also drops the Delta tables in Unity Catalog. When force=false + (default), the existing tables are preserved at their last state. + + This is a long-running operation. By default, the command waits for the + operation to complete. Use --no-wait to return immediately with the raw + operation details. The operation's 'name' field can then be used to poll for + completion using the get-operation command. + + Arguments: + NAME: The resource name of the CdfConfig to delete. Format: + projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteCdfConfigReq.Name = args[0] + + // Determine which mode to execute based on flags. + switch { + case deleteCdfConfigSkipWait: + wait, err := w.Postgres.DeleteCdfConfig(ctx, deleteCdfConfigReq) + if err != nil { + return err + } + + // Return operation immediately without waiting. + operation, err := w.Postgres.GetOperation(ctx, postgres.GetOperationRequest{ + Name: wait.Name(), + }) + if err != nil { + return err + } + return cmdio.Render(ctx, operation) + + default: + wait, err := w.Postgres.DeleteCdfConfig(ctx, deleteCdfConfigReq) + if err != nil { + return err + } + + // Show spinner while waiting for completion. + sp := cmdio.NewSpinner(ctx) + sp.Update("Waiting for delete-cdf-config to complete...") + + // Wait for completion. + opts := api.WithTimeout(deleteCdfConfigTimeout) + + err = wait.Wait(ctx, opts) + if err != nil { + return err + } + sp.Close() + return nil + } + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteCdfConfigOverrides { + fn(cmd, &deleteCdfConfigReq) + } + + return cmd +} + // start delete-data-api command // Slice with functions to override default command behavior. @@ -2187,6 +2448,133 @@ Get a Database Catalog. return cmd } +// start get-cdf-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getCdfConfigOverrides []func( + *cobra.Command, + *postgres.GetCdfConfigRequest, +) + +func newGetCdfConfig() *cobra.Command { + cmd := &cobra.Command{} + + var getCdfConfigReq postgres.GetCdfConfigRequest + + cmd.Use = "get-cdf-config NAME" + cmd.Short = `*Beta* Get a change data feed configuration.` + cmd.Long = `This command is in Beta and may change without notice. + +Get a change data feed configuration. + + Get a single Lakebase CDF configuration, including the source Postgres schema, + target Unity Catalog schema, and the identity under which writes are + authorized. + + Arguments: + NAME: The resource name of the CdfConfig to retrieve. Format: + projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getCdfConfigReq.Name = args[0] + + response, err := w.Postgres.GetCdfConfig(ctx, getCdfConfigReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getCdfConfigOverrides { + fn(cmd, &getCdfConfigReq) + } + + return cmd +} + +// start get-cdf-status command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getCdfStatusOverrides []func( + *cobra.Command, + *postgres.GetCdfStatusRequest, +) + +func newGetCdfStatus() *cobra.Command { + cmd := &cobra.Command{} + + var getCdfStatusReq postgres.GetCdfStatusRequest + + cmd.Use = "get-cdf-status NAME" + cmd.Short = `*Beta* Get a change data feed status.` + cmd.Long = `This command is in Beta and may change without notice. + +Get a change data feed status. + + Get the CDF status of a single table within a Lakebase CDF configuration, + including its current state and the last committed position in the feed. + + Arguments: + NAME: The resource name of the CdfStatus to retrieve. Format: + projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}/cdf-statuses/{cdf_status}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getCdfStatusReq.Name = args[0] + + response, err := w.Postgres.GetCdfStatus(ctx, getCdfStatusReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getCdfStatusOverrides { + fn(cmd, &getCdfStatusReq) + } + + return cmd +} + // start get-data-api command // Slice with functions to override default command behavior. @@ -2701,6 +3089,166 @@ List Branches. return cmd } +// start list-cdf-configs command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listCdfConfigsOverrides []func( + *cobra.Command, + *postgres.ListCdfConfigsRequest, +) + +func newListCdfConfigs() *cobra.Command { + cmd := &cobra.Command{} + + var listCdfConfigsReq postgres.ListCdfConfigsRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listCdfConfigsLimit int + + cmd.Flags().IntVar(&listCdfConfigsReq.PageSize, "page-size", listCdfConfigsReq.PageSize, `Maximum number of CdfConfigs to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listCdfConfigsLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listCdfConfigsReq.PageToken, "page-token", listCdfConfigsReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-cdf-configs PARENT" + cmd.Short = `*Beta* List change data feed configurations.` + cmd.Long = `This command is in Beta and may change without notice. + +List change data feed configurations. + + List all CDF configurations for a Lakebase database. Each configuration maps a + Postgres schema to a Unity Catalog schema where the change data feed is + materialized. + + Arguments: + PARENT: The parent database to list CdfConfigs for. Format: + projects/{project}/branches/{branch}/databases/{database}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listCdfConfigsReq.Parent = args[0] + + response := w.Postgres.ListCdfConfigs(ctx, listCdfConfigsReq) + if listCdfConfigsLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listCdfConfigsLimit) + } + if listCdfConfigsLimit > 0 { + ctx = cmdio.WithLimit(ctx, listCdfConfigsLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listCdfConfigsOverrides { + fn(cmd, &listCdfConfigsReq) + } + + return cmd +} + +// start list-cdf-statuses command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listCdfStatusesOverrides []func( + *cobra.Command, + *postgres.ListCdfStatusesRequest, +) + +func newListCdfStatuses() *cobra.Command { + cmd := &cobra.Command{} + + var listCdfStatusesReq postgres.ListCdfStatusesRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listCdfStatusesLimit int + + cmd.Flags().IntVar(&listCdfStatusesReq.PageSize, "page-size", listCdfStatusesReq.PageSize, `Maximum number of CdfStatuses to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listCdfStatusesLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listCdfStatusesReq.PageToken, "page-token", listCdfStatusesReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-cdf-statuses PARENT" + cmd.Short = `*Beta* List change data feed statuses.` + cmd.Long = `This command is in Beta and may change without notice. + +List change data feed statuses. + + List the per-table CDF statuses within a Lakebase CDF configuration. Each + status shows whether a table's change data feed is snapshotting, streaming, or + skipped. + + Arguments: + PARENT: The parent CdfConfig to list CdfStatuses for. Format: + projects/{project}/branches/{branch}/databases/{database}/cdf-configs/{cdf_config}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listCdfStatusesReq.Parent = args[0] + + response := w.Postgres.ListCdfStatuses(ctx, listCdfStatusesReq) + if listCdfStatusesLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listCdfStatusesLimit) + } + if listCdfStatusesLimit > 0 { + ctx = cmdio.WithLimit(ctx, listCdfStatusesLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listCdfStatusesOverrides { + fn(cmd, &listCdfStatusesReq) + } + + return cmd +} + // start list-databases command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/registered-models/registered-models.go b/cmd/workspace/registered-models/registered-models.go index dea2800643d..43e7acccda1 100644 --- a/cmd/workspace/registered-models/registered-models.go +++ b/cmd/workspace/registered-models/registered-models.go @@ -94,7 +94,6 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: aliases - cmd.Flags().BoolVar(&createReq.BrowseOnly, "browse-only", createReq.BrowseOnly, `Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.`) cmd.Flags().StringVar(&createReq.CatalogName, "catalog-name", createReq.CatalogName, `The name of the catalog where the schema and the registered model reside.`) cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `The comment attached to the registered model.`) cmd.Flags().Int64Var(&createReq.CreatedAt, "created-at", createReq.CreatedAt, `Creation timestamp of the registered model in milliseconds since the Unix epoch.`) @@ -595,7 +594,6 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: aliases - cmd.Flags().BoolVar(&updateReq.BrowseOnly, "browse-only", updateReq.BrowseOnly, `Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.`) cmd.Flags().StringVar(&updateReq.CatalogName, "catalog-name", updateReq.CatalogName, `The name of the catalog where the schema and the registered model reside.`) cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `The comment attached to the registered model.`) cmd.Flags().Int64Var(&updateReq.CreatedAt, "created-at", updateReq.CreatedAt, `Creation timestamp of the registered model in milliseconds since the Unix epoch.`) diff --git a/cmd/workspace/repos/repos.go b/cmd/workspace/repos/repos.go index 174d5850e49..2c6877823ea 100644 --- a/cmd/workspace/repos/repos.go +++ b/cmd/workspace/repos/repos.go @@ -479,7 +479,12 @@ func newList() *cobra.Command { cmd.Long = `Get repos. Returns repos that the calling user has Manage permissions on. Use - next_page_token to iterate through additional pages.` + next_page_token to iterate through additional pages. + + Deprecated: This operation does not return a complete list of the repos in the + workspace, because repos with the Git CLI enabled are not included in its + results. Instead, use the Repos and Workspace APIs to find repos and their + associated metadata in the workspace.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" diff --git a/cmd/workspace/rfa/rfa.go b/cmd/workspace/rfa/rfa.go index 4c2ba7f0e25..062a6b0628e 100644 --- a/cmd/workspace/rfa/rfa.go +++ b/cmd/workspace/rfa/rfa.go @@ -231,13 +231,10 @@ Update Access Request Destinations. Updates the access request destinations for the given securable. The caller must be a metastore admin, the owner of the securable, or a user that has the - **MANAGE** privilege on the securable in order to assign destinations. - Destinations cannot be updated for securables underneath schemas (tables, - volumes, functions, and models). For these securable types, destinations are - inherited from the parent securable. A maximum of 5 emails and 5 external - notification destinations (Slack, Microsoft Teams, and Generic Webhook - destinations) can be assigned to a securable. If a URL destination is - assigned, no other destinations can be set. + **MANAGE** privilege on the securable in order to assign destinations. A + maximum of 5 emails and 5 external notification destinations (Slack, Microsoft + Teams, and Generic Webhook destinations) can be assigned to a securable. If a + URL destination is assigned, no other destinations can be set. The supported securable types are: "metastore", "catalog", "schema", "table", "external_location", "connection", "credential", "function", diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index 0bb70086164..7c837c5d2dd 100644 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -58,6 +58,7 @@ func New() *cobra.Command { cmd.AddCommand(newList()) cmd.AddCommand(newLogs()) cmd.AddCommand(newPatch()) + cmd.AddCommand(newPatchTelemetryConfig()) cmd.AddCommand(newPut()) cmd.AddCommand(newPutAiGateway()) cmd.AddCommand(newQuery()) @@ -970,6 +971,83 @@ func newPatch() *cobra.Command { return cmd } +// start patch-telemetry-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var patchTelemetryConfigOverrides []func( + *cobra.Command, + *serving.PatchTelemetryConfigRequest, +) + +func newPatchTelemetryConfig() *cobra.Command { + cmd := &cobra.Command{} + + var patchTelemetryConfigReq serving.PatchTelemetryConfigRequest + var patchTelemetryConfigJson flags.JsonFlag + + cmd.Flags().Var(&patchTelemetryConfigJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: telemetry_config + + cmd.Use = "patch-telemetry-config NAME" + cmd.Short = `Update the telemetry config of a serving endpoint.` + cmd.Long = `Update the telemetry config of a serving endpoint. + + Updates the telemetry configuration of a serving endpoint. + + Arguments: + NAME: The name of the serving endpoint whose telemetry configuration is being + updated. This field is required.` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "GA" + cmd.Annotations["launch_stage_display"] = "GA" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := patchTelemetryConfigJson.Unmarshal(&patchTelemetryConfigReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + patchTelemetryConfigReq.Name = args[0] + + response, err := w.ServingEndpoints.PatchTelemetryConfig(ctx, patchTelemetryConfigReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range patchTelemetryConfigOverrides { + fn(cmd, &patchTelemetryConfigReq) + } + + return cmd +} + // start put command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/supervisor-agents/supervisor-agents.go b/cmd/workspace/supervisor-agents/supervisor-agents.go index f5532315c24..3e02f84325c 100644 --- a/cmd/workspace/supervisor-agents/supervisor-agents.go +++ b/cmd/workspace/supervisor-agents/supervisor-agents.go @@ -289,9 +289,11 @@ Create a Tool. Creates a Tool under a Supervisor Agent. Specify one of "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "app", "volume", "dashboard", "table", "vector_search_index", "catalog", "schema", - "supervisor_agent", "web_search", "skill" in the request body. The legacy - values "lakeview_dashboard" and "uc_table" are also accepted and remain - equivalent to "dashboard" and "table" respectively. + "supervisor_agent", "databricks_web_search", "skill" in the request body. The + legacy values "lakeview_dashboard", "uc_table", and "web_search" are also + accepted and remain equivalent to "dashboard", "table", and + "databricks_web_search" respectively. The "databricks_web_search" tool_type + maps to the web_search spec field. Arguments: PARENT: Parent resource where this tool will be created. Format: @@ -301,9 +303,11 @@ Create a Tool. TOOL_TYPE: Tool type. Must be one of: "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "uc_mcp", "app", "volume", "dashboard", "serving_endpoint", "table", "vector_search_index", "catalog", "schema", - "supervisor_agent", "web_search", "skill". The legacy values - "lakeview_dashboard" and "uc_table" are also accepted and remain - equivalent to "dashboard" and "table" respectively.` + "supervisor_agent", "databricks_web_search", "skill". The legacy values + "lakeview_dashboard", "uc_table", and "web_search" are also accepted and + remain equivalent to "dashboard", "table", and "databricks_web_search" + respectively. The "databricks_web_search" tool_type maps to the + web_search spec field.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_BETA" @@ -1491,9 +1495,11 @@ Update a Tool. TOOL_TYPE: Tool type. Must be one of: "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "uc_mcp", "app", "volume", "dashboard", "serving_endpoint", "table", "vector_search_index", "catalog", "schema", - "supervisor_agent", "web_search", "skill". The legacy values - "lakeview_dashboard" and "uc_table" are also accepted and remain - equivalent to "dashboard" and "table" respectively.` + "supervisor_agent", "databricks_web_search", "skill". The legacy values + "lakeview_dashboard", "uc_table", and "web_search" are also accepted and + remain equivalent to "dashboard", "table", and "databricks_web_search" + respectively. The "databricks_web_search" tool_type maps to the + web_search spec field.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_BETA" diff --git a/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go b/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go index d028c27a185..ed0be0fb600 100644 --- a/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go +++ b/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go @@ -73,6 +73,7 @@ func newCreateWorkspaceAssignmentDetailProxy() *cobra.Command { cmd.Flags().Var(&createWorkspaceAssignmentDetailProxyJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: effective_entitlements // TODO: array: entitlements cmd.Use = "create-workspace-assignment-detail-proxy PRINCIPAL_ID" @@ -704,6 +705,7 @@ func newUpdateWorkspaceAssignmentDetailProxy() *cobra.Command { cmd.Flags().Var(&updateWorkspaceAssignmentDetailProxyJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: effective_entitlements // TODO: array: entitlements cmd.Use = "update-workspace-assignment-detail-proxy PRINCIPAL_ID UPDATE_MASK PRINCIPAL_ID" diff --git a/experimental/air/cmd/snapshot.go b/experimental/air/cmd/snapshot.go index aba67f6109c..59041f5986d 100644 --- a/experimental/air/cmd/snapshot.go +++ b/experimental/air/cmd/snapshot.go @@ -42,7 +42,7 @@ func snapshotCodeSource(ctx context.Context, w *databricks.WorkspaceClient, snap return snapshotResult{}, err } - up, err := newSnapshotUploader(w, snap, userDir, funcDir, filepath.Base(repoPath)) + up, err := newSnapshotUploader(ctx, w, snap, userDir, funcDir, filepath.Base(repoPath)) if err != nil { return snapshotResult{}, err } @@ -255,7 +255,7 @@ func fileExists(ctx context.Context, store filer.Filer, name string) (bool, erro // newSnapshotUploader builds the uploader for a submission. The tarball store is a // Volume (when remote_volume is set) or the user's repo_snapshots workspace dir; // sidecars always go to the run's funcDir in the workspace. -func newSnapshotUploader(w *databricks.WorkspaceClient, snap *snapshotSourceConfig, userDir, funcDir, dirName string) (snapshotUploader, error) { +func newSnapshotUploader(ctx context.Context, w *databricks.WorkspaceClient, snap *snapshotSourceConfig, userDir, funcDir, dirName string) (snapshotUploader, error) { sidecarStore, err := filer.NewWorkspaceFilesClient(w, funcDir) if err != nil { return snapshotUploader{}, err @@ -263,7 +263,7 @@ func newSnapshotUploader(w *databricks.WorkspaceClient, snap *snapshotSourceConf if snap.RemoteVolume != nil { tarBase := strings.TrimRight(*snap.RemoteVolume, "/") - tarStore, err := filer.NewFilesClient(w, tarBase) + tarStore, err := filer.NewFilesClient(ctx, w, tarBase) if err != nil { return snapshotUploader{}, err } diff --git a/experimental/genie/cmd/genie.go b/experimental/genie/cmd/genie.go deleted file mode 100644 index fb079e43e97..00000000000 --- a/experimental/genie/cmd/genie.go +++ /dev/null @@ -1,16 +0,0 @@ -package geniecmd - -import "github.com/spf13/cobra" - -// NewGenieCmd creates the parent "genie" command group. -func NewGenieCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "genie", - Short: "Ask data questions via Databricks Genie", - Hidden: true, - } - - cmd.AddCommand(newAskCmd()) - - return cmd -} diff --git a/experimental/ssh/cmd/connect.go b/experimental/ssh/cmd/connect.go index 972ccf4a81a..2c50b871902 100644 --- a/experimental/ssh/cmd/connect.go +++ b/experimental/ssh/cmd/connect.go @@ -41,6 +41,7 @@ Connect to a dedicated cluster: var environmentVersion int var baseEnvironment string var autoApprove bool + var usagePolicyID string cmd.Flags().StringVar(&clusterID, "cluster", "", "Databricks dedicated cluster ID") cmd.Flags().DurationVar(&shutdownDelay, "shutdown-delay", defaultShutdownDelay, "Delay before shutting down the server after the last client disconnects") @@ -50,6 +51,7 @@ Connect to a dedicated cluster: cmd.Flags().StringVar(&connectionName, "name", "", "Connection name to reuse across sessions (serverless only)") cmd.Flags().StringVar(&accelerator, "accelerator", "", "Serverless GPU accelerator type (GPU_1xA10 or GPU_8xH100)") cmd.Flags().StringVar(&ide, "ide", "", "Open remote IDE window (vscode or cursor)") + cmd.Flags().StringVar(&usagePolicyID, "usage-policy-id", "", "Usage policy ID for the serverless SSH server job (serverless only)") cmd.Flags().BoolVar(&proxyMode, "proxy", false, "ProxyCommand mode") cmd.Flags().MarkHidden("proxy") @@ -130,6 +132,7 @@ Connect to a dedicated cluster: BaseEnvironment: baseEnvironment, AdditionalArgs: args, AutoApprove: autoApprove, + UsagePolicyID: usagePolicyID, } if err := opts.Validate(); err != nil { return err diff --git a/experimental/ssh/cmd/server.go b/experimental/ssh/cmd/server.go index 47e16cdc649..21c651b2365 100644 --- a/experimental/ssh/cmd/server.go +++ b/experimental/ssh/cmd/server.go @@ -29,6 +29,7 @@ and proxies them to local SSH daemon processes.`, var secretScopeName string var authorizedKeySecretName string var serverless bool + var usagePolicyID string cmd.Flags().StringVar(&clusterID, "cluster", "", "Databricks cluster ID") cmd.MarkFlagRequired("cluster") @@ -43,6 +44,7 @@ and proxies them to local SSH daemon processes.`, cmd.Flags().DurationVar(&shutdownDelay, "shutdown-delay", defaultShutdownDelay, "Delay before shutting down after no pings from clients") cmd.Flags().StringVar(&version, "version", "", "Client version of the Databricks CLI") cmd.Flags().BoolVar(&serverless, "serverless", false, "Enable serverless mode for Jupyter initialization") + cmd.Flags().StringVar(&usagePolicyID, "usage-policy-id", "", "Usage policy ID the job was submitted with") cmd.PreRunE = func(cmd *cobra.Command, args []string) error { // The server can be executed under a directory with an invalid bundle configuration. @@ -71,6 +73,7 @@ and proxies them to local SSH daemon processes.`, DefaultPort: defaultServerPort, PortRange: serverPortRange, Serverless: serverless, + UsagePolicyID: usagePolicyID, } return server.Run(ctx, wsc, opts) } diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 3cecb9c30ed..97d5245c181 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -119,6 +119,8 @@ type ClientOptions struct { BaseEnvironment string // If true, skip confirmation prompts for IDE extension install and IDE settings updates. AutoApprove bool + // Id of the usage policy to use for the serverless SSH server job. Serverless only. + UsagePolicyID string } func (o *ClientOptions) Validate() error { @@ -128,6 +130,9 @@ func (o *ClientOptions) Validate() error { if o.Accelerator != "" && o.ConnectionName == "" { return errors.New("--accelerator flag can only be used with serverless compute (--name flag)") } + if o.UsagePolicyID != "" && o.ClusterID != "" { + return errors.New("--usage-policy-id flag can only be used with serverless compute (--name flag)") + } if o.Accelerator != "" && o.Accelerator != "GPU_1xA10" && o.Accelerator != "GPU_8xH100" { return fmt.Errorf("invalid accelerator value: %q, expected %q or %q", o.Accelerator, "GPU_1xA10", "GPU_8xH100") } @@ -214,6 +219,9 @@ func (o *ClientOptions) ToProxyCommand() (string, error) { if o.Accelerator != "" { proxyCommand += " --accelerator=" + o.Accelerator } + if o.UsagePolicyID != "" { + proxyCommand += " --usage-policy-id=" + o.UsagePolicyID + } } else { proxyCommand = fmt.Sprintf("%q ssh connect --proxy --cluster=%s --auto-start-cluster=%t --shutdown-delay=%s", executablePath, o.ClusterID, o.AutoStartCluster, o.ShutdownDelay.String()) @@ -473,14 +481,25 @@ func ensureSSHConfigEntry(ctx context.Context, configPath, hostName, userName, k return nil } +// serverMetadata describes a running SSH server, combining the persisted workspace +// metadata with the user name validated live via Driver Proxy. +type serverMetadata struct { + Port int + UserName string + // ClusterID required for Driver Proxy connections. For serverless it comes from the persisted metadata. + ClusterID string + // UsagePolicyID the server was started with, used to decide whether a running server can be reused. + UsagePolicyID string +} + // getServerMetadata retrieves the server metadata from the workspace and validates it via Driver Proxy. // sessionID is the unique identifier for the session (cluster ID for dedicated clusters, connection name for serverless). // For dedicated clusters, clusterID should be the same as sessionID. // For serverless, clusterID is read from the workspace metadata. -func getServerMetadata(ctx context.Context, client *databricks.WorkspaceClient, sessionID, clusterID, version, liteswap string) (int, string, string, error) { +func getServerMetadata(ctx context.Context, client *databricks.WorkspaceClient, sessionID, clusterID, version, liteswap string) (serverMetadata, error) { wsMetadata, err := sshWorkspace.GetWorkspaceMetadata(ctx, client, version, sessionID) if err != nil { - return 0, "", "", errors.Join(errServerMetadata, err) + return serverMetadata{}, errors.Join(errServerMetadata, err) } log.Debugf(ctx, "Workspace metadata: %+v", wsMetadata) @@ -491,33 +510,38 @@ func getServerMetadata(ctx context.Context, client *databricks.WorkspaceClient, } if effectiveClusterID == "" { - return 0, "", "", errors.Join(errServerMetadata, errors.New("cluster ID not available in metadata")) + return serverMetadata{}, errors.Join(errServerMetadata, errors.New("cluster ID not available in metadata")) } req, err := newDriverProxyRequest(ctx, client, effectiveClusterID, wsMetadata.Port, "metadata", liteswap) if err != nil { - return 0, "", "", err + return serverMetadata{}, err } log.Debugf(ctx, "Metadata URL: %s", req.URL) httpClient := &http.Client{Transport: client.Config.HTTPTransport} resp, err := httpClient.Do(req) if err != nil { - return 0, "", "", err + return serverMetadata{}, err } defer resp.Body.Close() bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - return 0, "", "", err + return serverMetadata{}, err } log.Debugf(ctx, "Metadata response: %s", string(bodyBytes)) log.Debugf(ctx, "Metadata response status code: %d", resp.StatusCode) if resp.StatusCode != http.StatusOK { - return 0, "", "", errors.Join(errServerMetadata, fmt.Errorf("server is not ok, status code %d", resp.StatusCode)) + return serverMetadata{}, errors.Join(errServerMetadata, fmt.Errorf("server is not ok, status code %d", resp.StatusCode)) } - return wsMetadata.Port, string(bodyBytes), effectiveClusterID, nil + return serverMetadata{ + Port: wsMetadata.Port, + UserName: string(bodyBytes), + ClusterID: effectiveClusterID, + UsagePolicyID: wsMetadata.UsagePolicyID, + }, nil } // newDriverProxyRequest builds an authenticated GET request to one of the SSH server's @@ -569,36 +593,10 @@ func fetchServerErrorLogs(ctx context.Context, client *databricks.WorkspaceClien return strings.TrimSpace(string(body)) } -// submitSSHTunnelJob submits the bootstrap job and waits for the SSH server task to start. -// It returns the job run ID (when known) so callers can fetch and surface the run's error -// details if the server never comes up. -func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, version, secretScopeName string, opts ClientOptions) (int64, error) { +// Assemble the SubmitRun request that bootstraps the SSH server. +// Extracted from submitSSHTunnelJob so this logic can be unit tested. +func buildSSHServerSubmitRun(version, secretScopeName, jobNotebookPath, baseEnvironment string, opts ClientOptions) jobs.SubmitRun { sessionID := opts.SessionIdentifier() - contentDir, err := sshWorkspace.GetWorkspaceContentDir(ctx, client, version, sessionID) - if err != nil { - return 0, fmt.Errorf("failed to get workspace content directory: %w", err) - } - - err = client.Workspace.MkdirsByPath(ctx, contentDir) - if err != nil { - return 0, fmt.Errorf("failed to create directory in the remote workspace: %w", err) - } - - sshTunnelJobName := "ssh-server-bootstrap-" + sessionID - jobNotebookPath := filepath.ToSlash(filepath.Join(contentDir, "ssh-server-bootstrap")) - notebookContent := "# Databricks notebook source\n" + sshServerBootstrapScript - encodedContent := base64.StdEncoding.EncodeToString([]byte(notebookContent)) - - err = client.Workspace.Import(ctx, workspace.Import{ - Path: jobNotebookPath, - Format: workspace.ImportFormatSource, - Content: encodedContent, - Language: workspace.LanguagePython, - Overwrite: true, - }) - if err != nil { - return 0, fmt.Errorf("failed to create ssh-tunnel notebook: %w", err) - } baseParams := map[string]string{ "version": version, @@ -608,10 +606,11 @@ func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, "maxClients": strconv.Itoa(opts.MaxClients), "sessionId": sessionID, "serverless": strconv.FormatBool(opts.IsServerlessMode()), + // Recorded in the server's metadata.json so reconnects can tell which usage policy + // the running server was started under. + "usagePolicyId": opts.UsagePolicyID, } - log.Infof(ctx, "Submitting a job to start the ssh server...") - task := jobs.SubmitTask{ TaskKey: sshServerTaskKey, NotebookTask: &jobs.NotebookTask{ @@ -624,7 +623,6 @@ func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, if opts.IsServerlessMode() { task.EnvironmentKey = serverlessEnvironmentKey if opts.Accelerator != "" { - log.Infof(ctx, "Using accelerator: %s", opts.Accelerator) task.Compute = &jobs.Compute{ HardwareAccelerator: compute.HardwareAcceleratorType(opts.Accelerator), } @@ -634,20 +632,17 @@ func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, } submitRequest := jobs.SubmitRun{ - RunName: sshTunnelJobName, + RunName: "ssh-server-bootstrap-" + sessionID, TimeoutSeconds: int(opts.ServerTimeout.Seconds()), Tasks: []jobs.SubmitTask{task}, + BudgetPolicyId: opts.UsagePolicyID, } if opts.IsServerlessMode() { // base_environment and environment_version are mutually exclusive: a custom // base environment carries its own version, so we don't also set one. var spec compute.Environment - if opts.BaseEnvironment != "" { - baseEnvironment, err := resolveBaseEnvironment(ctx, client, opts.BaseEnvironment) - if err != nil { - return 0, err - } + if baseEnvironment != "" { spec.BaseEnvironment = baseEnvironment } else { spec.EnvironmentVersion = strconv.Itoa(max(opts.EnvironmentVersion, minEnvironmentVersion)) @@ -660,6 +655,54 @@ func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, } } + return submitRequest +} + +// submitSSHTunnelJob submits the bootstrap job and waits for the SSH server task to start. +// It returns the job run ID (when known) so callers can fetch and surface the run's error +// details if the server never comes up. +func submitSSHTunnelJob(ctx context.Context, client *databricks.WorkspaceClient, version, secretScopeName string, opts ClientOptions) (int64, error) { + sessionID := opts.SessionIdentifier() + contentDir, err := sshWorkspace.GetWorkspaceContentDir(ctx, client, version, sessionID) + if err != nil { + return 0, fmt.Errorf("failed to get workspace content directory: %w", err) + } + + err = client.Workspace.MkdirsByPath(ctx, contentDir) + if err != nil { + return 0, fmt.Errorf("failed to create directory in the remote workspace: %w", err) + } + + jobNotebookPath := filepath.ToSlash(filepath.Join(contentDir, "ssh-server-bootstrap")) + notebookContent := "# Databricks notebook source\n" + sshServerBootstrapScript + encodedContent := base64.StdEncoding.EncodeToString([]byte(notebookContent)) + + err = client.Workspace.Import(ctx, workspace.Import{ + Path: jobNotebookPath, + Format: workspace.ImportFormatSource, + Content: encodedContent, + Language: workspace.LanguagePython, + Overwrite: true, + }) + if err != nil { + return 0, fmt.Errorf("failed to create ssh-tunnel notebook: %w", err) + } + + log.Infof(ctx, "Submitting a job to start the ssh server...") + if opts.IsServerlessMode() && opts.Accelerator != "" { + log.Infof(ctx, "Using accelerator: %s", opts.Accelerator) + } + + var baseEnvironment string + if opts.IsServerlessMode() && opts.BaseEnvironment != "" { + baseEnvironment, err = resolveBaseEnvironment(ctx, client, opts.BaseEnvironment) + if err != nil { + return 0, err + } + } + + submitRequest := buildSSHServerSubmitRun(version, secretScopeName, jobNotebookPath, baseEnvironment, opts) + waiter, err := client.Jobs.Submit(ctx, submitRequest) if err != nil { return 0, fmt.Errorf("failed to submit job: %w", err) @@ -1103,18 +1146,31 @@ func hostKeyChangedHint(stderr, hostName, knownHostsFile string) string { "Remove the stale entry and reconnect:\n " + cmd } +func usagePolicyMatches(storedPolicy, requestedPolicy string) bool { + return requestedPolicy == "" || storedPolicy == requestedPolicy +} + func ensureSSHServerIsRunning(ctx context.Context, client *databricks.WorkspaceClient, version, secretScopeName string, opts ClientOptions) (string, int, string, error) { sessionID := opts.SessionIdentifier() // For dedicated clusters, use clusterID; for serverless, it will be read from metadata clusterID := opts.ClusterID - serverPort, userName, effectiveClusterID, err := getServerMetadata(ctx, client, sessionID, clusterID, version, opts.Liteswap) - if errors.Is(err, errServerMetadata) { + meta, err := getServerMetadata(ctx, client, sessionID, clusterID, version, opts.Liteswap) + if err != nil && !errors.Is(err, errServerMetadata) { + return "", 0, "", err + } + + // Start a new server when none is running, or when the running one was started under a + // different usage policy. A job's usage policy is fixed at submission, so we can't retarget + // the existing server; the new server overwrites metadata.json and the old one idles out via + // shutdownDelay. + needNewServer := err != nil || !usagePolicyMatches(meta.UsagePolicyID, opts.UsagePolicyID) + if needNewServer { cmdio.LogString(ctx, "Starting SSH server...") - runID, err := submitSSHTunnelJob(ctx, client, version, secretScopeName, opts) - if err != nil { - return "", 0, "", fmt.Errorf("failed to submit and start ssh server job: %w", err) + runID, submitErr := submitSSHTunnelJob(ctx, client, version, secretScopeName, opts) + if submitErr != nil { + return "", 0, "", fmt.Errorf("failed to submit and start ssh server job: %w", submitErr) } sp := cmdio.NewSpinner(ctx, cmdio.WithElapsedTime()) @@ -1125,7 +1181,13 @@ func ensureSSHServerIsRunning(ctx context.Context, client *databricks.WorkspaceC if ctx.Err() != nil { return "", 0, "", ctx.Err() } - serverPort, userName, effectiveClusterID, err = getServerMetadata(ctx, client, sessionID, clusterID, version, opts.Liteswap) + meta, err = getServerMetadata(ctx, client, sessionID, clusterID, version, opts.Liteswap) + // Accept only once metadata reflects the requested usage policy, so we don't latch + // onto a server a previous connection started under a different policy before our new + // server has overwritten metadata.json. + if err == nil && !usagePolicyMatches(meta.UsagePolicyID, opts.UsagePolicyID) { + err = fmt.Errorf("found a running SSH server with usage policy %q, waiting for the one with %q", meta.UsagePolicyID, opts.UsagePolicyID) + } if err == nil { cmdio.LogString(ctx, "Health check successful, starting ssh WebSocket connection...") break @@ -1142,14 +1204,20 @@ func ensureSSHServerIsRunning(ctx context.Context, client *databricks.WorkspaceC return "", 0, "", fmt.Errorf("failed to start the ssh server: %w\n%s", err, describeRunFailure(ctx, client, runID)) } } - } else if err != nil { - return "", 0, "", err } - return userName, serverPort, effectiveClusterID, nil + return meta.UserName, meta.Port, meta.ClusterID, nil } func logSshTunnelEvent(ctx context.Context, opts ClientOptions, isSuccess, isReconnect bool, serverStartTimeMs int64) { + telemetry.Log(ctx, protos.DatabricksCliLog{ + SshTunnelEvent: buildSshTunnelEvent(opts, isSuccess, isReconnect, serverStartTimeMs), + }) +} + +// buildSshTunnelEvent maps the connection options and outcome onto the telemetry +// event. It is separated from logSshTunnelEvent so the field mapping can be unit tested. +func buildSshTunnelEvent(opts ClientOptions, isSuccess, isReconnect bool, serverStartTimeMs int64) *protos.SshTunnelEvent { computeType := protos.SshTunnelComputeTypeDedicated if opts.IsServerlessMode() { computeType = protos.SshTunnelComputeTypeServerless @@ -1165,16 +1233,16 @@ func logSshTunnelEvent(ctx context.Context, opts ClientOptions, isSuccess, isRec clientMode = protos.SshTunnelClientModeSSH } - telemetry.Log(ctx, protos.DatabricksCliLog{ - SshTunnelEvent: &protos.SshTunnelEvent{ - ComputeType: computeType, - AcceleratorType: opts.Accelerator, - IdeType: opts.IDE, - ClientMode: clientMode, - IsReconnect: isReconnect, - AutoStartCluster: opts.AutoStartCluster, - ServerStartTimeMs: serverStartTimeMs, - IsSuccess: isSuccess, - }, - }) + return &protos.SshTunnelEvent{ + ComputeType: computeType, + AcceleratorType: opts.Accelerator, + IdeType: opts.IDE, + ClientMode: clientMode, + IsReconnect: isReconnect, + AutoStartCluster: opts.AutoStartCluster, + ServerStartTimeMs: serverStartTimeMs, + IsSuccess: isSuccess, + HasBaseEnvironment: opts.BaseEnvironment != "", + HasUsagePolicy: opts.UsagePolicyID != "", + } } diff --git a/experimental/ssh/internal/client/client_internal_test.go b/experimental/ssh/internal/client/client_internal_test.go index 57861eb25cd..f71f38d0898 100644 --- a/experimental/ssh/internal/client/client_internal_test.go +++ b/experimental/ssh/internal/client/client_internal_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/environments" @@ -416,3 +417,68 @@ func TestTailWriterRetainsTail(t *testing.T) { assert.Equal(t, "ab", w.String()) }) } + +func TestBuildSshTunnelEvent(t *testing.T) { + tests := []struct { + name string + opts ClientOptions + want protos.SshTunnelEvent + }{ + { + name: "dedicated cluster via raw SSH client", + opts: ClientOptions{ClusterID: "abc-123", AutoStartCluster: true}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeDedicated, + ClientMode: protos.SshTunnelClientModeSSH, + AutoStartCluster: true, + }, + }, + { + name: "serverless with accelerator", + opts: ClientOptions{ConnectionName: "my-conn", Accelerator: "GPU_1xA10"}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeServerless, + AcceleratorType: "GPU_1xA10", + ClientMode: protos.SshTunnelClientModeSSH, + }, + }, + { + name: "proxy mode takes precedence over IDE", + opts: ClientOptions{ConnectionName: "my-conn", ProxyMode: true, IDE: "vscode"}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeServerless, + IdeType: "vscode", + ClientMode: protos.SshTunnelClientModeProxy, + }, + }, + { + name: "IDE mode", + opts: ClientOptions{ConnectionName: "my-conn", IDE: "cursor"}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeServerless, + IdeType: "cursor", + ClientMode: protos.SshTunnelClientModeIDE, + }, + }, + { + // The raw --base-environment value can carry PII, so only its presence is recorded. + name: "custom base environment records presence only", + opts: ClientOptions{ConnectionName: "my-conn", BaseEnvironment: "/Workspace/Users/me@example.com/env.yaml"}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeServerless, + ClientMode: protos.SshTunnelClientModeSSH, + HasBaseEnvironment: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildSshTunnelEvent(tt.opts, true, true, 1500) + tt.want.IsSuccess = true + tt.want.IsReconnect = true + tt.want.ServerStartTimeMs = 1500 + assert.Equal(t, &tt.want, got) + }) + } +} diff --git a/experimental/ssh/internal/client/client_test.go b/experimental/ssh/internal/client/client_test.go index 48fb6f0c1f4..09fd1067d4f 100644 --- a/experimental/ssh/internal/client/client_test.go +++ b/experimental/ssh/internal/client/client_test.go @@ -112,6 +112,15 @@ func TestValidate(t *testing.T) { name: "base environment with serverless GPU accelerator", opts: client.ClientOptions{ConnectionName: "my-conn", Accelerator: "GPU_1xA10", BaseEnvironment: "my-gpu-env"}, }, + { + name: "usage policy with cluster ID", + opts: client.ClientOptions{ClusterID: "abc-123", UsagePolicyID: "pol-1"}, + wantErr: "--usage-policy-id flag can only be used with serverless compute (--name flag)", + }, + { + name: "usage policy with connection name", + opts: client.ClientOptions{ConnectionName: "my-conn", UsagePolicyID: "pol-1"}, + }, } for _, tt := range tests { @@ -233,6 +242,11 @@ func TestToProxyCommand(t *testing.T) { opts: client.ClientOptions{ConnectionName: "my-conn", Accelerator: "GPU_1xA10", ShutdownDelay: 2 * time.Minute}, want: quoted + " ssh connect --proxy --name=my-conn --shutdown-delay=2m0s --accelerator=GPU_1xA10", }, + { + name: "serverless with usage policy", + opts: client.ClientOptions{ConnectionName: "my-conn", UsagePolicyID: "pol-1", ShutdownDelay: 2 * time.Minute}, + want: quoted + " ssh connect --proxy --name=my-conn --shutdown-delay=2m0s --usage-policy-id=pol-1", + }, { name: "with metadata", opts: client.ClientOptions{ClusterID: "abc-123", ServerMetadata: "user,2222,abc-123"}, diff --git a/experimental/ssh/internal/client/policy_internal_test.go b/experimental/ssh/internal/client/policy_internal_test.go new file mode 100644 index 00000000000..f501f0ba6e6 --- /dev/null +++ b/experimental/ssh/internal/client/policy_internal_test.go @@ -0,0 +1,26 @@ +package client + +import "testing" + +func TestUsagePolicyMatches(t *testing.T) { + tests := []struct { + name string + stored string + requested string + want bool + }{ + {name: "empty request matches any server", stored: "pol-1", requested: "", want: true}, + {name: "empty request matches server without policy", stored: "", requested: "", want: true}, + {name: "equal policies match", stored: "pol-1", requested: "pol-1", want: true}, + {name: "different policies do not match", stored: "pol-1", requested: "pol-2", want: false}, + {name: "request against server without policy does not match", stored: "", requested: "pol-1", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := usagePolicyMatches(tt.stored, tt.requested); got != tt.want { + t.Errorf("usagePolicyMatches(%q, %q) = %v, want %v", tt.stored, tt.requested, got, tt.want) + } + }) + } +} diff --git a/experimental/ssh/internal/client/ssh-server-bootstrap.py b/experimental/ssh/internal/client/ssh-server-bootstrap.py index 87d0d2756fe..28a20f73688 100644 --- a/experimental/ssh/internal/client/ssh-server-bootstrap.py +++ b/experimental/ssh/internal/client/ssh-server-bootstrap.py @@ -26,6 +26,7 @@ dbutils.widgets.text("shutdownDelay", "10m") dbutils.widgets.text("sessionId", "") dbutils.widgets.text("serverless", "false") +dbutils.widgets.text("usagePolicyId", "") def cleanup(): @@ -126,6 +127,7 @@ def run_ssh_server(): if not session_id: raise RuntimeError("Session ID is required. Please provide it using the 'sessionId' widget.") serverless = dbutils.widgets.get("serverless") + usage_policy_id = dbutils.widgets.get("usagePolicyId") # Mark this process's WSFS command origin so workspace-file activity from the # remote SSH session is attributable @@ -172,6 +174,10 @@ def run_ssh_server(): "--log-file=stdout", ] + # Recorded in the server's metadata.json so reconnects can match the usage policy. + if usage_policy_id: + server_args.append(f"--usage-policy-id={usage_policy_id}") + # Tee the server output instead of inheriting stdout: the run-page logs remain the only # place to debug a RUNNING server, but on failure we attach the log tail to the exception # so "ssh connect" can print it (the Jobs run-output API has no stdout logs for notebook tasks). diff --git a/experimental/ssh/internal/client/submit_internal_test.go b/experimental/ssh/internal/client/submit_internal_test.go new file mode 100644 index 00000000000..5b1af006b1d --- /dev/null +++ b/experimental/ssh/internal/client/submit_internal_test.go @@ -0,0 +1,76 @@ +package client + +import ( + "testing" + "time" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildSSHServerSubmitRun(t *testing.T) { + const notebookPath = "/Workspace/Users/me/.databricks/ssh-tunnel/v1/conn/ssh-server-bootstrap" + + t.Run("serverless with usage policy", func(t *testing.T) { + opts := ClientOptions{ + ConnectionName: "conn", + UsagePolicyID: "pol-1", + ServerTimeout: time.Hour, + EnvironmentVersion: 4, + } + got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "", opts) + + // Usage policy flows onto the run and into the base params the server reads. + assert.Equal(t, "pol-1", got.BudgetPolicyId) + assert.Equal(t, "pol-1", got.Tasks[0].NotebookTask.BaseParameters["usagePolicyId"]) + assert.Equal(t, "true", got.Tasks[0].NotebookTask.BaseParameters["serverless"]) + + // Serverless runs on an environment, not an existing cluster. + assert.Equal(t, serverlessEnvironmentKey, got.Tasks[0].EnvironmentKey) + assert.Empty(t, got.Tasks[0].ExistingClusterId) + assert.Len(t, got.Environments, 1) + assert.Nil(t, got.Tasks[0].Compute) + }) + + t.Run("serverless with accelerator", func(t *testing.T) { + opts := ClientOptions{ + ConnectionName: "conn", + Accelerator: "GPU_1xA10", + ServerTimeout: time.Hour, + } + got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "", opts) + + assert.Equal(t, compute.HardwareAcceleratorType("GPU_1xA10"), got.Tasks[0].Compute.HardwareAccelerator) + }) + + t.Run("serverless with base environment", func(t *testing.T) { + opts := ClientOptions{ + ConnectionName: "conn", + ServerTimeout: time.Hour, + EnvironmentVersion: 4, + BaseEnvironment: "my-env", + } + got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "workspace-base-environments/dbe_123", opts) + + // A resolved base environment carries its own version, so environment_version is not set. + require.Len(t, got.Environments, 1) + assert.Equal(t, "workspace-base-environments/dbe_123", got.Environments[0].Spec.BaseEnvironment) + assert.Empty(t, got.Environments[0].Spec.EnvironmentVersion) + }) + + t.Run("dedicated cluster", func(t *testing.T) { + opts := ClientOptions{ + ClusterID: "abc-123", + ServerTimeout: time.Hour, + } + got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "", opts) + + // Usage policy is serverless-only; a dedicated run carries none and targets the cluster. + assert.Empty(t, got.BudgetPolicyId) + assert.Empty(t, got.Tasks[0].NotebookTask.BaseParameters["usagePolicyId"]) + assert.Equal(t, "abc-123", got.Tasks[0].ExistingClusterId) + assert.Empty(t, got.Tasks[0].EnvironmentKey) + assert.Empty(t, got.Environments) + }) +} diff --git a/experimental/ssh/internal/server/server.go b/experimental/ssh/internal/server/server.go index 74a3a00e503..4356700d87d 100644 --- a/experimental/ssh/internal/server/server.go +++ b/experimental/ssh/internal/server/server.go @@ -41,6 +41,9 @@ type ServerOptions struct { SessionID string // Serverless indicates whether the server is running on serverless compute. Serverless bool + // UsagePolicyID the job was submitted with. Persisted to metadata.json so reconnects + // can tell which usage policy the running server was started under. + UsagePolicyID string // The directory to store sshd configuration ConfigDir string // The name of the secrets scope to use for client and server keys @@ -70,8 +73,9 @@ func Run(ctx context.Context, client *databricks.WorkspaceClient, opts ServerOpt // Save metadata including ClusterID (required for Driver Proxy connections in serverless mode) metadata := &workspace.WorkspaceMetadata{ - Port: port, - ClusterID: opts.ClusterID, + Port: port, + ClusterID: opts.ClusterID, + UsagePolicyID: opts.UsagePolicyID, } err = workspace.SaveWorkspaceMetadata(ctx, client, opts.Version, opts.SessionID, metadata) if err != nil { diff --git a/experimental/ssh/internal/workspace/workspace.go b/experimental/ssh/internal/workspace/workspace.go index 0a28b684ebc..576e8a6df9f 100644 --- a/experimental/ssh/internal/workspace/workspace.go +++ b/experimental/ssh/internal/workspace/workspace.go @@ -19,6 +19,9 @@ type WorkspaceMetadata struct { Port int `json:"port"` // ClusterID is required for Driver Proxy websocket connections (for any compute type, including serverless) ClusterID string `json:"cluster_id,omitempty"` + // UsagePolicyID records the usage policy the server's job was submitted with, so a + // reconnect can tell whether a running server matches the requested usage policy. + UsagePolicyID string `json:"usage_policy_id,omitempty"` } func getWorkspaceRootDir(ctx context.Context, client *databricks.WorkspaceClient) (string, error) { diff --git a/go.mod b/go.mod index a6e3179d04e..c58c3eaffb0 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,14 @@ require ( github.com/charmbracelet/huh v1.0.0 // MIT github.com/charmbracelet/lipgloss v1.1.0 // MIT github.com/charmbracelet/x/ansi v0.11.7 // MIT - github.com/databricks/databricks-sdk-go v0.154.0 // Apache-2.0 + github.com/databricks/databricks-sdk-go v0.165.0 // Apache-2.0 github.com/google/jsonschema-go v0.4.3 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/gorilla/websocket v1.5.3 // BSD-2-Clause github.com/hashicorp/go-version v1.9.0 // MPL-2.0 github.com/hashicorp/hc-install v0.9.5 // MPL-2.0 github.com/hashicorp/terraform-exec v0.25.2 // MPL-2.0 - github.com/hashicorp/terraform-json v0.27.2 // MPL-2.0 + github.com/hashicorp/terraform-json v0.28.0 // MPL-2.0 github.com/hexops/gotextdiff v1.0.3 // BSD-3-Clause github.com/jackc/pgx/v5 v5.10.0 // MIT github.com/mattn/go-isatty v0.0.22 // MIT @@ -35,16 +35,24 @@ require ( github.com/tailscale/hujson v0.0.0-20250605163823-992244df8c5a // BSD-3-Clause github.com/zalando/go-keyring v0.2.8 // MIT go.yaml.in/yaml/v3 v3.0.4 // MIT AND Apache-2.0 - golang.org/x/crypto v0.53.0 // BSD-3-Clause - golang.org/x/mod v0.37.0 // BSD-3-Clause - golang.org/x/net v0.56.0 // BSD-3-Clause + golang.org/x/crypto v0.54.0 // BSD-3-Clause + golang.org/x/mod v0.38.0 // BSD-3-Clause + golang.org/x/net v0.57.0 // BSD-3-Clause golang.org/x/oauth2 v0.36.0 // BSD-3-Clause - golang.org/x/sync v0.21.0 // BSD-3-Clause - golang.org/x/sys v0.46.0 // BSD-3-Clause - golang.org/x/text v0.38.0 // BSD-3-Clause + golang.org/x/sync v0.22.0 // BSD-3-Clause + golang.org/x/sys v0.47.0 // BSD-3-Clause + golang.org/x/text v0.40.0 // BSD-3-Clause gopkg.in/ini.v1 v1.67.3 // Apache-2.0 ) +require github.com/databricks/sdk-go/core v0.0.1-dev // Apache-2.0 + +require ( + github.com/databricks/sdk-go/auth v0.0.0-dev // Apache-2.0 + github.com/databricks/sdk-go/files v0.0.0-dev.1 // Apache-2.0 + github.com/databricks/sdk-go/options v0.0.0-dev // Apache-2.0 +) + require ( cloud.google.com/go/auth v0.18.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -102,8 +110,8 @@ require ( golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/api v0.265.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect - google.golang.org/grpc v1.79.3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 222ac1324a5..411ab2158c7 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,16 @@ github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22r github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= -github.com/databricks/databricks-sdk-go v0.154.0 h1:Vmif0i0rbu7kgphoEBPRroZNd5uLBOITvjU4dr2lwXY= -github.com/databricks/databricks-sdk-go v0.154.0/go.mod h1:C5LNgGe6hGuRrTwoxFmuup3XtQQEaqtq0e+K8IFDIS4= +github.com/databricks/databricks-sdk-go v0.165.0 h1:b9tFvqDQTegM0vuf6BBp0g8xI9Xn/8mBsBl8kqAE2KE= +github.com/databricks/databricks-sdk-go v0.165.0/go.mod h1:C5LNgGe6hGuRrTwoxFmuup3XtQQEaqtq0e+K8IFDIS4= +github.com/databricks/sdk-go/auth v0.0.0-dev h1:SyRGZvExH5TG9ynZEqNn+m+xTY42VOXr2GOxgNOQexk= +github.com/databricks/sdk-go/auth v0.0.0-dev/go.mod h1:Tj09W13MScUaix94cR0He9msAwe40JAQKuS2jP/ofQA= +github.com/databricks/sdk-go/core v0.0.1-dev h1:sIA1hCEJyl8/012vOBUIkDjsycNCD8RwfP76SCM0QbQ= +github.com/databricks/sdk-go/core v0.0.1-dev/go.mod h1:7Ckau34bOsaZhHJE6r7ORNa+/kO33jIs6DfrJL6UbsM= +github.com/databricks/sdk-go/files v0.0.0-dev.1 h1:MAcqxXpTFuM8dV+Yeb+X2+2iQlXWV2MVYEsYKle09+0= +github.com/databricks/sdk-go/files v0.0.0-dev.1/go.mod h1:4xc94QVa2BnkqR3eVC4o73phlwPgB/N2EAPKlK/2/zQ= +github.com/databricks/sdk-go/options v0.0.0-dev h1:+3bgKy5OH6G8VqzAox0eh0Ca28eQnoi9qDi5j/deiRY= +github.com/databricks/sdk-go/options v0.0.0-dev/go.mod h1:+lMasXZ/AfRAUYFNC8KFuT2vCJoF+TYNZ7XE0t26zfk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -130,8 +138,8 @@ github.com/hashicorp/hc-install v0.9.5 h1:XHCjcMn2563ysuaQ9v9ec2FNc7c2PJOIEEGobA github.com/hashicorp/hc-install v0.9.5/go.mod h1:ihEW4LshrNkxq2bU/MpVbKyn+yt1is2hYqUTHDGhG84= github.com/hashicorp/terraform-exec v0.25.2 h1:fFLAVEtAjKdGfawGUXDnKooCnqJi+TuohT3W99AGbhk= github.com/hashicorp/terraform-exec v0.25.2/go.mod h1:uaQV2oqVLqM4cixJryk6qIWS1qji3GtuwPG5pjGXYfc= -github.com/hashicorp/terraform-json v0.27.2 h1:BwGuzM6iUPqf9JYM/Z4AF1OJ5VVJEEzoKST/tRDBJKU= -github.com/hashicorp/terraform-json v0.27.2/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE= +github.com/hashicorp/terraform-json v0.28.0 h1:dOkJT55rWfU6T1/VklHde51ym4LfNP+9xYR3ZizAJe4= +github.com/hashicorp/terraform-json v0.28.0/go.mod h1:PJIRf+Yzu5iLb52c/xYp1tUOL4jzMzfIAB5gvWWKIWE= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -239,36 +247,36 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/ go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration/cmd/fs/helpers_test.go b/integration/cmd/fs/helpers_test.go index e1bebb28f42..0f16ac5fdb8 100644 --- a/integration/cmd/fs/helpers_test.go +++ b/integration/cmd/fs/helpers_test.go @@ -30,14 +30,14 @@ func setupDbfsFiler(t testutil.TestingT) (filer.Filer, string) { } func setupUcVolumesFiler(t testutil.TestingT) (filer.Filer, string) { - _, wt := acc.WorkspaceTest(t) + ctx, wt := acc.WorkspaceTest(t) if os.Getenv("TEST_METASTORE_ID") == "" { t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") } tmpdir := acc.TemporaryVolume(wt) - f, err := filer.NewFilesClient(wt.W, tmpdir) + f, err := filer.NewFilesClient(ctx, wt.W, tmpdir) require.NoError(t, err) return f, path.Join("dbfs:/", tmpdir) diff --git a/integration/libs/filer/helpers_test.go b/integration/libs/filer/helpers_test.go index ead83d66b08..baca230b791 100644 --- a/integration/libs/filer/helpers_test.go +++ b/integration/libs/filer/helpers_test.go @@ -58,14 +58,14 @@ func setupDbfsFiler(t testutil.TestingT) (filer.Filer, string) { } func setupUcVolumesFiler(t testutil.TestingT) (filer.Filer, string) { - _, wt := acc.WorkspaceTest(t) + ctx, wt := acc.WorkspaceTest(t) if os.Getenv("TEST_METASTORE_ID") == "" { t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") } tmpdir := acc.TemporaryVolume(wt) - f, err := filer.NewFilesClient(wt.W, tmpdir) + f, err := filer.NewFilesClient(ctx, wt.W, tmpdir) require.NoError(t, err) return f, path.Join("dbfs:/", tmpdir) diff --git a/integration/libs/git/git_fetch_test.go b/integration/libs/git/git_fetch_test.go index 95ca99eec30..094389284cb 100644 --- a/integration/libs/git/git_fetch_test.go +++ b/integration/libs/git/git_fetch_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/databricks/cli/integration/internal/acc" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/git" "github.com/stretchr/testify/assert" @@ -47,6 +48,13 @@ func ensureWorkspacePrefix(root string) string { } func TestFetchRepositoryInfoAPI_FromRepo(t *testing.T) { + // On GCP the Repos API returns no branch, commit, or origin URL for a + // freshly-cloned repo, so FetchRepositoryInfo comes back empty and the + // assertions in assertFullGitInfo fail. + if testutil.GetCloud(t) == testutil.GCP { + t.Skip("Skipping on GCP: Repos API does not return git metadata") + } + ctx, wt := acc.WorkspaceTest(t) targetPath := ensureWorkspacePrefix(acc.TemporaryRepo(wt, examplesRepoUrl)) diff --git a/internal/build/notice_test.go b/internal/build/notice_test.go index 972c7aa9c0c..ea71f1f77a8 100644 --- a/internal/build/notice_test.go +++ b/internal/build/notice_test.go @@ -22,6 +22,10 @@ var moduleToGitHub = map[string]string{ // Modules excluded from NOTICE requirements (Databricks-owned). var noticeExclude = map[string]bool{ "github.com/databricks/databricks-sdk-go": true, + "github.com/databricks/sdk-go/core": true, + "github.com/databricks/sdk-go/auth": true, + "github.com/databricks/sdk-go/files": true, + "github.com/databricks/sdk-go/options": true, } // Additional entries required in the NOTICE file that are not direct go.mod diff --git a/internal/genkit/tagging.py b/internal/genkit/tagging.py index 0c8cd582296..3a75f372835 100644 --- a/internal/genkit/tagging.py +++ b/internal/genkit/tagging.py @@ -10,7 +10,7 @@ import subprocess import time import json -from github import Github, Repository, InputGitTreeElement, InputGitAuthor +from github import Auth, Github, Repository, InputGitTreeElement, InputGitAuthor from datetime import datetime, timezone NEXT_CHANGELOG_FILE_NAME = "NEXT_CHANGELOG.md" @@ -135,12 +135,26 @@ def _read_local_head_sha() -> str: return subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() +def _release_branch() -> str: + """ + Returns the branch this release is being cut from. + + The tagging workflow sets ``DECO_TAGGING_REF`` to the branch it was + dispatched on (``github.ref_name``) so a release can be cut from a + branch other than main. It is unset for local runs and the historical + main-only release path, so we default to ``main`` and every existing + caller is unaffected. + """ + return os.environ.get("DECO_TAGGING_REF", "").strip() or "main" + + class MainAdvancedError(Exception): """ - Raised when ``origin/main`` has advanced since the workflow's - checkout — i.e., another commit landed during this run. The local - working tree is now stale, so any commit produced from it would - silently revert whatever the concurrent push added. + Raised when the release branch (``origin/main`` by default; see + ``_release_branch``) has advanced since the workflow's checkout — + i.e., another commit landed during this run. The local working tree + is now stale, so any commit produced from it would silently revert + whatever the concurrent push added. """ @@ -151,7 +165,10 @@ class GitHubRepo: def __init__(self, repo: Repository): self.repo = repo self.changed_files: list[InputGitTreeElement] = [] - self.ref = "heads/main" + # Branch the changelog-bump commit + tag land on. Defaults to + # ``heads/main``; ``DECO_TAGGING_REF`` overrides it for a branch + # release. See ``_release_branch``. + self.ref = f"heads/{_release_branch()}" # Anchor ``self.sha`` to the **local checkout** rather than a # live API call. ``actions/checkout`` populates the working tree # at this SHA, and every subsequent file read in this run is @@ -167,12 +184,19 @@ def add_file(self, loc: str, content: str): element = InputGitTreeElement(path=local_path, mode="100644", type="blob", sha=blob.sha) self.changed_files.append(element) + # Replaces "git rm file" + def delete_file(self, loc: str): + """``git rm`` equivalent for GitHubRepo: stage a tree deletion (sha=None).""" + local_path = os.path.relpath(loc, os.getcwd()) + print(f"Deleting file {local_path}") + self.changed_files.append(InputGitTreeElement(path=local_path, mode="100644", type="blob", sha=None)) + # Replaces "git commit && git push" def commit_and_push(self, message: str): head_ref = self.repo.get_git_ref(self.ref) if head_ref.object.sha != self.sha: raise MainAdvancedError( - f"origin/main advanced from {self.sha} to {head_ref.object.sha} " + f"{self.ref} advanced from {self.sha} to {head_ref.object.sha} " f"during this run. Local working tree is stale; aborting before " f"the commit would silently revert the new content. Re-run the " f"workflow." @@ -754,8 +778,10 @@ def reset_repository(hash: Optional[str] = None) -> None: # Fetch the latest changes from the remote repository. subprocess.run(["git", "fetch"]) - # Determine the commit hash (default to origin/main if none is provided). - commit_hash = hash or "origin/main" + # Determine the commit hash (default to the release branch's remote + # head if none is provided). ``_release_branch`` is ``main`` unless + # ``DECO_TAGGING_REF`` selects a branch release. + commit_hash = hash or f"origin/{_release_branch()}" # ``git reset --hard`` must land before ``gh.reset(None)``, since # ``gh.reset(None)`` reads ``git rev-parse HEAD`` to anchor @@ -983,7 +1009,7 @@ def get_packages_from_args() -> List[str]: def init_github(): token = os.environ["GITHUB_TOKEN"] repo_name = os.environ["GITHUB_REPOSITORY"] - g = Github(token) + g = Github(auth=Auth.Token(token)) repo = g.get_repo(repo_name) global gh gh = GitHubRepo(repo) @@ -1050,7 +1076,11 @@ def validate_git_root(): raise Exception("Please run this script from the root of the repository.") -if __name__ == "__main__": +def main(): validate_git_root() init_github() process() + + +if __name__ == "__main__": + main() diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go index deb7e59a2f5..5f2d1a23b75 100644 --- a/internal/testcli/runner.go +++ b/internal/testcli/runner.go @@ -236,9 +236,19 @@ func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, var wg sync.WaitGroup defer wg.Wait() + // Closed when this function returns to release any condition goroutine still + // blocked sending into the size-1 [ch]. Without it, a timeout leaves in-flight + // senders stuck forever and the deferred [wg.Wait] deadlocks. The defer order + // matters: [close] runs before [wg.Wait] (defers run last-in-first-out). + done := make(chan struct{}) + defer close(done) + // Kick off condition check immediately. wg.Go(func() { - ch <- condition() + select { + case ch <- condition(): + case <-done: + } }) for tick := ticker.C; ; { @@ -252,7 +262,10 @@ func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, case <-tick: tick = nil wg.Go(func() { - ch <- condition() + select { + case ch <- condition(): + case <-done: + } }) case v := <-ch: if v { diff --git a/libs/aitools/installer/aidevkit.go b/libs/aitools/installer/aidevkit.go new file mode 100644 index 00000000000..857aa04d427 --- /dev/null +++ b/libs/aitools/installer/aidevkit.go @@ -0,0 +1,73 @@ +package installer + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/useragent" +) + +// aiDevKitStateDir is the marker directory the ai-dev-kit installer +// (github.com/databricks-solutions/ai-dev-kit) writes into both the project root +// and the global home (or $AIDEVKIT_HOME). +const aiDevKitStateDir = ".ai-dev-kit" + +// aiDevKitVersionFile is the version file the ai-dev-kit installer writes under aiDevKitStateDir. +// See https://github.com/databricks-solutions/ai-dev-kit install.sh. +const aiDevKitVersionFile = "version" + +// aiDevKitHomeEnv overrides the global ai-dev-kit install location. +const aiDevKitHomeEnv = "AIDEVKIT_HOME" + +// AiDevKitVersion returns the installed ai-dev-kit version and whether it is +// installed, sanitized for the user agent. +func AiDevKitVersion(ctx context.Context) (string, bool) { + installed := false + for _, dir := range aiDevKitStateDirs(ctx) { + version, ok := readAiDevKitVersion(dir) + if !ok { + continue + } + installed = true + // A blank marker still means installed; keep scanning so a real version + // in a lower-precedence scope isn't masked by an empty higher one. + if version != "" { + return version, true + } + } + return "", installed +} + +// aiDevKitStateDirs returns the candidate ai-dev-kit state directories to probe, +// project scope first. A scope whose base path can't be resolved is skipped +// rather than guessed. +func aiDevKitStateDirs(ctx context.Context) []string { + var dirs []string + if cwd, err := os.Getwd(); err == nil { + dirs = append(dirs, filepath.Join(cwd, aiDevKitStateDir)) + } + // AIDEVKIT_HOME already points at the install root, so the marker lives + // directly under it; otherwise it defaults to ~/.ai-dev-kit. + if home := env.Get(ctx, aiDevKitHomeEnv); home != "" { + dirs = append(dirs, home) + } else if home, err := env.UserHomeDir(ctx); err == nil { + dirs = append(dirs, filepath.Join(home, aiDevKitStateDir)) + } + return dirs +} + +// readAiDevKitVersion reads the version marker under dir, reporting whether it +// exists. A present-but-blank marker reports ("", true). +func readAiDevKitVersion(dir string) (string, bool) { + data, err := os.ReadFile(filepath.Join(dir, aiDevKitVersionFile)) + if err != nil { + return "", false + } + // Sanitize the first line: the value is set outside our control and + // useragent panics on anything but alphanumerics/semver. + firstLine, _, _ := strings.Cut(string(data), "\n") + return useragent.Sanitize(strings.TrimSpace(firstLine)), true +} diff --git a/libs/aitools/installer/aidevkit_test.go b/libs/aitools/installer/aidevkit_test.go new file mode 100644 index 00000000000..1f04c4f62aa --- /dev/null +++ b/libs/aitools/installer/aidevkit_test.go @@ -0,0 +1,86 @@ +package installer + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeAiDevKitMarker writes a .ai-dev-kit/version marker under baseDir (a temp +// dir standing in for a project root or the global home). +func writeAiDevKitMarker(t *testing.T, baseDir, contents string) { + t.Helper() + dir := filepath.Join(baseDir, aiDevKitStateDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, aiDevKitVersionFile), []byte(contents), 0o644)) +} + +func TestAiDevKitVersion(t *testing.T) { + tests := []struct { + name string + // global/project contents; "" means no marker written for that scope. + global string + project string + wantVersion string + wantOK bool + }{ + {"not installed", "", "", "", false}, + {"global marker", "1.2.3\n", "", "1.2.3", true}, + {"project marker", "", "2.0.0\n", "2.0.0", true}, + {"project overrides global", "1.2.3\n", "2.0.0\n", "2.0.0", true}, + {"dev version", "dev\n", "", "dev", true}, + {"empty marker still installed", "\n", "", "", true}, + {"whitespace-only marker", " \n", "", "", true}, + // A blank project marker must not mask a real global version: still + // installed, but the global version wins over project's "unknown". + {"empty project does not mask global", "1.2.3\n", "\n", "1.2.3", true}, + {"trailing newline trimmed", "1.2.3\n", "", "1.2.3", true}, + {"multi-line keeps first line", "1.2.3\nextra\n", "", "1.2.3", true}, + {"multi-line CRLF has no trailing hyphen", "1.2.3\r\nextra\r\n", "", "1.2.3", true}, + {"sanitizes illegal chars", "1.2.3 (beta/rc)\n", "", "1.2.3--beta-rc-", true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + project := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Chdir(project) + + if tc.global != "" { + writeAiDevKitMarker(t, home, tc.global) + } + if tc.project != "" { + writeAiDevKitMarker(t, project, tc.project) + } + + version, ok := AiDevKitVersion(t.Context()) + assert.Equal(t, tc.wantOK, ok) + assert.Equal(t, tc.wantVersion, version) + }) + } +} + +// TestAiDevKitVersionHomeOverride verifies AIDEVKIT_HOME points directly at the +// install root, so the marker is read from $AIDEVKIT_HOME/version, not +// $AIDEVKIT_HOME/.ai-dev-kit/version. +func TestAiDevKitVersionHomeOverride(t *testing.T) { + home := t.TempDir() + installRoot := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv(aiDevKitHomeEnv, installRoot) + t.Chdir(t.TempDir()) + + require.NoError(t, os.WriteFile(filepath.Join(installRoot, aiDevKitVersionFile), []byte("3.1.4\n"), 0o644)) + // A marker under ~/.ai-dev-kit must be ignored once AIDEVKIT_HOME is set. + writeAiDevKitMarker(t, home, "9.9.9\n") + + version, ok := AiDevKitVersion(t.Context()) + assert.True(t, ok) + assert.Equal(t, "3.1.4", version) +} diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index eb627ad3ed0..93521292735 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -316,14 +316,15 @@ func InstallSkillsForAgents(ctx context.Context, src ManifestSource, targetAgent return fmt.Errorf("failed to load install state: %w", err) } - // Detect legacy installs (skills on disk but no state file). Global only. - // Block targeted installs on legacy setups to avoid writing incomplete state - // that would hide the legacy warning on future runs. - if state == nil && scope == ScopeGlobal { - isLegacy := checkLegacyInstall(ctx, baseDir) - if isLegacy && len(opts.SpecificSkills) > 0 { - return errors.New("legacy install detected without state tracking; run 'databricks aitools install' (without a skill name) first to rebuild state") - } + // A "legacy install" is one from before the CLI tracked install state in a + // JSON file: skills exist on disk but there's no state file (state == nil). + // A targeted install (--skills foo) here would write a state file listing only + // foo, leaving the other on-disk skills untracked. Block it so the user runs a + // full install first, which rebuilds complete state. Global scope only, since + // legacy installs only ever wrote to the global dir. + // hasLegacyInstall lives in update.go (shared with the update/uninstall paths). + if state == nil && scope == ScopeGlobal && len(opts.SpecificSkills) > 0 && hasLegacyInstall(ctx, baseDir) { + return errors.New("legacy install detected without state tracking; run 'databricks aitools install' (without a skill name) first to rebuild state") } // Filter skills based on options, experimental flag, and CLI version. @@ -342,6 +343,11 @@ func InstallSkillsForAgents(ctx context.Context, src ManifestSource, targetAgent // Install each skill in sorted order for determinism. skillNames := slices.Sorted(maps.Keys(targetSkills)) + // Scoped to the loop, not the manifest fetch above, so earlier warnings print + // as plain lines instead of racing the live spinner's redraw. + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + // Accumulate file provenance for skills we (re)fetch this run. Skipped // (already-installed) skills keep their existing records via the merge below. fileRecords := map[string]FileRecord{} @@ -360,6 +366,7 @@ func InstallSkillsForAgents(ctx context.Context, src ManifestSource, targetAgent } } + sp.Update("Installing " + name + "...") records, err := installSkillForAgents(ctx, name, meta, targetAgents, params) if err != nil { return err @@ -407,6 +414,10 @@ func InstallSkillsForAgents(ctx context.Context, src ManifestSource, targetAgent return err } + // Stop the spinner before the summary so its transient line doesn't linger + // above the persistent "Installed N skills." output. + sp.Close() + noun := "skills" if len(targetSkills) == 1 { noun = "skill" @@ -522,25 +533,6 @@ func printNoAgentsDetected(ctx context.Context) { cmdio.LogString(ctx, "Please install at least one coding agent first.") } -// checkLegacyInstall prints a message if skills exist on disk but no state file was found. -// Returns true if a legacy install was detected. -func checkLegacyInstall(ctx context.Context, globalDir string) bool { - if hasSkillsOnDisk(globalDir) { - cmdio.LogString(ctx, "Found skills installed before state tracking was added. Run 'databricks aitools install' to refresh.") - return true - } - homeDir, err := env.UserHomeDir(ctx) - if err != nil { - return false - } - legacyDir := filepath.Join(homeDir, ".databricks", "agent-skills") - if hasSkillsOnDisk(legacyDir) { - cmdio.LogString(ctx, "Found skills installed before state tracking was added. Run 'databricks aitools install' to refresh.") - return true - } - return false -} - // hasSkillsOnDisk checks if a directory contains subdirectories starting with "databricks". func hasSkillsOnDisk(dir string) bool { entries, err := os.ReadDir(dir) @@ -580,7 +572,6 @@ type installParams struct { func installSkillForAgents(ctx context.Context, skillName string, meta SkillMeta, detectedAgents []*agents.Agent, params installParams) (map[string]FileRecord, error) { canonicalDir := filepath.Join(params.baseDir, skillName) - cmdio.LogString(ctx, fmt.Sprintf("Downloading %s...", skillName)) records, err := installSkillToDir(ctx, params.ref, meta.RepoDir, meta.SourceName, canonicalDir, meta.Files) if err != nil { return nil, err @@ -588,7 +579,6 @@ func installSkillForAgents(ctx context.Context, skillName string, meta SkillMeta // For project scope, always symlink. For global, symlink when multiple agents. useSymlinks := params.scope == ScopeProject || len(detectedAgents) > 1 - cmdio.LogString(ctx, fmt.Sprintf("Exposing %s to %d %s...", skillName, len(detectedAgents), agentNoun(len(detectedAgents)))) for _, agent := range detectedAgents { agentSkillDir, err := agentSkillsDirForScope(ctx, agent, params.scope, params.cwd) @@ -634,13 +624,6 @@ func installSkillForAgents(ctx context.Context, skillName string, meta SkillMeta return records, nil } -func agentNoun(n int) string { - if n == 1 { - return "agent" - } - return "agents" -} - // agentSkillsDirForScope returns the agent's skills directory for the given scope. func agentSkillsDirForScope(ctx context.Context, agent *agents.Agent, scope, cwd string) (string, error) { if scope == ScopeProject { diff --git a/libs/aitools/installer/installer_test.go b/libs/aitools/installer/installer_test.go index 6e12ef3d3aa..f9256d2d809 100644 --- a/libs/aitools/installer/installer_test.go +++ b/libs/aitools/installer/installer_test.go @@ -439,9 +439,10 @@ func TestInstallSkillsForAgentsWritesState(t *testing.T) { assert.NotEmpty(t, state.Files["databricks-sql/SKILL.md"].SHA256) assert.Equal(t, testSkillsRef, state.Files["databricks-sql/SKILL.md"].Origin) + // Per-skill progress now goes through a spinner, which is silent in this + // non-interactive test, so only these plain lines remain. + assert.Contains(t, stderr.String(), "Using skills version") assert.Contains(t, stderr.String(), "Fetching skills manifest...") - assert.Contains(t, stderr.String(), "Downloading databricks-sql...") - assert.Contains(t, stderr.String(), "Exposing databricks-sql to 1 agent...") assert.Contains(t, stderr.String(), "Installed 2 skills.") } @@ -713,42 +714,26 @@ func TestIdempotentInstallUpdatesNewVersions(t *testing.T) { assert.Equal(t, "0.2.0", state.Skills["databricks-sql"]) } -func TestLegacyDetectMessagePrinted(t *testing.T) { +func TestLegacyTargetedInstallBlockedFromLegacyDir(t *testing.T) { tmp := setupTestHome(t) - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - setupFetchMock(t) - t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) - - // Create skills on disk at canonical location but no state file. - globalDir := filepath.Join(tmp, ".databricks", "aitools", "skills") - require.NoError(t, os.MkdirAll(filepath.Join(globalDir, "databricks-sql"), 0o755)) - - src := &mockManifestSource{manifest: testManifest()} - agent := testAgent(tmp) - - err := InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{}) - require.NoError(t, err) - - assert.Contains(t, stderr.String(), "Found skills installed before state tracking was added.") -} - -func TestLegacyDetectLegacyDir(t *testing.T) { - tmp := setupTestHome(t) - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx := cmdio.MockDiscard(t.Context()) setupFetchMock(t) t.Setenv("DATABRICKS_SKILLS_REF", testSkillsRef) - // Create skills in the legacy location. + // Skills in the legacy location (~/.databricks/agent-skills), no state file. legacyDir := filepath.Join(tmp, ".databricks", "agent-skills") require.NoError(t, os.MkdirAll(filepath.Join(legacyDir, "databricks-sql"), 0o755)) src := &mockManifestSource{manifest: testManifest()} agent := testAgent(tmp) - err := InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{}) - require.NoError(t, err) - - assert.Contains(t, stderr.String(), "Found skills installed before state tracking was added.") + // A targeted install must be blocked when a legacy install is detected via + // the legacy dir, mirroring the canonical-dir case. + err := InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{ + SpecificSkills: []string{"databricks-sql"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "legacy install detected") } func TestIdempotentInstallReinstallsForNewAgent(t *testing.T) { @@ -831,11 +816,11 @@ func TestLegacyFullInstallAllowed(t *testing.T) { src := &mockManifestSource{manifest: testManifest()} agent := testAgent(tmp) - // Full install (no SpecificSkills) should succeed and rebuild state. + // Full install (no SpecificSkills) should succeed and rebuild state silently. err := InstallSkillsForAgents(ctx, src, []*agents.Agent{agent}, InstallOptions{}) require.NoError(t, err) - assert.Contains(t, stderr.String(), "Found skills installed before state tracking was added.") + assert.Contains(t, stderr.String(), "Installed 2 skills.") state, err := LoadState(globalDir) require.NoError(t, err) diff --git a/libs/aitools/installer/update.go b/libs/aitools/installer/update.go index 0b1e30efcf6..36a7ef9c279 100644 --- a/libs/aitools/installer/update.go +++ b/libs/aitools/installer/update.go @@ -187,15 +187,20 @@ func UpdateSkills(ctx context.Context, src ManifestSource, targetAgents []*agent ref: latestTag, } + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + fileRecords := map[string]FileRecord{} for _, change := range allChanges { meta := manifest.Skills[change.Name] + sp.Update("Installing " + change.Name + "...") records, err := installSkillForAgents(ctx, change.Name, meta, targetAgents, params) if err != nil { return nil, err } maps.Copy(fileRecords, records) } + sp.Close() // Update state. state.Release = latestTag diff --git a/libs/aitools/installer/useragent.go b/libs/aitools/installer/useragent.go index 02d61095ae8..d0c9894582a 100644 --- a/libs/aitools/installer/useragent.go +++ b/libs/aitools/installer/useragent.go @@ -10,6 +10,11 @@ import ( // dimension. Each installed tool contributes an "/_" pair. const aiToolsUserAgentKey = "aitools" +// aiDevKitUserAgentKey is the user-agent key for the ai-dev-kit toolkit +// (github.com/databricks-solutions/ai-dev-kit). Its skills have been subsumed +// into databricks-agent-skills, but we track ai-dev-kit installs separately. +const aiDevKitUserAgentKey = "aidevkit" + // WithAiToolsInUserAgent adds one aitools/_ pair to the user // agent for each coding tool that has the Databricks plugin installed, e.g. // "aitools/claude-code_0.2.9 aitools/codex_0.2.9". Nothing is added when no tool @@ -24,3 +29,19 @@ func WithAiToolsInUserAgent(ctx context.Context) context.Context { } return ctx } + +// WithAiDevKitInUserAgent adds an "aidevkit/" pair to the user agent +// when the ai-dev-kit toolkit is installed (project scope preferred, else the +// global $AIDEVKIT_HOME/~/.ai-dev-kit marker), e.g. "aidevkit/1.2.3". Nothing is +// added when it is not installed. An installed marker with an unreadable version +// emits "aidevkit/unknown" so adoption is still counted. +func WithAiDevKitInUserAgent(ctx context.Context) context.Context { + version, ok := AiDevKitVersion(ctx) + if !ok { + return ctx + } + if version == "" { + version = "unknown" + } + return useragent.InContext(ctx, aiDevKitUserAgentKey, version) +} diff --git a/libs/aitools/installer/useragent_test.go b/libs/aitools/installer/useragent_test.go index c08e5eab74d..9c8aeb9c575 100644 --- a/libs/aitools/installer/useragent_test.go +++ b/libs/aitools/installer/useragent_test.go @@ -67,3 +67,53 @@ func TestWithAiToolsInUserAgent(t *testing.T) { }) } } + +// TestWithAiDevKitInUserAgent drives WithAiDevKitInUserAgent against a prepared +// HOME containing (or lacking) an ai-dev-kit version marker and asserts the +// resulting user-agent string. +func TestWithAiDevKitInUserAgent(t *testing.T) { + tests := []struct { + name string + // marker, when set, is written to ~/.ai-dev-kit/version. + marker string + wantContains []string + wantNotContain []string + }{ + { + name: "not installed adds no pair", + wantNotContain: []string{"aidevkit/"}, + }, + { + name: "version from the marker", + marker: "1.2.3\n", + wantContains: []string{"aidevkit/1.2.3"}, + }, + { + name: "installed but unversioned reports unknown", + marker: "\n", + wantContains: []string{"aidevkit/unknown"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + // Fresh cwd so a project-scope marker never leaks from the package dir. + t.Chdir(t.TempDir()) + + if tc.marker != "" { + writeAiDevKitMarker(t, home, tc.marker) + } + + ua := useragent.FromContext(WithAiDevKitInUserAgent(t.Context())) + for _, want := range tc.wantContains { + assert.Contains(t, ua, want) + } + for _, notWant := range tc.wantNotContain { + assert.NotContains(t, ua, notWant) + } + }) + } +} diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index 66451124293..1a47b64fef8 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -133,6 +133,13 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptio return dyn.InvalidValue, err } + // If the field carries the bundle:"sensitive" tag and resolved to a string + // (including empty), wrap it as a sensitive value so that all downstream + // serializers (JSON, YAML) redact it automatically. + if info.Sensitive[k] && nv.Kind() == dyn.KindString { + nv = dyn.NewSensitiveValue(nv.MustString(), nv.Locations()) + } + // Either if the key was set in the reference, the field is not zero-valued, OR it's forced if ok || nv.Kind() != dyn.KindNil || isForced { // If v isZero, it could be because it's a variable reference; so we check that nv is zero as well diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go index 0c8cf902bb4..38daab1aa30 100644 --- a/libs/dyn/convert/from_typed_test.go +++ b/libs/dyn/convert/from_typed_test.go @@ -926,3 +926,55 @@ func TestFromTypedForceSendFieldsEmbedded(t *testing.T) { assert.Equal(t, dyn.KindNil, field.Kind(), "embedded field should be present due to ForceSendFields") assert.Equal(t, dyn.V("value"), other) } + +func TestFromTypedSensitiveField(t *testing.T) { + type Tmp struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + src := Tmp{ + Name: "my-resource", + Token: "super-secret", + } + + nv, err := FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + name := nv.Get("name") + token := nv.Get("token") + + assert.False(t, name.IsSensitive(), "name should not be sensitive") + assert.True(t, token.IsSensitive(), "token field should be sensitive") + + // MustString returns the real value. + assert.Equal(t, "super-secret", token.MustString()) + // AsAny returns the redaction placeholder. + assert.Equal(t, dyn.SensitiveValueRedacted, token.AsAny()) +} + +func TestFromTypedSensitiveFieldEmpty(t *testing.T) { + // An empty sensitive string must still be wrapped as sensitive. + type Tmp struct { + Token string `json:"token" bundle:"sensitive"` + } + + nv, err := FromTyped(Tmp{Token: ""}, dyn.NilValue) + require.NoError(t, err) + + token := nv.Get("token") + // Empty sensitive fields resolve to KindNil when the struct omits them + // (omitempty is the default). When the field IS present (e.g. forced via + // a non-nil pointer or ForceSendFields), it should be sensitive. + // Use a pointer-to-struct to force the zero value through. + src := &Tmp{Token: ""} + nv2, err := FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + token2 := nv2.Get("token") + if token2.Kind() == dyn.KindString { + assert.True(t, token2.IsSensitive(), "empty sensitive string should still be sensitive") + assert.Empty(t, token2.MustString()) + } + _ = token // non-pointer form may be KindNil, which is fine +} diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 79cfee37441..2e13cfa3fa4 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -18,10 +18,19 @@ const ( // IncludeMissingFields causes the normalization to include fields that defined on the given // type but are missing in the source value. They are included with their zero values. IncludeMissingFields NormalizeOption = iota + + // DropEmptyStrings drops struct fields whose value is an empty string, unless + // the field is tagged without omitempty (i.e. its zero value must be sent). + // This mirrors JSON serialization, where an omitempty string field set to "" + // is omitted from the request. Without this, an explicitly-set "" reaches + // ToTyped, which force-sends it, and the backend rejects it (e.g. + // "'' is not a valid cluster policy ID"). + DropEmptyStrings ) type normalizeOptions struct { includeMissingFields bool + dropEmptyStrings bool } func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag.Diagnostics) { @@ -30,6 +39,8 @@ func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag switch opt { case IncludeMissingFields: n.includeMissingFields = true + case DropEmptyStrings: + n.dropEmptyStrings = true } } @@ -87,6 +98,36 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost } } +// isAnchorContainer reports whether v is a YAML anchor or a non-empty +// sequence/map composed entirely of anchor containers. Anchors define reusable +// blocks and must not trigger "unknown field" warnings, including when nested +// inside a container. +func isAnchorContainer(v dyn.Value) bool { + if v.IsAnchor() { + return true + } + + var elements []dyn.Value + switch v.Kind() { + case dyn.KindSequence: + elements = v.MustSequence() + case dyn.KindMap: + elements = v.MustMap().Values() + default: + return false + } + + if len(elements) == 0 { + return false + } + for _, e := range elements { + if !isAnchorContainer(e) { + return false + } + } + return true +} + func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics @@ -101,7 +142,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen fieldName := pk.MustString() index, ok := info.Fields[fieldName] if !ok { - if !pv.IsAnchor() { + if !isAnchorContainer(pv) { // Special case: provide a more helpful message for "valueFrom" vs "value_from" if fieldName == "valueFrom" { if _, hasValueFrom := info.Fields["value_from"]; hasValueFrom { @@ -136,6 +177,15 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen } } + // Drop an empty string on an omitempty field so it is not force-sent + // to the backend (see DropEmptyStrings). ForceEmpty marks fields whose + // zero value must be kept, so those are left in place. + if n.dropEmptyStrings && !info.ForceEmpty[fieldName] { + if s, ok := nv.AsString(); ok && s == "" { + continue + } + } + out.SetLoc(pk.MustString(), pk.Locations(), nv) } @@ -285,6 +335,15 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path switch src.Kind() { case dyn.KindString: + // A sensitive string must be returned as-is: dyn.NewValue(MustString(), ...) + // would construct a plain string and strip the secretString wrapper. Type + // normalization (e.g. coercing a bool "true" to string) cannot apply to a + // sensitive value because its Kind is already KindString, so returning early + // here is correct. Updating the secret value is done via FromTyped, not + // Normalize: FromTyped re-wraps the new value as NewSensitiveValue. + if src.IsSensitive() { + return src, nil + } return dyn.NewValue(src.MustString(), src.Locations()), nil case dyn.KindBool: return dyn.NewValue(strconv.FormatBool(src.MustBool()), src.Locations()), nil diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index cf8ca061733..69000200293 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -860,6 +860,79 @@ func TestNormalizeAnchors(t *testing.T) { }, vout.AsAny()) } +func TestNormalizeAnchorContainers(t *testing.T) { + type Tmp struct { + Foo string `json:"foo"` + } + + anchor := func() dyn.Value { + return dyn.V(map[string]dyn.Value{"name": dyn.V("x")}).MarkAnchor() + } + + tcases := []struct { + name string + value dyn.Value + wantWarn bool + }{ + { + name: "list of anchors", + value: dyn.V([]dyn.Value{anchor(), anchor()}), + wantWarn: false, + }, + { + name: "map of anchors", + value: dyn.V(map[string]dyn.Value{ + "a": anchor(), + "b": anchor(), + }), + wantWarn: false, + }, + { + name: "nested list of anchors", + value: dyn.V([]dyn.Value{dyn.V([]dyn.Value{anchor()})}), + wantWarn: false, + }, + { + name: "list with a non-anchor element", + value: dyn.V([]dyn.Value{anchor(), dyn.V(map[string]dyn.Value{"name": dyn.V("x")})}), + wantWarn: true, + }, + { + name: "empty list", + value: dyn.V([]dyn.Value{}), + wantWarn: true, + }, + { + name: "empty map", + value: dyn.V(map[string]dyn.Value{}), + wantWarn: true, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + var typ Tmp + vin := dyn.V(map[string]dyn.Value{ + "foo": dyn.V("bar"), + "thing": tc.value, + }) + + vout, diags := Normalize(typ, vin) + if tc.wantWarn { + assert.Len(t, diags, 1) + assert.Equal(t, "unknown field: thing", diags[0].Summary) + } else { + assert.Empty(t, diags) + } + + // The unknown field is never retained regardless of the warning. + assert.Equal(t, map[string]any{ + "foo": "bar", + }, vout.AsAny()) + }) + } +} + func TestNormalizeAnyFromSlice(t *testing.T) { var typ any v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) @@ -901,3 +974,40 @@ func TestNormalizeAnyFromTime(t *testing.T) { assert.Empty(t, err) assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) } + +func TestNormalizeStructDropEmptyStrings(t *testing.T) { + type Tmp struct { + // Optional: dropped when empty. + Opt string `json:"opt,omitempty"` + // Required (no omitempty): kept even when empty. + Req string `json:"req"` + } + + vin := dyn.V(map[string]dyn.Value{ + "opt": dyn.V(""), + "req": dyn.V(""), + }) + + vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) + assert.Empty(t, diags) + + _, hasOpt := vout.MustMap().Get(dyn.V("opt")) + assert.False(t, hasOpt, "empty omitempty string should be dropped") + req, hasReq := vout.MustMap().Get(dyn.V("req")) + assert.True(t, hasReq, "empty non-omitempty string should be kept") + assert.Empty(t, req.MustString()) +} + +func TestNormalizeStructDropEmptyStringsKeepsNonEmpty(t *testing.T) { + type Tmp struct { + Opt string `json:"opt,omitempty"` + } + + vin := dyn.V(map[string]dyn.Value{"opt": dyn.V("value")}) + + vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) + assert.Empty(t, diags) + opt, ok := vout.MustMap().Get(dyn.V("opt")) + assert.True(t, ok) + assert.Equal(t, "value", opt.MustString()) +} diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go index 69e8f868ed5..c78b49a0184 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/dyn/convert/struct_info.go @@ -28,6 +28,10 @@ type structInfo struct { // Maps JSON-name of the field to Golang struct name GolangNames map[string]string + // Sensitive tracks fields tagged `bundle:"sensitive"` by their JSON name. + // Values for these fields should be masked in display output. + Sensitive map[string]bool + // ForceSendFieldsIndex maps the JSON-name of the field to the index path (for // use with [reflect.Value.FieldByIndex]) of the ForceSendFields slice that // governs it: the one declared by the struct that also declares the field. @@ -68,6 +72,7 @@ func buildStructInfo(typ reflect.Type) structInfo { ForceEmpty: make(map[string]bool), GolangNames: make(map[string]string), ForceSendFieldsIndex: make(map[string][]int), + Sensitive: make(map[string]bool), } // Queue holds the indexes of the structs to visit. @@ -134,6 +139,11 @@ func buildStructInfo(typ reflect.Type) structInfo { } out.GolangNames[name] = sf.Name + btag := structtag.BundleTag(sf.Tag.Get("bundle")) + if btag.Sensitive() { + out.Sensitive[name] = true + } + // The field is declared directly in this struct, so it is governed by // this struct's ForceSendFields (if it has one). if forceSendFieldsIndex != nil { @@ -176,6 +186,25 @@ func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { return out } +// SensitiveFieldNames returns the JSON field names of typ that carry the +// `bundle:"sensitive"` tag. A pointer type is dereferenced before inspection. +// Returns nil for non-struct types. Callers use this to identify fields that +// must be masked in display output (validate -o json, plan -o json) without +// touching the typed values used by the actual deployment pipeline. +func SensitiveFieldNames(typ reflect.Type) map[string]bool { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if typ.Kind() != reflect.Struct { + return nil + } + si := getStructInfo(typ) + if len(si.Sensitive) == 0 { + return nil + } + return si.Sensitive +} + // isForceSend reports whether the field named k is listed in the ForceSendFields // that governs it (see structInfo.ForceSendFieldsIndex). func (s *structInfo) isForceSend(v reflect.Value, k string) bool { diff --git a/libs/dyn/convert/struct_info_test.go b/libs/dyn/convert/struct_info_test.go index f921523c391..9a0a68ec9d6 100644 --- a/libs/dyn/convert/struct_info_test.go +++ b/libs/dyn/convert/struct_info_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStructInfoPlain(t *testing.T) { @@ -226,3 +227,103 @@ func TestStructInfoValueFieldMultiple(t *testing.T) { getStructInfo(reflect.TypeFor[Tmp]()) }) } + +func TestSensitiveFieldNamesPlain(t *testing.T) { + type Tmp struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + fields := SensitiveFieldNames(reflect.TypeFor[Tmp]()) + assert.True(t, fields["token"]) + assert.False(t, fields["name"]) +} + +func TestSensitiveFieldNamesPointerDereference(t *testing.T) { + type Tmp struct { + Token string `json:"token" bundle:"sensitive"` + } + + fields := SensitiveFieldNames(reflect.TypeFor[*Tmp]()) + assert.True(t, fields["token"]) +} + +func TestSensitiveFieldNamesNilForNonStruct(t *testing.T) { + assert.Nil(t, SensitiveFieldNames(reflect.TypeFor[string]())) +} + +func TestSensitiveFieldNamesNilWhenNone(t *testing.T) { + type Tmp struct { + Name string `json:"name"` + } + + assert.Nil(t, SensitiveFieldNames(reflect.TypeFor[Tmp]())) +} + +func TestSensitiveFieldNamesEmbeddedByValue(t *testing.T) { + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + + type Outer struct { + Name string `json:"name"` + Inner + } + + fields := SensitiveFieldNames(reflect.TypeFor[Outer]()) + assert.True(t, fields["token"]) + assert.False(t, fields["name"]) +} + +func TestSensitiveFieldNamesEmbeddedByPointer(t *testing.T) { + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + + type Outer struct { + Name string `json:"name"` + *Inner + } + + fields := SensitiveFieldNames(reflect.TypeFor[Outer]()) + assert.True(t, fields["token"]) + assert.False(t, fields["name"]) +} + +func TestSensitiveFieldNamesTopLevelPrecedence(t *testing.T) { + // A sensitive field in an embedded struct is shadowed by a non-sensitive + // field of the same JSON name at the top level — top level wins. + type Inner struct { + Token string `json:"token" bundle:"sensitive"` + } + + type Outer struct { + Token string `json:"token"` // not sensitive; shadows Inner.Token + Inner + } + + fields := SensitiveFieldNames(reflect.TypeFor[Outer]()) + assert.False(t, fields["token"]) +} + +// TestSensitiveFieldNamesUsage demonstrates using SensitiveFieldNames with +// FromTyped: a sensitive field's value round-trips through dyn.Value and the +// caller can check which fields to mask before marshaling. +func TestSensitiveFieldNamesUsage(t *testing.T) { + type Resource struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + src := Resource{Name: "my-resource", Token: "s3cr3t"} + v, err := FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + fields := SensitiveFieldNames(reflect.TypeFor[Resource]()) + assert.True(t, fields["token"], "token should be identified as sensitive") + + // Verify the value round-tripped correctly before masking. + tok, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("token"))) + require.NoError(t, err) + assert.Equal(t, "s3cr3t", tok.MustString()) +} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index 6f9269f4974..612f9c26fad 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -153,10 +153,18 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { // of where it is used. This also means that relative path resolution is done // relative to where a variable is used, not where it is defined. // + // Preserve sensitivity: NewValue strips the secretString wrapper, so use + // NewSensitiveValue when the resolved value is a sensitive string. + if resolved[0].IsSensitive() { + s, _ := resolved[0].AsString() + return dyn.NewSensitiveValue(s, ref.Value.Locations()), nil + } return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil } // Not pure; perform string interpolation. + // Track whether any resolved value is sensitive; if so, the result is also sensitive. + anySensitive := false for j := range ref.Matches { // The value is invalid if resolution returned [ErrSkipResolution]. // We must skip those and leave the original variable reference in place. @@ -164,7 +172,12 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { continue } + if resolved[j].IsSensitive() { + anySensitive = true + } + // Try to turn the resolved value into a string. + // Use AsString (not AsAny) to get the real value even for sensitive strings. s, ok := resolved[j].AsString() if !ok { // Only allow primitive types to be converted to string. @@ -179,7 +192,11 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { ref.Str = strings.Replace(ref.Str, ref.Matches[j][0], s, 1) } - return dyn.NewValue(ref.Str, ref.Value.Locations()), nil + result := dyn.NewValue(ref.Str, ref.Value.Locations()) + if anySensitive { + result = result.MarkSensitive() + } + return result, nil } func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go index c2bbaae74f1..3399f2057a6 100644 --- a/libs/dyn/dynvar/resolve_test.go +++ b/libs/dyn/dynvar/resolve_test.go @@ -371,6 +371,39 @@ func TestResolveMapVariable(t *testing.T) { assert.Equal(t, "value2", getByPath(t, mapVal, "key2").MustString()) } +func TestResolveSensitivePureSubstitution(t *testing.T) { + // A pure reference to a sensitive value must produce a sensitive result. + in := dyn.V(map[string]dyn.Value{ + "secret": dyn.NewSensitiveValue("top-secret", nil), + "ref": dyn.V("${secret}"), + }) + + out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) + require.NoError(t, err) + + result := getByPath(t, out, "ref") + assert.True(t, result.IsSensitive()) + // MustString returns the real value even when sensitive. + assert.Equal(t, "top-secret", result.MustString()) +} + +func TestResolveSensitiveStringInterpolation(t *testing.T) { + // When any resolved value is sensitive, the interpolated result must also be sensitive. + in := dyn.V(map[string]dyn.Value{ + "secret": dyn.NewSensitiveValue("password123", nil), + "prefix": dyn.V("token"), + "ref": dyn.V("${prefix}:${secret}"), + }) + + out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) + require.NoError(t, err) + + result := getByPath(t, out, "ref") + assert.True(t, result.IsSensitive()) + // The real interpolated string is still accessible. + assert.Equal(t, "token:password123", result.MustString()) +} + func TestResolveSequenceVariable(t *testing.T) { in := dyn.V(map[string]dyn.Value{ "seq": dyn.V([]dyn.Value{ diff --git a/libs/dyn/jsonsaver/marshal.go b/libs/dyn/jsonsaver/marshal.go index a78a68f2998..cbeb5b28919 100644 --- a/libs/dyn/jsonsaver/marshal.go +++ b/libs/dyn/jsonsaver/marshal.go @@ -39,6 +39,17 @@ func (w wrap) MarshalJSON() ([]byte, error) { // marshalValue recursively writes JSON for a [dyn.Value] to the buffer. func marshalValue(buf *bytes.Buffer, v dyn.Value) error { + if v.IsSensitive() { + out, err := marshalNoEscape(dyn.SensitiveValueRedacted) + if err != nil { + return err + } + // The encoder writes a trailing newline, so we need to remove it. + out = out[:len(out)-1] + buf.Write(out) + return nil + } + switch v.Kind() { case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: out, err := marshalNoEscape(v.AsAny()) diff --git a/libs/dyn/jsonsaver/marshal_test.go b/libs/dyn/jsonsaver/marshal_test.go index 68c2017397f..9866f46dc37 100644 --- a/libs/dyn/jsonsaver/marshal_test.go +++ b/libs/dyn/jsonsaver/marshal_test.go @@ -64,6 +64,25 @@ func TestMarshal_Sequence(t *testing.T) { } } +func TestMarshal_Sensitive(t *testing.T) { + v := dyn.NewSensitiveValue("real-secret", nil) + b, err := Marshal(v) + if assert.NoError(t, err) { + assert.JSONEq(t, `"`+dyn.SensitiveValueRedacted+`"`, string(b)) + } +} + +func TestMarshal_SensitiveInMap(t *testing.T) { + m := dyn.NewMapping() + m.SetLoc("token", nil, dyn.NewSensitiveValue("my-token", nil)) + m.SetLoc("name", nil, dyn.V("public")) + + b, err := Marshal(dyn.V(m)) + if assert.NoError(t, err) { + assert.JSONEq(t, `{"token":"`+dyn.SensitiveValueRedacted+`","name":"public"}`, string(b)) + } +} + func TestMarshal_Complex(t *testing.T) { map1 := dyn.NewMapping() map1.SetLoc("str1", nil, dyn.V("value1")) diff --git a/libs/dyn/kind.go b/libs/dyn/kind.go index 1890e9e2ccd..684239503b3 100644 --- a/libs/dyn/kind.go +++ b/libs/dyn/kind.go @@ -25,7 +25,7 @@ func kindOf(v any) Kind { return KindMap case []Value: return KindSequence - case string: + case string, secretString: return KindString case bool: return KindBool diff --git a/libs/dyn/sensitive.go b/libs/dyn/sensitive.go new file mode 100644 index 00000000000..e6c363d95ab --- /dev/null +++ b/libs/dyn/sensitive.go @@ -0,0 +1,31 @@ +package dyn + +import "slices" + +// SensitiveValueRedacted is the placeholder emitted in JSON/YAML output +// whenever a sensitive string value is serialized. +const SensitiveValueRedacted = "" + +// secretString is the internal storage type for sensitive string values. +// Using a distinct Go type (rather than a plain string plus a flag) makes +// sensitivity impossible to strip by accident: every path that copies v.v +// preserves the wrapper automatically, and a switch on v.v that handles +// `string` but not `secretString` will miss it — making omissions auditable. +// +// Kind() still returns KindString so all existing switch-on-Kind logic +// continues to work unchanged. +type secretString struct { + value string +} + +// NewSensitiveValue returns a KindString Value whose content is treated as +// sensitive. JSON and YAML serializers replace it with [SensitiveValueRedacted]; +// AsString / MustString still return the real value for use in the deployment +// pipeline. +func NewSensitiveValue(s string, loc []Location) Value { + return Value{ + v: secretString{s}, + k: KindString, + l: slices.Clone(loc), + } +} diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 72803511b80..e6ea328f082 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -73,6 +73,24 @@ func (v Value) AppendLocationsFromValue(w Value) Value { } } +// IsSensitive reports whether this value holds a sensitive string. +// Sensitivity is encoded in the type of v.v (secretString vs plain string), +// so it is preserved automatically whenever v.v is copied. +func (v Value) IsSensitive() bool { + _, ok := v.v.(secretString) + return ok +} + +// MarkSensitive returns a copy of this value marked as sensitive. +// If the value is already a KindString, its content is re-wrapped as a +// secretString; otherwise the value is returned unchanged. +func (v Value) MarkSensitive() Value { + if s, ok := v.v.(string); ok { + v.v = secretString{s} + } + return v +} + func (v Value) Kind() Kind { return v.k } @@ -120,6 +138,10 @@ func (v Value) AsAny() any { case KindNil: return v.v case KindString: + // secretString holds a sensitive value; return the redaction placeholder. + if _, ok := v.v.(secretString); ok { + return SensitiveValueRedacted + } return v.v case KindBool: return v.v diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go index 8717aae620e..a2edf4e0128 100644 --- a/libs/dyn/value_test.go +++ b/libs/dyn/value_test.go @@ -50,6 +50,34 @@ func TestValueIsValid(t *testing.T) { assert.True(t, intValue.IsValid()) } +func TestNewSensitiveValue(t *testing.T) { + v := dyn.NewSensitiveValue("secret-value", nil) + assert.True(t, v.IsSensitive()) + assert.Equal(t, dyn.KindString, v.Kind()) + + // AsAny returns the redaction placeholder. + assert.Equal(t, dyn.SensitiveValueRedacted, v.AsAny()) + + // AsString / MustString return the real underlying value. + s, ok := v.AsString() + assert.True(t, ok) + assert.Equal(t, "secret-value", s) + assert.Equal(t, "secret-value", v.MustString()) +} + +func TestPlainStringNotSensitive(t *testing.T) { + v := dyn.V("hello") + assert.False(t, v.IsSensitive()) + assert.Equal(t, "hello", v.AsAny()) +} + +func TestSensitivePreservedByWithLocations(t *testing.T) { + locs := []dyn.Location{{File: "file", Line: 1, Column: 1}} + v := dyn.NewSensitiveValue("secret", nil).WithLocations(locs) + assert.True(t, v.IsSensitive()) + assert.Equal(t, locs, v.Locations()) +} + func TestIsZero(t *testing.T) { assert.True(t, dyn.V(0).IsZero(), "int") assert.True(t, dyn.V(int(0)).IsZero(), "int") diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go index a33ecd38ed8..792db7dbe00 100644 --- a/libs/dyn/value_underlying.go +++ b/libs/dyn/value_underlying.go @@ -40,9 +40,16 @@ func (v Value) MustSequence() []Value { // AsString returns the underlying string if this value is a string, // the zero value and false otherwise. +// For sensitive values (stored as [secretString]) it returns the real plaintext. func (v Value) AsString() (string, bool) { - vv, ok := v.v.(string) - return vv, ok + switch vv := v.v.(type) { + case string: + return vv, true + case secretString: + return vv.value, true + default: + return "", false + } } // MustString returns the underlying string if this value is a string, diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go index 4c302e26f0d..01b345923d2 100644 --- a/libs/dyn/yamlsaver/saver.go +++ b/libs/dyn/yamlsaver/saver.go @@ -72,6 +72,10 @@ func (s *saver) toYamlNode(v dyn.Value) (*yaml.Node, error) { } func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node, error) { + if v.IsSensitive() { + return &yaml.Node{Kind: yaml.ScalarNode, Value: dyn.SensitiveValueRedacted, Style: style}, nil + } + switch v.Kind() { case dyn.KindMap: m, _ := v.AsMap() diff --git a/libs/filer/errors.go b/libs/filer/errors.go index 67f15ecb013..8648502c229 100644 --- a/libs/filer/errors.go +++ b/libs/filer/errors.go @@ -106,6 +106,10 @@ func (err cannotDeleteRootError) Is(other error) bool { // when attempting to create a directory but lacking write permissions. type permissionError struct { path string + // err is the underlying API error, preserved so callers can inspect its + // error_code (e.g. to distinguish a real permission denial from a workspace + // directory that is at its child-node limit, which the API also reports as 403). + err error } func (err permissionError) Error() string { @@ -115,3 +119,7 @@ func (err permissionError) Error() string { func (err permissionError) Is(other error) bool { return other == fs.ErrPermission } + +func (err permissionError) Unwrap() error { + return err.err +} diff --git a/libs/filer/errors_test.go b/libs/filer/errors_test.go index a1fb066e099..e12fe946450 100644 --- a/libs/filer/errors_test.go +++ b/libs/filer/errors_test.go @@ -4,7 +4,9 @@ import ( "io/fs" "testing" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFileAlreadyExistsError_Is(t *testing.T) { @@ -94,3 +96,16 @@ func TestPermissionError_Error(t *testing.T) { err := permissionError{path: "/test/path"} assert.Equal(t, "access denied: /test/path", err.Error()) } + +func TestPermissionError_Unwrap(t *testing.T) { + // A wrapped API error must remain matchable as fs.ErrPermission while also + // being reachable via errors.As so callers can inspect its error_code. + apiErr := &apierr.APIError{StatusCode: 403, ErrorCode: "MAX_CHILD_NODE_SIZE_EXCEEDED"} + err := permissionError{path: "/test/path", err: apiErr} + + assert.ErrorIs(t, err, fs.ErrPermission) + + var got *apierr.APIError + require.ErrorAs(t, err, &got) + assert.Equal(t, "MAX_CHILD_NODE_SIZE_EXCEEDED", got.ErrorCode) +} diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 232cbe20b9c..bf349b6cb26 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -4,26 +4,39 @@ import ( "cmp" "context" "errors" - "fmt" "io" "io/fs" - "maps" "net/http" - "net/url" "path" "slices" - "strings" "time" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/client" - "github.com/databricks/databricks-sdk-go/listing" - "github.com/databricks/databricks-sdk-go/service/files" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/sdk-go/core/apierr" + files "github.com/databricks/sdk-go/files/v2" + "github.com/databricks/sdk-go/options/client" "golang.org/x/sync/errgroup" ) +// cloudResponseHeaderTimeout bounds the wait for a cloud storage response header +// on a large-file part transfer. It matches the files/v2 engine's own default. +const cloudResponseHeaderTimeout = 60 * time.Second + +// httpStatus returns the HTTP status code of err if it is (or wraps) an +// [apierr.APIError], and -1 otherwise. The filer keys its error mapping off the +// HTTP status rather than the SDK's canonical codes.Code: the Files API reports +// "path already exists" as 409 Conflict, which the SDK maps to codes.Aborted +// (not codes.AlreadyExists), so matching on the code would miss it. +func httpStatus(err error) int { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + return aerr.HTTPStatusCode() + } + return -1 +} + // As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent // requests and 100 QPS. We limit the number of concurrent requests to 5 to // avoid hitting the rate limit. @@ -88,112 +101,208 @@ func (e filesApiDirEntry) Info() (fs.FileInfo, error) { return e.i, nil } +// uploadConcurrency bounds the concurrent part uploads of a large-file +// (multipart) write. Parts go to cloud storage rather than the rate-limited +// Files API, so this can fan out wider than the file-level copy parallelism. +const uploadConcurrency = 64 + +// multipartUploadEnvVar gates whether large Volumes writes are split into parts +// by the files/v2 upload engine. The engine is new, so multipart is off by +// default: when unset or not truthy, Write sends a single-shot PUT (via the +// files/v2 UploadFile endpoint), leaving fs cp and bundle behavior unchanged. +const multipartUploadEnvVar = "DATABRICKS_EXPERIMENTAL_MULTIPART_UPLOAD" + +// MultipartUploadEnabled reports whether large files written to UC Volumes are +// split into parts by the files/v2 upload engine, gated by +// DATABRICKS_EXPERIMENTAL_MULTIPART_UPLOAD (off by default). Both paths use the +// files/v2 client; the flag only selects the multipart engine over a single-shot +// PUT. +func MultipartUploadEnabled(ctx context.Context) bool { + enabled, _ := env.GetBool(ctx, multipartUploadEnvVar) + return enabled +} + +// uploadProgressKey is the context key for an optional large-file upload +// progress callback. +type uploadProgressKey struct{} + +// WithUploadProgress returns a context carrying a progress callback for +// large-file (multipart) uploads. FilesClient.Write forwards it to the upload +// engine; it has no effect on writes that do not go through the engine (small +// files, non-seekable streams, non-Volumes targets, or when multipart upload is +// disabled). +func WithUploadProgress(ctx context.Context, fn files.ProgressFunc) context.Context { + return context.WithValue(ctx, uploadProgressKey{}, fn) +} + +func uploadProgressFromContext(ctx context.Context) files.ProgressFunc { + fn, _ := ctx.Value(uploadProgressKey{}).(files.ProgressFunc) + return fn +} + // FilesClient implements the [Filer] interface for the Files API backend. type FilesClient struct { - workspaceClient *databricks.WorkspaceClient - apiClient *client.DatabricksClient + client *files.Client // File operations will be relative to this path. root WorkspaceRootPath + + // Large files are uploaded with the multipart engine. The limiter and transfer + // client are shared across every Write on this filer, so concurrent uploads + // (e.g. fs cp -r) draw from one bounded budget and one connection pool. + limiter files.Limiter + transferClient *http.Client } -func NewFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) { - apiClient, err := client.New(w.Config) +func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) { + c, err := newFilesAPIClient(ctx, w.Config) if err != nil { return nil, err } return &FilesClient{ - workspaceClient: w, - apiClient: apiClient, + client: c, root: NewWorkspaceRootPath(root), + + limiter: files.NewLimiter(uploadConcurrency), + transferClient: newTransferClient(uploadConcurrency), }, nil } -func (w *FilesClient) urlPath(name string) (string, string, error) { - absPath, err := w.root.Join(name) - if err != nil { - return "", "", err - } - - // The user specified part of the path must be escaped. - urlPath := "/api/2.0/fs/files/" + url.PathEscape(strings.TrimLeft(absPath, "/")) +// newTransferClient returns an HTTP client for the cloud-leg part transfers of a +// large-file upload, sized for n concurrent transfers so idle connections are +// reused rather than re-dialed (Go's default of 2 per host would force +// re-dialing). It attaches no Databricks credentials (presigned URLs are +// self-authenticating) and sets no whole-request timeout, which would abort a +// legitimately long transfer; the upload is bounded by its context instead. +// files/v2 builds an equivalent client internally when one is not supplied; this +// filer supplies a shared one so all writes on it draw from a single pool. +func newTransferClient(n int) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = cloudResponseHeaderTimeout + transport.MaxIdleConnsPerHost = n + transport.MaxIdleConns = max(transport.MaxIdleConns, n) + return &http.Client{Transport: transport} +} - return absPath, urlPath, nil +// newFilesAPIClient builds the files/v2 client from the CLI's already resolved +// config, reusing its auth instead of re-reading a profile. cfg is passed by +// pointer because config.Config embeds a sync.Mutex and must not be copied. +func newFilesAPIClient(ctx context.Context, cfg *config.Config) (*files.Client, error) { + copts := []client.Option{ + client.WithHost(cfg.Host), + client.WithCredentials(configCredentials{cfg: cfg}), + client.WithoutProfileResolution(), + } + // The workspace routing header is needed on unified ("SPOG") hosts; the CLI's + // "none" sentinel means "no workspace ID", so it is not forwarded. + if id := cfg.WorkspaceID; id != "" && id != auth.WorkspaceIDNone { + copts = append(copts, client.WithWorkspaceID(id)) + } + return files.NewClient(ctx, copts...) } func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { - absPath, urlPath, err := w.urlPath(name) + absPath, err := w.root.Join(name) if err != nil { return err } // Check that target path exists if CreateParentDirectories mode is not set if !slices.Contains(mode, CreateParentDirectories) { - err := w.workspaceClient.Files.GetDirectoryMetadataByDirectoryPath(ctx, path.Dir(absPath)) + dir := path.Dir(absPath) + _, err := w.client.GetDirectoryMetadata(ctx, &files.GetDirectoryMetadataRequest{DirectoryPath: &dir}) if err != nil { - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err + // This API returns a 404 if the directory doesn't exist. + if httpStatus(err) == http.StatusNotFound { + return noSuchDirectoryError{dir} } - - // This API returns a 404 if the file doesn't exist. - if aerr.StatusCode == http.StatusNotFound { - return noSuchDirectoryError{path.Dir(absPath)} - } - return err } } overwrite := slices.Contains(mode, OverwriteIfExists) - urlPath = fmt.Sprintf("%s?overwrite=%t", urlPath, overwrite) - headers := map[string]string{"Content-Type": "application/octet-stream"} - maps.Copy(headers, auth.WorkspaceIDHeaders(w.workspaceClient.Config)) - err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, nil, reader, nil) - // Return early on success. + // When the multipart flag is enabled, seekable uploads go through the files/v2 + // upload engine, which sends small files in a single PUT and splits large ones + // into parts. Non-seekable streams and the flag-off case use the single-shot + // UploadFile endpoint below: the former to avoid buffering the whole stream in + // memory, the latter to keep the default behavior a plain PUT. The engine + // recovers an io.ReaderAt for concurrent positioned reads when the source + // provides one (a local file). + if MultipartUploadEnabled(ctx) && isSeekable(reader) { + opts := []files.UploadOption{ + files.WithOverwrite(overwrite), + files.WithLimiter(w.limiter), + files.WithTransferClient(w.transferClient), + } + // A caller (fs cp) can attach a progress callback via the context to + // render an upload bar; it is absent for other writers (e.g. bundle). + if fn := uploadProgressFromContext(ctx); fn != nil { + opts = append(opts, files.WithProgress(fn)) + } + _, uerr := w.client.Upload(ctx, absPath, reader, opts...) + return mapUploadError(uerr, absPath) + } + + _, err = w.client.UploadFile(ctx, &files.UploadFileRequest{ + FilePath: &absPath, + Contents: io.NopCloser(reader), + Overwrite: &overwrite, + }) if err == nil { return nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) + // This API returns 409 if a file already exists at the path. + if httpStatus(err) == http.StatusConflict { + return fileAlreadyExistsError{absPath} + } + return err +} + +// isSeekable reports whether r can be seeked, without moving it: a no-op +// Seek(0, io.SeekCurrent) returns the current offset for a working seeker and +// errors for a broken one. Probing this way (rather than seeking to the end and +// back) guarantees the reader keeps its position, so a false result never leaves +// it parked at EOF for the single-shot fallback, which reads from the current +// offset and would otherwise upload a truncated object. The engine sizes the +// stream itself once it takes over. +func isSeekable(r io.Reader) bool { + s, ok := r.(io.Seeker) if !ok { - return err + return false } + _, err := s.Seek(0, io.SeekCurrent) + return err == nil +} - // This API returns 409 if the file already exists, when the object type is file - if aerr.StatusCode == http.StatusConflict && aerr.ErrorCode == "ALREADY_EXISTS" { +// mapUploadError translates the upload engine's already-exists sentinel into the +// filer's error so skip-if-exists logic (which checks fs.ErrExist) keeps working. +// A nil error passes through unchanged. +func mapUploadError(err error, absPath string) error { + if errors.Is(err, files.ErrAlreadyExists) { return fileAlreadyExistsError{absPath} } - return err } func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { - absPath, urlPath, err := w.urlPath(name) + absPath, err := w.root.Join(name) if err != nil { return nil, err } - var reader io.ReadCloser - err = w.apiClient.Do(ctx, http.MethodGet, urlPath, auth.WorkspaceIDHeaders(w.workspaceClient.Config), nil, nil, &reader) + resp, err := w.client.DownloadFile(ctx, &files.DownloadFileRequest{FilePath: &absPath}) // Return early on success. if err == nil { - return reader, nil - } - - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return nil, err + return resp.Contents, nil } // This API returns a 404 if the specified path does not exist. - if aerr.StatusCode == http.StatusNotFound { + if httpStatus(err) == http.StatusNotFound { // Check if the path is a directory. If so, return not a file error. if _, err := w.statDir(ctx, name); err == nil { return nil, notAFile{absPath} @@ -217,21 +326,15 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error { return cannotDeleteRootError{} } - err = w.workspaceClient.Files.DeleteByFilePath(ctx, absPath) + _, err = w.client.DeleteFile(ctx, &files.DeleteFileRequest{FilePath: &absPath}) // Return early on success. if err == nil { return nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err - } - // This files delete API returns a 404 if the specified path does not exist. - if aerr.StatusCode == http.StatusNotFound { + if httpStatus(err) == http.StatusNotFound { return fileDoesNotExistError{absPath} } @@ -249,27 +352,20 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { return cannotDeleteRootError{} } - err = w.workspaceClient.Files.DeleteDirectoryByDirectoryPath(ctx, absPath) + _, err = w.client.DeleteDirectory(ctx, &files.DeleteDirectoryRequest{DirectoryPath: &absPath}) - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err + // Return early on success. + if err == nil { + return nil } - // The directory delete API returns a 400 if the directory is not empty - if aerr.StatusCode == http.StatusBadRequest { - var reasons []string - details := aerr.ErrorDetails() - if details.ErrorInfo != nil { - reasons = append(reasons, details.ErrorInfo.Reason) - } - // Error code 400 is generic and can be returned for other reasons. Make - // sure one of the reasons for the error is that the directory is not empty. - if !slices.Contains(reasons, "FILES_API_DIRECTORY_IS_NOT_EMPTY") { - return err + // The directory delete API returns a 400 if the directory is not empty. That + // status is generic, so confirm the specific reason before mapping it. + if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.HTTPStatusCode() == http.StatusBadRequest { + if info := aerr.Details().ErrorInfo; info != nil && info.Reason == "FILES_API_DIRECTORY_IS_NOT_EMPTY" { + return directoryNotEmptyError{absPath} } - return directoryNotEmptyError{absPath} + return err } // On GCS-backed storage a directory created implicitly is just a key prefix @@ -277,7 +373,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { // deleted. The delete API then returns 404 for that already-vanished // directory; treat it as a not-found error so recursive delete can consider // its goal satisfied. - if apierr.IsMissing(err) { + if httpStatus(err) == http.StatusNotFound { return noSuchDirectoryError{absPath} } return err @@ -377,48 +473,37 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, return nil, err } - iter := w.workspaceClient.Files.ListDirectoryContents(ctx, files.ListDirectoryContentsRequest{ - DirectoryPath: absPath, - }) - - files, err := listing.ToSlice(ctx, iter) - - // Return early on success. - if err == nil { - entries := make([]fs.DirEntry, len(files)) - for i, file := range files { - entries[i] = filesApiDirEntry{ - i: filesApiFileInfo{ - absPath: file.Path, - isDir: file.IsDirectory, - fileSize: file.FileSize, - lastModified: file.LastModified, - }, + var entries []fs.DirEntry + for entry, err := range w.client.ListDirectoryContentsIter(ctx, &files.ListDirectoryContentsRequest{ + DirectoryPath: &absPath, + }) { + if err != nil { + // This API returns a 404 if the specified path does not exist. + if httpStatus(err) == http.StatusNotFound { + // Check if the path is a file. If so, return not a directory error. + if _, ferr := w.statFile(ctx, name); ferr == nil { + return nil, notADirectory{absPath} + } + + // No file or directory exists at the specified path. Return no such directory error. + return nil, noSuchDirectoryError{absPath} } + return nil, err } - // Sort by name for parity with os.ReadDir. - slices.SortFunc(entries, func(a, b fs.DirEntry) int { return cmp.Compare(a.Name(), b.Name()) }) - return entries, nil - } - - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return nil, err + entries = append(entries, filesApiDirEntry{ + i: filesApiFileInfo{ + absPath: value(entry.Path), + isDir: value(entry.IsDirectory), + fileSize: int64(value(entry.FileSize)), + lastModified: int64(value(entry.LastModified)), + }, + }) } - // This API returns a 404 if the specified path does not exist. - if aerr.StatusCode == http.StatusNotFound { - // Check if the path is a file. If so, return not a directory error. - if _, err := w.statFile(ctx, name); err == nil { - return nil, notADirectory{absPath} - } - - // No file or directory exists at the specified path. Return no such directory error. - return nil, noSuchDirectoryError{absPath} - } - return nil, err + // Sort by name for parity with os.ReadDir. + slices.SortFunc(entries, func(a, b fs.DirEntry) int { return cmp.Compare(a.Name(), b.Name()) }) + return entries, nil } func (w *FilesClient) Mkdir(ctx context.Context, name string) error { @@ -427,12 +512,11 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error { return err } - err = w.workspaceClient.Files.CreateDirectory(ctx, files.CreateDirectoryRequest{ - DirectoryPath: absPath, - }) + _, err = w.client.CreateDirectory(ctx, &files.CreateDirectoryRequest{DirectoryPath: &absPath}) - // Special handling of this error only if it is an API error. - if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.StatusCode == http.StatusConflict { + // This API returns a 409 when a file already exists at the path (the create + // is not idempotent over a file). + if httpStatus(err) == http.StatusConflict { return fileAlreadyExistsError{absPath} } @@ -446,25 +530,19 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e return nil, err } - fileInfo, err := w.workspaceClient.Files.GetMetadataByFilePath(ctx, absPath) + resp, err := w.client.GetFileMetadata(ctx, &files.GetFileMetadataRequest{FilePath: &absPath}) // If the HEAD requests succeeds, the file exists. if err == nil { return filesApiFileInfo{ absPath: absPath, isDir: false, - fileSize: fileInfo.ContentLength, + fileSize: value(resp.ContentLength), }, nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return nil, err - } - // This API returns a 404 if the specified path does not exist. - if aerr.StatusCode == http.StatusNotFound { + if httpStatus(err) == http.StatusNotFound { return nil, fileDoesNotExistError{absPath} } @@ -478,21 +556,15 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er return nil, err } - err = w.workspaceClient.Files.GetDirectoryMetadataByDirectoryPath(ctx, absPath) + _, err = w.client.GetDirectoryMetadata(ctx, &files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}) // If the HEAD requests succeeds, the directory exists. if err == nil { return filesApiFileInfo{absPath: absPath, isDir: true}, nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return nil, err - } - // The directory metadata API returns a 404 if the specified path does not exist. - if aerr.StatusCode == http.StatusNotFound { + if httpStatus(err) == http.StatusNotFound { return nil, noSuchDirectoryError{absPath} } @@ -516,3 +588,12 @@ func (w *FilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error // Since the path is not a directory, assume that it is a file and issue a stat call. return w.statFile(ctx, name) } + +// value returns *p, or the zero value of T when p is nil. +func value[T any](p *T) T { + if p == nil { + var zero T + return zero + } + return *p +} diff --git a/libs/filer/files_client_auth.go b/libs/filer/files_client_auth.go new file mode 100644 index 00000000000..6ed9098532f --- /dev/null +++ b/libs/filer/files_client_auth.go @@ -0,0 +1,50 @@ +package filer + +import ( + "context" + "net/http" + + "github.com/databricks/databricks-sdk-go/config" + sdkauth "github.com/databricks/sdk-go/auth" +) + +// configCredentials adapts the CLI's resolved SDK config into the credentials +// interface expected by the files/v2 client. It signs a throwaway +// request with config.Authenticate and hands the resulting headers to the +// client, so the files/v2 client reuses the exact auth the rest of the CLI +// already resolved instead of re-reading a profile. +type configCredentials struct { + cfg *config.Config +} + +// Name reports the resolved auth mechanism (e.g. "pat", "oauth-m2m"), which the +// files/v2 client folds into its User-Agent. The CLI resolves auth when it +// builds the workspace client, so AuthType is normally set before the filer is +// created; fall back to "unk" when it is empty because the files/v2 client +// rejects an empty auth name with an "invalid value" error at construction. +func (c configCredentials) Name() string { + if c.cfg.AuthType == "" { + return "unk" + } + return c.cfg.AuthType +} + +func (c configCredentials) AuthHeaders(ctx context.Context) ([]sdkauth.Header, error) { + // The URL is irrelevant: config.Authenticate only reads it to pick the auth + // scheme, and every files/v2 request targets the same workspace host. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.cfg.Host, nil) + if err != nil { + return nil, err + } + if err := c.cfg.Authenticate(req); err != nil { + return nil, err + } + + headers := make([]sdkauth.Header, 0, len(req.Header)) + for key, values := range req.Header { + for _, value := range values { + headers = append(headers, sdkauth.Header{Key: key, Value: value}) + } + } + return headers, nil +} diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 57b93f7c6cf..acc9a206d39 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -1,11 +1,17 @@ package filer import ( + "bytes" + "errors" + "fmt" + "io" "io/fs" "testing" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + files "github.com/databricks/sdk-go/files/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,7 +43,7 @@ func deleteDirectoryWithError(t *testing.T, statusCode int, errorCode, reason st }) require.NoError(t, err) - f, err := NewFilesClient(client, "/test") + f, err := NewFilesClient(t.Context(), client, "/test") require.NoError(t, err) return f.(*FilesClient).deleteDirectory(t.Context(), "dir") @@ -55,3 +61,106 @@ func TestFilesClientDeleteDirectoryNotEmpty(t *testing.T) { err := deleteDirectoryWithError(t, 400, "INVALID_PARAMETER_VALUE", "FILES_API_DIRECTORY_IS_NOT_EMPTY") assert.ErrorIs(t, err, fs.ErrInvalid) } + +func newTestFilesClient(t *testing.T) Filer { + t.Helper() + + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewFilesClient(t.Context(), client, "/") + require.NoError(t, err) + return f +} + +func TestFilesClientMkdirWhenFileExists(t *testing.T) { + // The Files API reports "a file already exists at this path" as a 409, which + // the SDK maps to codes.Aborted (not codes.AlreadyExists); the filer keys off + // the HTTP status so it still surfaces as fs.ErrExist. + ctx := t.Context() + f := newTestFilesClient(t) + + require.NoError(t, f.Mkdir(ctx, "/Volumes/main/schema/vol")) + require.NoError(t, f.Write(ctx, "/Volumes/main/schema/vol/hello", bytes.NewReader([]byte("abc")))) + + err := f.Mkdir(ctx, "/Volumes/main/schema/vol/hello") + assert.ErrorIs(t, err, fs.ErrExist) +} + +// onlyReader hides the Seek method of an underlying reader, modelling a +// non-seekable stream (e.g. a remote download body). +type onlyReader struct{ io.Reader } + +func TestIsSeekable(t *testing.T) { + in := []byte("hello, files API") + + if !isSeekable(bytes.NewReader(in)) { + t.Fatal("bytes.Reader should be seekable") + } + + // The position must be left at the start so a subsequent read covers every byte. + r := bytes.NewReader(in) + if !isSeekable(r) { + t.Fatal("bytes.Reader should be seekable") + } + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, in) { + t.Errorf("read %q after isSeekable, want %q (position not left at start)", b, in) + } + + if isSeekable(onlyReader{bytes.NewReader(in)}) { + t.Error("a non-seekable reader should report false") + } +} + +func TestMultipartUploadEnabled(t *testing.T) { + ctx := t.Context() + if MultipartUploadEnabled(ctx) { + t.Error("multipart upload must be disabled by default") + } + for _, on := range []string{"true", "1", "yes", "on"} { + if !MultipartUploadEnabled(env.Set(ctx, multipartUploadEnvVar, on)) { + t.Errorf("value %q should enable multipart upload", on) + } + } + for _, off := range []string{"false", "0", "", "nonsense"} { + if MultipartUploadEnabled(env.Set(ctx, multipartUploadEnvVar, off)) { + t.Errorf("value %q should not enable multipart upload", off) + } + } +} + +func TestMapUploadError(t *testing.T) { + const p = "/Volumes/c/s/v/f.bin" + + if err := mapUploadError(nil, p); err != nil { + t.Errorf("nil error should pass through, got %v", err) + } + + // The engine's already-exists sentinel (even wrapped) must surface as fs.ErrExist + // so skip-if-exists keeps working. + for _, in := range []error{ + files.ErrAlreadyExists, + fmt.Errorf("upload failed: %w", files.ErrAlreadyExists), + } { + got := mapUploadError(in, p) + if !errors.Is(got, fs.ErrExist) { + t.Errorf("mapUploadError(%v) = %v, want errors.Is fs.ErrExist", in, got) + } + } + + // Other errors pass through unchanged. + other := errors.New("boom") + if got := mapUploadError(other, p); got != other { + t.Errorf("mapUploadError(other) = %v, want it unchanged", got) + } +} diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 5d88f858a05..e1b62481243 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -191,7 +191,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { - return permissionError{absPath} + return permissionError{absPath, mkdirErr} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -217,7 +217,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // This API returns StatusForbidden when you have read access but don't have write access to a file if aerr.StatusCode == http.StatusForbidden { - return permissionError{absPath} + return permissionError{absPath, aerr} } return err diff --git a/experimental/genie/adapter.go b/libs/genie/adapter.go similarity index 99% rename from experimental/genie/adapter.go rename to libs/genie/adapter.go index b0f4b8b9161..c84b319c7e1 100644 --- a/experimental/genie/adapter.go +++ b/libs/genie/adapter.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/databricks/cli/experimental/genie/agentstream" + "github.com/databricks/cli/libs/genie/agentstream" ) // SSE event type constants. diff --git a/experimental/genie/adapter_test.go b/libs/genie/adapter_test.go similarity index 99% rename from experimental/genie/adapter_test.go rename to libs/genie/adapter_test.go index 4daccac88fd..ba821dfbd99 100644 --- a/experimental/genie/adapter_test.go +++ b/libs/genie/adapter_test.go @@ -3,7 +3,7 @@ package genie import ( "testing" - "github.com/databricks/cli/experimental/genie/agentstream" + "github.com/databricks/cli/libs/genie/agentstream" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/experimental/genie/agentstream/event.go b/libs/genie/agentstream/event.go similarity index 100% rename from experimental/genie/agentstream/event.go rename to libs/genie/agentstream/event.go diff --git a/experimental/genie/agentstream/markdown.go b/libs/genie/agentstream/markdown.go similarity index 100% rename from experimental/genie/agentstream/markdown.go rename to libs/genie/agentstream/markdown.go diff --git a/experimental/genie/agentstream/markdown_test.go b/libs/genie/agentstream/markdown_test.go similarity index 100% rename from experimental/genie/agentstream/markdown_test.go rename to libs/genie/agentstream/markdown_test.go diff --git a/experimental/genie/agentstream/renderer.go b/libs/genie/agentstream/renderer.go similarity index 100% rename from experimental/genie/agentstream/renderer.go rename to libs/genie/agentstream/renderer.go diff --git a/experimental/genie/agentstream/renderer_test.go b/libs/genie/agentstream/renderer_test.go similarity index 100% rename from experimental/genie/agentstream/renderer_test.go rename to libs/genie/agentstream/renderer_test.go diff --git a/experimental/genie/agentstream/sql.go b/libs/genie/agentstream/sql.go similarity index 100% rename from experimental/genie/agentstream/sql.go rename to libs/genie/agentstream/sql.go diff --git a/experimental/genie/agentstream/sql_test.go b/libs/genie/agentstream/sql_test.go similarity index 100% rename from experimental/genie/agentstream/sql_test.go rename to libs/genie/agentstream/sql_test.go diff --git a/experimental/genie/agentstream/sse.go b/libs/genie/agentstream/sse.go similarity index 100% rename from experimental/genie/agentstream/sse.go rename to libs/genie/agentstream/sse.go diff --git a/experimental/genie/agentstream/sse_test.go b/libs/genie/agentstream/sse_test.go similarity index 100% rename from experimental/genie/agentstream/sse_test.go rename to libs/genie/agentstream/sse_test.go diff --git a/experimental/genie/agentstream/termchart.go b/libs/genie/agentstream/termchart.go similarity index 100% rename from experimental/genie/agentstream/termchart.go rename to libs/genie/agentstream/termchart.go diff --git a/experimental/genie/agentstream/termchart_test.go b/libs/genie/agentstream/termchart_test.go similarity index 100% rename from experimental/genie/agentstream/termchart_test.go rename to libs/genie/agentstream/termchart_test.go diff --git a/experimental/genie/client.go b/libs/genie/client.go similarity index 86% rename from experimental/genie/client.go rename to libs/genie/client.go index a1ce57a3194..73522afde5a 100644 --- a/experimental/genie/client.go +++ b/libs/genie/client.go @@ -7,8 +7,8 @@ import ( "io" "maps" - "github.com/databricks/cli/experimental/genie/agentstream" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/genie/agentstream" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/config" @@ -18,6 +18,11 @@ import ( // "onechat" even though the CLI command is "genie", so the path keeps that name. const genieResponsesPath = "/api/2.0/data-rooms/tools/onechat/responses" +// SourceDatabricksCLI is the source tag sent on every request so the OneChat +// backend attributes CLI traffic distinctly (maps to OneChatEventSource. +// DATABRICKS_CLI / ConversationSource.CONVERSATION_SOURCE_DATABRICKS_CLI). +const SourceDatabricksCLI = "databricks_cli" + // StreamingTimeoutSeconds is how long the SDK waits between body reads // before canceling the stream. The Genie agent can take minutes between SSE // events when executing multi-step tool calls (search, SQL, etc.), so this @@ -29,6 +34,10 @@ const StreamingTimeoutSeconds = 600 // Genie picks a default warehouse itself. A non-empty conversationID continues // that conversation instead of starting a new one. func BuildRequest(question, warehouseID, conversationID string) GenieRequest { + // The CLI runs in a terminal that cannot render visualizations, so viz is + // always disabled. Bound as a local because GenieRequest.EnableViz is a + // *bool (see the type's comment for why it can't be a plain bool). + enableViz := false return GenieRequest{ Input: []InputItem{ { @@ -41,6 +50,8 @@ func BuildRequest(question, warehouseID, conversationID string) GenieRequest { }, WarehouseID: warehouseID, ConversationID: conversationID, + Source: SourceDatabricksCLI, + EnableViz: &enableViz, } } diff --git a/experimental/genie/client_test.go b/libs/genie/client_test.go similarity index 90% rename from experimental/genie/client_test.go rename to libs/genie/client_test.go index b8a2502a30d..28cecffc003 100644 --- a/experimental/genie/client_test.go +++ b/libs/genie/client_test.go @@ -8,7 +8,7 @@ import ( "net/http/httptest" "testing" - "github.com/databricks/cli/experimental/genie/agentstream" + "github.com/databricks/cli/libs/genie/agentstream" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" @@ -17,7 +17,9 @@ import ( func TestBuildRequest_WireFormat(t *testing.T) { // The backend expects camelCase warehouseId; this pins the wire format, - // not just the Go field values. + // not just the Go field values. source and enable_viz are always sent: + // the CLI tags its traffic (databricks_cli) and disables viz because a + // terminal cannot render charts. b, err := json.Marshal(BuildRequest("What tables exist?", "abc123", "")) require.NoError(t, err) assert.JSONEq(t, `{ @@ -26,10 +28,22 @@ func TestBuildRequest_WireFormat(t *testing.T) { "role": "user", "content": [{"type": "input_text", "text": "What tables exist?"}] }], - "warehouseId": "abc123" + "warehouseId": "abc123", + "source": "databricks_cli", + "enable_viz": false }`, string(b)) } +func TestBuildRequest_TagsSourceAndDisablesViz(t *testing.T) { + // enable_viz:false must always be on the wire (a plain false would be + // dropped by omitempty, letting the backend default viz on), and source + // tags CLI traffic for backend attribution. + b, err := json.Marshal(BuildRequest("q", "", "")) + require.NoError(t, err) + assert.Contains(t, string(b), `"source":"databricks_cli"`) + assert.Contains(t, string(b), `"enable_viz":false`) +} + func TestBuildRequest_OmitsEmptyWarehouse(t *testing.T) { b, err := json.Marshal(BuildRequest("q", "", "")) require.NoError(t, err) diff --git a/experimental/genie/types.go b/libs/genie/types.go similarity index 90% rename from experimental/genie/types.go rename to libs/genie/types.go index fb7afa3c37d..b63e6777e56 100644 --- a/experimental/genie/types.go +++ b/libs/genie/types.go @@ -7,6 +7,15 @@ type GenieRequest struct { // Tag is camelCase (conversationId), unlike the response's snake_case // conversation_id; the server does not read the snake_case form on input. ConversationID string `json:"conversationId,omitempty"` + // Source identifies this client to the OneChat backend so CLI traffic is + // attributed distinctly from webapp/Slack/MCP. Always SourceDatabricksCLI; + // the server matches it case-insensitively against a fixed allowlist. + Source string `json:"source,omitempty"` + // EnableViz is a pointer so it always serializes (a plain false would be + // dropped by omitempty). The CLI sends false: a terminal cannot render the + // chart/plot items, so we tell the agent to skip registering visualization + // tools. Nil would let the backend default (viz on) apply. + EnableViz *bool `json:"enable_viz,omitempty"` } // InputItem is a message in the input array. diff --git a/experimental/genie/types_test.go b/libs/genie/types_test.go similarity index 100% rename from experimental/genie/types_test.go rename to libs/genie/types_test.go diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index e103b86e5ff..5dff23a67dd 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -47,7 +47,12 @@ func RepoConstraintBaseURL(ctx context.Context) string { if repo == "" { return "" } - return "https://raw.githubusercontent.com/" + repo + "/main" + // The databricks/environments repo nests its language ecosystems under a + // top-level directory, so the Python artifacts live at python// + // pyproject.toml (e.g. python/serverless/serverless-v5, python/dbr/), + // not at the repo root. Anchor the base URL at that python/ subtree so an + // env key of "serverless/serverless-v5" resolves to the real path. + return "https://raw.githubusercontent.com/" + repo + "/main/python" } // errEnvKeyNotFound is returned by fetchURL when the constraint artifact does @@ -139,7 +144,7 @@ func writeCacheAtomic(path string, data []byte) error { // reported below as a fetch-phase error. // // writeCache controls whether a successful live fetch populates the on-disk -// cache. Callers pass false for a dry run (--check), which must not mutate +// cache. Callers pass false for a dry run (--dry-run), which must not mutate // disk; an existing cache is still read for offline fallback, since reading is // not a mutation. func FetchConstraints(ctx context.Context, baseURL, envKey, cacheDir string, writeCache bool) (*Constraints, error) { @@ -164,7 +169,7 @@ func FetchConstraints(ctx context.Context, baseURL, envKey, cacheDir string, wri } // Write the cache copy (creating cacheDir if needed, atomically); non-fatal // so a read-only cacheDir doesn't break the command. Skipped under a dry - // run so --check performs no disk writes at all. + // run so --dry-run performs no disk writes at all. if writeCache { if err := writeCacheAtomic(cachePath, data); err != nil { log.Debugf(ctx, "failed to write constraint cache %s: %v", filepath.ToSlash(cachePath), err) @@ -184,7 +189,7 @@ func FetchConstraints(ctx context.Context, baseURL, envKey, cacheDir string, wri // fallback: the target resolved to an environment that isn't published. if errors.Is(fetchErr, errEnvKeyNotFound) { return nil, NewError(ErrEnvUnsupported, fetchErr, - "no published environment for %q. If this is a new runtime, try the latest LTS target (e.g. --serverless v4 or a supported --cluster DBR)", envKey) + "no published environment for %q. If this is a new runtime, try the latest LTS target (e.g. --serverless-version 5 or a supported --cluster-id DBR)", envKey) } // Network or HTTP failure: attempt to serve from cache. @@ -271,6 +276,14 @@ func parseConstraints(data []byte) (requiresPython, dbconnect string, deps []str if strings.TrimSpace(requiresPython) == "" { return "", "", nil, errors.New("constraint artifact has no [project].requires-python") } + // Reject a requires-python that is present but yields no installable floor + // (e.g. ">=3", "<3.13", "*") here, before the body is cached. This is the + // same check the pipeline applies later; running it in the pre-cache guard + // ensures an unusable 2xx body cannot overwrite a valid cached copy and + // break offline fallback, which is the guard's stated purpose. + if _, err = PythonMinorFromRequires(requiresPython); err != nil { + return "", "", nil, fmt.Errorf("constraint artifact has an unusable [project].requires-python: %w", err) + } for _, entry := range p.DependencyGroups.Dev { if isDatabricksConnectDep(entry) { diff --git a/libs/localenv/constraints_test.go b/libs/localenv/constraints_test.go index c330c4bba2f..472078749e3 100644 --- a/libs/localenv/constraints_test.go +++ b/libs/localenv/constraints_test.go @@ -18,9 +18,10 @@ func TestRepoConstraintBaseURL(t *testing.T) { // can report the missing source at the fetch phase rather than aborting early. assert.Empty(t, RepoConstraintBaseURL(t.Context())) - // The env var supplies the repo and is turned into a raw main-branch URL. + // The env var supplies the repo and is turned into a raw main-branch URL + // anchored at the python/ subtree where the Python artifacts live. ctx := env.Set(t.Context(), EnvConstraintRepo, "databricks/environments") - assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main", RepoConstraintBaseURL(ctx)) + assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", RepoConstraintBaseURL(ctx)) // Whitespace-only is treated as unset. ctx = env.Set(t.Context(), EnvConstraintRepo, " ") @@ -125,7 +126,7 @@ func TestFetchConstraintsCreatesCacheDir(t *testing.T) { } func TestFetchConstraintsSkipsCacheWriteWhenDisabled(t *testing.T) { - // With writeCache=false (the --check dry-run path), a successful live fetch + // With writeCache=false (the --dry-run dry-run path), a successful live fetch // must not write anything to cacheDir. cacheDir := t.TempDir() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -138,7 +139,7 @@ func TestFetchConstraintsSkipsCacheWriteWhenDisabled(t *testing.T) { assert.False(t, c.FromCache) entries, err := os.ReadDir(cacheDir) require.NoError(t, err) - assert.Empty(t, entries, "no cache file should be written under --check") + assert.Empty(t, entries, "no cache file should be written under --dry-run") } func TestCacheFileNameInjective(t *testing.T) { @@ -220,3 +221,42 @@ func TestFetchConstraintsFallsBackToCache(t *testing.T) { require.NoError(t, err) assert.True(t, c.FromCache) } + +func TestParseConstraintsRejectsUnusableRequiresPython(t *testing.T) { + // requires-python present but with no installable floor is not a usable + // artifact; it must be rejected before caching (not only later in the + // pipeline) so it cannot poison the cache. + for _, rp := range []string{">=3", "<3.13", "!=3.12", "*", ">3"} { + toml := "[project]\nrequires-python = \"" + rp + "\"\n" + _, _, _, err := parseConstraints([]byte(toml)) + require.Error(t, err, "requires-python %q should be rejected", rp) + assert.Contains(t, err.Error(), "unusable") + } +} + +func TestFetchConstraintsUnusableBodyDoesNotPoisonCache(t *testing.T) { + // A good artifact populates the cache; a later 2xx body that is valid TOML but + // carries an unusable requires-python must NOT overwrite the last-good copy, + // so an offline run still recovers via the cache. This is the guard's purpose. + cacheDir := t.TempDir() + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleToml)) + })) + defer good.Close() + _, err := FetchConstraints(t.Context(), good.URL, "serverless/serverless-v4", cacheDir, true) + require.NoError(t, err) + + // The repo now publishes a TOML-valid but unusable body (no installable floor). + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("[project]\nrequires-python = \">=3\"\n")) + })) + defer bad.Close() + _, err = FetchConstraints(t.Context(), bad.URL, "serverless/serverless-v4", cacheDir, true) + require.Error(t, err) + + // The cache still holds the last-good copy: an offline fetch recovers it. + c, err := FetchConstraints(t.Context(), "http://127.0.0.1:0", "serverless/serverless-v4", cacheDir, true) + require.NoError(t, err) + assert.True(t, c.FromCache) + assert.Equal(t, "==3.12.*", c.RequiresPython) +} diff --git a/libs/localenv/envkey.go b/libs/localenv/envkey.go index 070668ba826..4e947b0c716 100644 --- a/libs/localenv/envkey.go +++ b/libs/localenv/envkey.go @@ -14,8 +14,30 @@ import ( // by 3.10.6, so only the former bumps the minor. var clauseRe = regexp.MustCompile(`^(>=|<=|===|==|~=|!=|<|>)?\s*(\d+)\.(\d+)(\.\d+)?`) +// defaultServerlessVersion is the serverless environment version used when the +// source (a serverless job with no recorded version, or a bundle that only +// records "serverless") does not pin one. It is a documented stand-in for the +// latest LTS (spec §4.3 / §target-resolution); VS Code resolves the real version +// itself and passes --serverless-version explicitly, so this fallback only +// applies when the version is genuinely unknown. +const defaultServerlessVersion = "5" + +// serverlessVersionRe matches an accepted --serverless-version input: a bare +// number ("5", the documented form) or a "v"-prefixed one ("v5"/"V5"). +var serverlessVersionRe = regexp.MustCompile(`^[vV]?[0-9]+$`) + +// ValidServerlessVersion reports whether s is an accepted serverless version +// input. It is validated at resolve time so a malformed value (e.g. "vv5", "v", +// " 5") fails fast with an actionable error rather than resolving to a bogus +// env key that only 404s two phases later at fetch. +func ValidServerlessVersion(s string) bool { + return serverlessVersionRe.MatchString(s) +} + // NormalizeServerless returns the canonical "vN" spelling of a serverless -// version accepting "4", "v4", or "V4". +// version. The documented input is a bare number ("5"), but a "v"-prefixed form +// ("v5"/"V5") is also accepted; both map to the "serverless-vN" env key. +// Callers should validate with ValidServerlessVersion first. func NormalizeServerless(version string) string { return "v" + strings.TrimPrefix(strings.ToLower(version), "v") } diff --git a/libs/localenv/envkey_test.go b/libs/localenv/envkey_test.go index 898e93434b6..c502cac57b4 100644 --- a/libs/localenv/envkey_test.go +++ b/libs/localenv/envkey_test.go @@ -8,9 +8,30 @@ import ( ) func TestEnvKeyForServerless(t *testing.T) { + // The documented input is a bare number; a "v"-prefixed form is also accepted. + // Both map to the "serverless-vN" env key. for _, in := range []string{"4", "v4", "V4"} { assert.Equal(t, "serverless/serverless-v4", EnvKeyForServerless(in)) } + for _, in := range []string{"5", "v5", "V5"} { + assert.Equal(t, "serverless/serverless-v5", EnvKeyForServerless(in)) + } +} + +func TestDefaultServerlessVersionIsBareNumber(t *testing.T) { + // The default stand-in is stored in the documented bare form; it still maps + // to the serverless-vN env key via NormalizeServerless. + assert.Equal(t, "5", defaultServerlessVersion) + assert.Equal(t, "serverless/serverless-v5", EnvKeyForServerless(defaultServerlessVersion)) +} + +func TestValidServerlessVersion(t *testing.T) { + for _, ok := range []string{"5", "4", "v5", "V5", "17", "0"} { + assert.True(t, ValidServerlessVersion(ok), "%q should be valid", ok) + } + for _, bad := range []string{"", "v", "vv5", "5x", "latest", " 5", "5 ", "v5.1", "-5"} { + assert.False(t, ValidServerlessVersion(bad), "%q should be invalid", bad) + } } func TestEnvKeyForSparkVersion(t *testing.T) { diff --git a/libs/localenv/merge.go b/libs/localenv/merge.go index d0bc39ad1da..f8a8a88e45e 100644 --- a/libs/localenv/merge.go +++ b/libs/localenv/merge.go @@ -15,12 +15,13 @@ var ( errNoProjectTable = errors.New("pyproject.toml has no [project] table to hold requires-python") ) -// managedMarkerStart and managedMarkerEnd bracket the region of pyproject.toml that -// "databricks local-env python sync" owns. Everything between them is rewritten on each merge; -// everything outside is preserved byte-for-byte. -const ( - managedMarkerStart = "# managed by databricks local-env python sync — do not edit" - managedMarkerEnd = "# end managed by databricks local-env python sync" +// managedMarkerStart and managedMarkerEnd bracket the region of pyproject.toml +// that this command owns. Everything between them is rewritten on each merge; +// everything outside is preserved byte-for-byte. They derive from CommandName so +// a command rename stays a single-place change (spec §0 / invariant 8). +var ( + managedMarkerStart = "# managed by databricks " + CommandName + " — do not edit" + managedMarkerEnd = "# end managed by databricks " + CommandName ) // Region names reported back to the caller via MergeManaged's regions return value. diff --git a/libs/localenv/pipeline.go b/libs/localenv/pipeline.go index db8496f85fa..fb4b29a9381 100644 --- a/libs/localenv/pipeline.go +++ b/libs/localenv/pipeline.go @@ -1,6 +1,7 @@ package localenv import ( + "bytes" "context" "errors" "fmt" @@ -61,7 +62,7 @@ type Pipeline struct { // Result always carries the full canonical phase list: phases completed before a // failure are "ok", the failing phase is "error", and the rest are "pending". func (p *Pipeline) Run(ctx context.Context) (*Result, error) { - log.Debugf(ctx, "local-env: mode=%s check=%v project=%s cacheDir=%s constraintBaseURL=%s flags=%+v", + log.Debugf(ctx, CommandName+": mode=%s check=%v project=%s cacheDir=%s constraintBaseURL=%s flags=%+v", p.Mode, p.Check, filepath.ToSlash(p.ProjectDir), @@ -90,17 +91,26 @@ func (p *Pipeline) Run(ctx context.Context) (*Result, error) { // run drives the phases and returns the first phase error. Result bookkeeping // (phase status, error object) is handled by fail / markOK. func (p *Pipeline) run(ctx context.Context) error { - // Phase: preflight — manager detection, writability, package-manager availability. + // Phase: preflight — flag validation, manager detection, writability, + // package-manager availability. + // + // Incompatible target flags are a usage error (E_USAGE), reported at preflight + // before any other work so the failure flows through the phase/JSON reporting + // (a plain Cobra mutual-exclusion error would print no command JSON object, + // which the --output json consumer needs). + if err := ValidateTargetFlags(p.Flags); err != nil { + return p.fail(PhasePreflight, false, NewError(ErrUsage, err, "invalid compute target flags")) + } // P0 supports only uv; any other detected manager is a clean, non-blaming exit. if m := detectManager(p.ProjectDir); m != managerUv { return p.fail(PhasePreflight, false, NewError(ErrManagerUnsupported, nil, "%s", managerGuidance(m))) } - // Under --check the pipeline only reads and reports a plan, so it must not + // Under --dry-run the pipeline only reads and reports a plan, so it must not // mutate anything at preflight. Two preflight steps can write: // - ensureWritable creates and removes a temp file (and would fail a // read-only project the user only wants to inspect); // - PackageManager.EnsureAvailable may install the manager (uv) if missing. - // Both exist to fail fast before real writes, which --check never performs, so + // Both exist to fail fast before real writes, which --dry-run never performs, so // they are skipped in a dry run. Neither result is needed to compute the plan. if p.Check { p.markOK(PhasePreflight, "check") @@ -191,7 +201,7 @@ func (p *Pipeline) resolve(ctx context.Context) (*TargetInfo, error) { } // fetch fetches constraints for the resolved target and records the fetch phase. -// Under --check the cache is not populated, so a dry run performs no disk writes +// Under --dry-run the cache is not populated, so a dry run performs no disk writes // (an existing cache is still read for offline fallback). func (p *Pipeline) fetch(ctx context.Context, target *TargetInfo) (*Constraints, error) { c, err := FetchConstraints(ctx, p.ConstraintBaseURL, target.EnvKey, p.CacheDir, !p.Check) @@ -218,7 +228,7 @@ func (p *Pipeline) backupPath() string { // mergePlan computes the merged pyproject.toml bytes (without writing to disk), // decides greenfield vs. existing, and builds the Plan (populated only under -// --check). dbcPin is the databricks-connect pin to inject, or "" in +// --dry-run). dbcPin is the databricks-connect pin to inject, or "" in // constraints-only mode. func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, dbcPin string) (merged []byte, greenfield bool, err error) { pyproject := p.pyprojectPath() @@ -267,7 +277,7 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, } } - // Under --check, build the plan (with a diff) for reporting. A real run does + // Under --dry-run, build the plan (with a diff) for reporting. A real run does // not need the diff. if p.Check { oldStr := "" @@ -329,6 +339,16 @@ func (p *Pipeline) applyMerge(_ context.Context, mergedBytes []byte, greenfield return p.fail(PhaseMerge, false, NewError(ErrMerge, statErr, "cannot stat backup %s", filepath.ToSlash(backup))) } p.res.BackupPath = filepath.ToSlash(backup) + + // Skip the write when the merged output already matches what is on disk. + // On an idempotent re-run mergePlan reproduces the current file byte for + // byte, so rewriting it would only advance the mtime — spuriously + // invalidating file watchers and uv.lock freshness checks — without + // changing content. The backup above is untouched (the existing .bak is + // kept), so this leaves disk exactly as it was. + if current, readErr := os.ReadFile(pyproject); readErr == nil && bytes.Equal(current, mergedBytes) { + return nil + } } if err := os.WriteFile(pyproject, mergedBytes, 0o644); err != nil { @@ -347,7 +367,7 @@ func (p *Pipeline) provision(ctx context.Context, pyMinor string) error { if err := p.PM.EnsurePython(ctx, pyMinor); err != nil { return p.fail(PhaseProvision, true, asPipelineError(err, ErrPythonInstall, "ensure python %s failed", pyMinor)) } - if err := p.PM.Provision(ctx, p.ProjectDir); err != nil { + if err := p.PM.Provision(ctx, p.ProjectDir, pyMinor); err != nil { return p.fail(PhaseProvision, true, asPipelineError(err, ErrProvision, "provision failed")) } if err := p.PM.PostProvision(ctx, p.ProjectDir); err != nil { @@ -402,7 +422,11 @@ func (p *Pipeline) validate(ctx context.Context, expectedPyMinor, dbcPin string) } p.markOK(PhaseValidate, detail) - p.res.VenvPath = filepath.ToSlash(filepath.Join(p.ProjectDir, venvDir)) + // venvPath is reported relative to the project root (spec §6.1), not as an + // absolute path: the value names the ".venv" the command provisions inside + // ProjectDir, and the VS Code consumer already knows the project root (it + // sets the working directory when it shells out). venvDir is already ".venv". + p.res.VenvPath = venvDir if p.res.Resolved != nil { if defaultMode { p.res.Resolved.DBConnectVersion = dbcVer diff --git a/libs/localenv/pipeline_test.go b/libs/localenv/pipeline_test.go index 1823bcdcc90..da2d0a50436 100644 --- a/libs/localenv/pipeline_test.go +++ b/libs/localenv/pipeline_test.go @@ -19,36 +19,36 @@ type fakePM struct{ py, dbc string } func (fakePM) Name() string { return "fake" } func (fakePM) EnsureAvailable(context.Context) (string, error) { return "fake 1.0", nil } func (fakePM) EnsurePython(context.Context, string) error { return nil } -func (fakePM) Provision(context.Context, string) error { return nil } +func (fakePM) Provision(context.Context, string, string) error { return nil } func (fakePM) PostProvision(context.Context, string) error { return nil } func (f fakePM) Validate(context.Context, string) (string, string, error) { return f.py, f.dbc, nil } // noProvisionPM fails any method that could touch the machine (install the -// manager, install Python, sync, seed pip, validate). It asserts that --check +// manager, install Python, sync, seed pip, validate). It asserts that --dry-run // never reaches those write-side operations. type noProvisionPM struct{} func (noProvisionPM) Name() string { return "noprov" } func (noProvisionPM) EnsureAvailable(context.Context) (string, error) { - return "", errors.New("EnsureAvailable must not be called under --check") + return "", errors.New("EnsureAvailable must not be called under --dry-run") } func (noProvisionPM) EnsurePython(context.Context, string) error { - return errors.New("EnsurePython must not be called under --check") + return errors.New("EnsurePython must not be called under --dry-run") } -func (noProvisionPM) Provision(context.Context, string) error { - return errors.New("Provision must not be called under --check") +func (noProvisionPM) Provision(context.Context, string, string) error { + return errors.New("Provision must not be called under --dry-run") } func (noProvisionPM) PostProvision(context.Context, string) error { - return errors.New("PostProvision must not be called under --check") + return errors.New("PostProvision must not be called under --dry-run") } func (noProvisionPM) Validate(context.Context, string) (string, string, error) { - return "", "", errors.New("Validate must not be called under --check") + return "", "", errors.New("Validate must not be called under --dry-run") } // uvMissingPM fails EnsureAvailable, simulating a machine where the package @@ -78,6 +78,25 @@ func newTestServer(t *testing.T) *httptest.Server { })) } +func TestPipelineRejectsConflictingTargetFlagsAtPreflight(t *testing.T) { + // Incompatible target flags are a usage error surfaced as E_USAGE at + // preflight, before any manager/writability/fetch work. + dir := writeProject(t) + p := &Pipeline{ + Mode: ModeDefault, Check: true, ProjectDir: dir, CacheDir: t.TempDir(), + Flags: TargetFlags{Cluster: "abc", Serverless: "v4"}, + Compute: stubCompute{}, PM: fakePM{py: "3.12", dbc: "17.2.0"}, + } + res, err := p.Run(t.Context()) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrUsage, pe.Code) + assert.Equal(t, PhasePreflight, pe.FailurePhase) + assert.False(t, pe.DiskMutated) + require.NotNil(t, res.Error) + assert.Equal(t, ErrUsage, res.Error.Code) +} + func TestPipelineCheckMutatesNothing(t *testing.T) { dir := writeProject(t) before, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) @@ -100,7 +119,7 @@ func TestPipelineCheckMutatesNothing(t *testing.T) { after, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) assert.Equal(t, string(before), string(after)) // unchanged - // --check must not populate the constraint cache either (no disk writes). + // --dry-run must not populate the constraint cache either (no disk writes). entries, err := os.ReadDir(cacheDir) require.NoError(t, err) assert.Empty(t, entries) @@ -108,7 +127,7 @@ func TestPipelineCheckMutatesNothing(t *testing.T) { func TestPipelineCheckReRunPlanMatchesRealRun(t *testing.T) { // On a re-run where the .bak already exists and the live file already equals - // the merged output, --check must report a plan a real run would perform: no + // the merged output, --dry-run must report a plan a real run would perform: no // backup (the .bak is kept, not rewritten) and an empty diff (nothing changes). dir := writeProject(t) srv := newTestServer(t) @@ -128,16 +147,16 @@ func TestPipelineCheckReRunPlanMatchesRealRun(t *testing.T) { require.NoError(t, err) require.FileExists(t, filepath.Join(dir, "pyproject.toml.bak")) - // Now --check on the already-synced project. + // Now --dry-run on the already-synced project. res, err := newPipe(true).Run(t.Context()) require.NoError(t, err) require.NotNil(t, res.Plan) - assert.Empty(t, res.Plan.WouldBackup, "a re-run keeps the existing .bak, so --check must not claim a backup") + assert.Empty(t, res.Plan.WouldBackup, "a re-run keeps the existing .bak, so --dry-run must not claim a backup") assert.Empty(t, res.Plan.Diff, "the live file already equals the merged output; the diff must be empty") } func TestPipelineCheckDoesNotProvision(t *testing.T) { - // --check must not call any PackageManager method that could mutate the + // --dry-run must not call any PackageManager method that could mutate the // machine (EnsureAvailable may install uv). noProvisionPM errors on all of // them; the dry run must still succeed and produce a plan. dir := writeProject(t) @@ -164,7 +183,7 @@ func TestPipelineCheckWorksOnReadOnlyDir(t *testing.T) { srv := newTestServer(t) defer srv.Close() - // Make the project dir read-only: --check must still compute the plan without + // Make the project dir read-only: --dry-run must still compute the plan without // a writability probe (which would both mutate disk and fail here). require.NoError(t, os.Chmod(dir, 0o555)) t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) @@ -205,7 +224,10 @@ func TestPipelineProvisionsAndValidatesExisting(t *testing.T) { require.NotNil(t, res.Resolved) assert.Equal(t, "3.12", res.Resolved.PythonVersion) assert.Equal(t, "17.2.0", res.Resolved.DBConnectVersion) - assert.Equal(t, ".venv", filepath.Base(res.VenvPath)) + // venvPath is reported relative to the project root (spec §6.1), so it is + // exactly ".venv" — not an absolute path under the temp ProjectDir. Asserting + // the full value (not just filepath.Base) is what pins the relative contract. + assert.Equal(t, ".venv", res.VenvPath) merged, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) assert.Contains(t, string(merged), `"databricks-connect~=17.2.0"`) assert.FileExists(t, filepath.Join(dir, "pyproject.toml.bak")) @@ -284,6 +306,44 @@ func TestPipelineExistingBacksUp(t *testing.T) { assert.Equal(t, "pyproject.toml.bak", filepath.Base(res.BackupPath)) } +func TestPipelineReRunDoesNotRewriteUnchangedPyproject(t *testing.T) { + // An idempotent re-run reproduces the merged pyproject.toml byte for byte, so + // applyMerge must skip the write rather than advance the file's mtime (which + // would spuriously invalidate file watchers and uv.lock freshness checks). + dir := writeProject(t) + srv := newTestServer(t) + defer srv.Close() + + newPipe := func() *Pipeline { + return &Pipeline{ + Mode: ModeDefault, ProjectDir: dir, + ConstraintBaseURL: srv.URL, CacheDir: t.TempDir(), + Flags: TargetFlags{Serverless: "v4"}, + Compute: stubCompute{}, PM: fakePM{py: "3.12", dbc: "17.2.0"}, + } + } + + pyproject := filepath.Join(dir, "pyproject.toml") + _, err := newPipe().Run(t.Context()) + require.NoError(t, err) + firstContent, err := os.ReadFile(pyproject) + require.NoError(t, err) + firstInfo, err := os.Stat(pyproject) + require.NoError(t, err) + + // A second run computes the same merged output; the file must be left as-is, + // content and mtime both unchanged. + _, err = newPipe().Run(t.Context()) + require.NoError(t, err) + secondContent, err := os.ReadFile(pyproject) + require.NoError(t, err) + secondInfo, err := os.Stat(pyproject) + require.NoError(t, err) + + assert.Equal(t, string(firstContent), string(secondContent), "content must be unchanged") + assert.Equal(t, firstInfo.ModTime(), secondInfo.ModTime(), "unchanged pyproject.toml must not be rewritten") +} + func TestCopyFilePreservesMode(t *testing.T) { if runtime.GOOS == "windows" { // Windows does not honor Unix permission bits. diff --git a/libs/localenv/pkgmanager.go b/libs/localenv/pkgmanager.go index fff370c76fc..2dd407af37c 100644 --- a/libs/localenv/pkgmanager.go +++ b/libs/localenv/pkgmanager.go @@ -15,8 +15,10 @@ type PackageManager interface { // available via the package manager. EnsurePython(ctx context.Context, minor string) error - // Provision installs the project dependencies inside projectDir. - Provision(ctx context.Context, projectDir string) error + // Provision installs the project dependencies inside projectDir, pinning the + // environment to the given Python minor (e.g. "3.12") so it matches the target + // rather than whatever newer interpreter the manager might otherwise pick. + Provision(ctx context.Context, projectDir, pyMinor string) error // PostProvision seeds pip into the virtual environment inside projectDir. // This step is required because VS Code's ms-python.vscode-python-envs diff --git a/libs/localenv/provision_integration_test.go b/libs/localenv/provision_integration_test.go new file mode 100644 index 00000000000..e0a270aec07 --- /dev/null +++ b/libs/localenv/provision_integration_test.go @@ -0,0 +1,159 @@ +package localenv + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/process" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// EnvTestProvision opts a machine into the real provision/validate integration +// tests below. They are skipped by default because they run a real `uv sync` +// (installing an interpreter and databricks-connect from a package index), which +// needs uv on PATH and network access to a PyPI mirror — neither is guaranteed in +// unit-test CI. The rest of the pipeline is covered hermetically by the unit and +// acceptance suites (all of which use --dry-run or a fake PackageManager); this +// file is the one place the real merge -> provision -> validate path is exercised. +const EnvTestProvision = "DATABRICKS_LOCALENV_TEST_PROVISION" + +// requireRealProvision skips unless the machine opted in and uv is discoverable. +func requireRealProvision(t *testing.T) { + t.Helper() + if env.Get(t.Context(), EnvTestProvision) == "" { + t.Skipf("%s is not set; skipping real uv provision integration test", EnvTestProvision) + } + if _, err := discoverUv(t.Context()); err != nil { + t.Skipf("uv not available (%v); skipping real uv provision integration test", err) + } +} + +// realProvisionCtx allows uv to install itself and bridges the user's pip index +// so `uv sync` can reach databricks-connect on networks where pypi.org is blocked +// but a corporate mirror is declared in pip.conf. +func realProvisionCtx(t *testing.T) context.Context { + ctx := env.Set(t.Context(), EnvAutoInstallUv, "1") + if idx := pipConfIndexURL(ctx); idx != "" { + ctx = env.Set(ctx, "UV_INDEX_URL", idx) + } + return ctx +} + +// serverlessConstraintServer serves a per-env pyproject.toml for the serverless +// target, mirroring what databricks/environments will publish. +func serverlessConstraintServer(t *testing.T) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[project] +requires-python = ">=3.12" + +[dependency-groups] +dev = ["databricks-connect~=17.2.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +`)) + })) + t.Cleanup(srv.Close) + return srv +} + +// TestProvisionCreatesAndValidatesVenv drives the full non-dry-run pipeline +// (merge -> provision -> validate) against a stubbed constraint fetch and asserts +// a real .venv is created and validated with the target's versions. This is the +// only test that exercises a real uv provision; everything else is dry-run. +func TestProvisionCreatesAndValidatesVenv(t *testing.T) { + requireRealProvision(t) + dir := t.TempDir() // greenfield: no pre-existing pyproject.toml + srv := serverlessConstraintServer(t) + + p := &Pipeline{ + Mode: ModeDefault, Check: false, ProjectDir: dir, + ConstraintBaseURL: srv.URL, CacheDir: t.TempDir(), + Flags: TargetFlags{Serverless: "5"}, + Compute: stubCompute{}, PM: NewUvManager(), + } + res, err := p.Run(realProvisionCtx(t)) + require.NoError(t, err) + require.NotNil(t, res) + assert.True(t, res.OK) + assert.False(t, res.DryRun) + + // Every phase reached ok — including the real provision and validate. + for _, ph := range res.Phases { + assert.Equalf(t, StatusOK, ph.Status, "phase %s not ok", ph.Phase) + } + + // A real interpreter exists in the created venv. + assert.FileExists(t, venvPython(dir)) + + // pyproject.toml + uv.lock were written; the env-owned pin was applied. + pyproject, err := os.ReadFile(filepath.Join(dir, "pyproject.toml")) + require.NoError(t, err) + assert.Contains(t, string(pyproject), "databricks-connect") + assert.FileExists(t, filepath.Join(dir, "uv.lock")) + + // Validate observed the real installed environment. + require.NotNil(t, res.Resolved) + assert.Equal(t, "3.12", res.Resolved.PythonVersion) + assert.Truef(t, hasMajorPrefix(res.Resolved.DBConnectVersion, "17"), + "expected databricks-connect major 17, got %q", res.Resolved.DBConnectVersion) +} + +// TestProvisionValidateIgnoresActiveVirtualEnv is the regression guard for the +// validate fix: an active VIRTUAL_ENV pointing at a *real* interpreter of a +// different version must not make validation read that interpreter instead of +// the provisioned .venv. The bug only reproduces with a real, differently- +// versioned active env — the pre-fix `uv run --no-project` reads it (verified: +// reports 3.13), while invoking the .venv interpreter directly reads 3.12. A +// bogus/non-existent VIRTUAL_ENV does NOT reproduce it (uv falls back), so this +// test builds a real second venv on a different Python. +func TestProvisionValidateIgnoresActiveVirtualEnv(t *testing.T) { + requireRealProvision(t) + baseCtx := realProvisionCtx(t) + + // Build a real "active" venv on a different Python minor (3.13) than the + // target (3.12). Skip if that interpreter can't be provisioned here. + activeEnv := filepath.Join(t.TempDir(), "active-venv") + if _, err := process.Background(baseCtx, + []string{mustUv(t, baseCtx), "venv", "--python", "3.13", activeEnv}); err != nil { + t.Skipf("could not create a 3.13 active venv for the negative control: %v", err) + } + + dir := t.TempDir() + srv := serverlessConstraintServer(t) + // Activate the 3.13 env. Pre-fix, validation would report 3.13; post-fix it + // reads the provisioned 3.12 .venv directly and ignores this. + ctx := env.Set(baseCtx, "VIRTUAL_ENV", activeEnv) + + p := &Pipeline{ + Mode: ModeDefault, Check: false, ProjectDir: dir, + ConstraintBaseURL: srv.URL, CacheDir: t.TempDir(), + Flags: TargetFlags{Serverless: "5"}, + Compute: stubCompute{}, PM: NewUvManager(), + } + res, err := p.Run(ctx) + require.NoError(t, err) + require.NotNil(t, res.Resolved) + assert.Equal(t, "3.12", res.Resolved.PythonVersion, + "validation must read the provisioned .venv (3.12), not the active VIRTUAL_ENV (3.13)") +} + +// mustUv returns the discovered uv binary path, failing the test if uv is not +// available (requireRealProvision has already gated on this). +func mustUv(t *testing.T, ctx context.Context) string { + t.Helper() + bin, err := discoverUv(ctx) + require.NoError(t, err) + return bin +} + +// hasMajorPrefix reports whether version starts with major + ".". +func hasMajorPrefix(version, major string) bool { + return len(version) > len(major) && version[:len(major)+1] == major+"." +} diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 1de8f66ba50..53e50a6b919 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -1,16 +1,23 @@ package localenv -import "fmt" +import ( + "encoding/json" + "fmt" +) // Command path components, defined once so a rename touches a single place -// (spec §0 / invariant 8 / scenario 21). The cmd layer builds the Cobra -// command tree from CommandGroup/CommandSubgroup/CommandVerb; the --json -// "command" field uses CommandName. No other string re-spells the command path. +// (spec §0 / invariant 8 / scenario 21). The verb is a subcommand of the +// generated "environments" group; the --json "command" field uses CommandName. +// No other string re-spells the command path. +// +// P0 is Python-only and takes no language selector: the verb is bare +// "setup-local" (spec §naming). A language axis (setup-local python / scala) is +// the preferred shape only if more languages are ever added, and nothing is +// reserved for it here. const ( - CommandGroup = "local-env" - CommandSubgroup = "python" - CommandVerb = "sync" - CommandName = CommandGroup + " " + CommandSubgroup + " " + CommandVerb + CommandGroup = "environments" + CommandVerb = "setup-local" + CommandName = CommandGroup + " " + CommandVerb // SchemaVersion is the version of the --json output contract (spec §6). // Bump it on any breaking change to the JSON shape. @@ -57,27 +64,36 @@ const ( // ErrorCode is a stable failure-class identifier surfaced in --json error.code // (spec §7). Values are compared via the ErrorCode constants, never by // string-matching messages, and are defined once here. +// +// This set is the source of truth for the spec's error-code table; each code is +// annotated with the phase that emits it. Two codes from the spec are +// intentionally not defined because the CLI never emits them: E_PYTHON_POLICY +// (a policy-gated failure whose signal source does not yet exist) and E_AUTH +// (the shared workspace-client preflight, MustWorkspaceClient, surfaces the +// standard CLI auth error before a command JSON object is ever built). type ErrorCode string const ( - ErrNoTarget ErrorCode = "E_NO_TARGET" - ErrManagerUnsupported ErrorCode = "E_MANAGER_UNSUPPORTED" - ErrUvMissing ErrorCode = "E_UV_MISSING" - ErrNotWritable ErrorCode = "E_NOT_WRITABLE" - ErrResolve ErrorCode = "E_RESOLVE" - ErrEnvUnsupported ErrorCode = "E_ENV_UNSUPPORTED" - ErrFetch ErrorCode = "E_FETCH" - ErrWrite ErrorCode = "E_WRITE" - ErrMerge ErrorCode = "E_MERGE" - ErrPythonInstall ErrorCode = "E_PYTHON_INSTALL" - ErrProvision ErrorCode = "E_PROVISION" - ErrValidate ErrorCode = "E_VALIDATE" + ErrUsage ErrorCode = "E_USAGE" // preflight: incompatible flags; resolve: --job-task names a job but no task + ErrManagerUnsupported ErrorCode = "E_MANAGER_UNSUPPORTED" // preflight: manager is not uv + ErrNotWritable ErrorCode = "E_NOT_WRITABLE" // preflight: project dir not writable + ErrUvMissing ErrorCode = "E_UV_MISSING" // preflight: uv not found / install failed + ErrNoTarget ErrorCode = "E_NO_TARGET" // resolve: no target from any source + ErrResolve ErrorCode = "E_RESOLVE" // resolve: target read failed / ambiguous name + ErrEnvUnsupported ErrorCode = "E_ENV_UNSUPPORTED" // fetch: no published env key + ErrFetch ErrorCode = "E_FETCH" // fetch: repo unreachable, no usable cache + ErrWrite ErrorCode = "E_WRITE" // merge: greenfield write failed + ErrMerge ErrorCode = "E_MERGE" // merge: existing-project merge failed + ErrPythonInstall ErrorCode = "E_PYTHON_INSTALL" // provision: uv python install failed + ErrProvision ErrorCode = "E_PROVISION" // provision: uv sync failed + ErrValidate ErrorCode = "E_VALIDATE" // validate: post-provision version mismatch ) // PipelineError is a failure carrying a stable code, the phase at which it // occurred, and whether disk was mutated before the failure. It marshals to the // --json error object (spec §6.2). Code and FailurePhase are the stable -// contract; Err holds the wrapped cause for errors.Is/As and is not serialized. +// contract; Err holds the wrapped cause for errors.Is/As and is not serialized +// directly (its text is folded into the "message" field — see MarshalJSON). type PipelineError struct { Code ErrorCode `json:"code"` FailurePhase PhaseName `json:"failurePhase"` @@ -97,6 +113,21 @@ func (e *PipelineError) Unwrap() error { return e.Err } +// MarshalJSON serializes the full message (Error(), i.e. Msg plus any wrapped +// cause) into the "message" field. Without this the --json error object would +// carry only Msg and drop the cause (Err is json:"-"), so a JSON consumer would +// get strictly less detail than the text output — e.g. "resolving cluster name" +// without the "ambiguous"/"not found" reason. Text and JSON must agree. +func (e *PipelineError) MarshalJSON() ([]byte, error) { + type alias PipelineError // avoid recursing into this method + return json.Marshal((*alias)(&PipelineError{ + Code: e.Code, + FailurePhase: e.FailurePhase, + Msg: e.Error(), + DiskMutated: e.DiskMutated, + })) +} + // NewError creates a PipelineError with a code and message. FailurePhase and // DiskMutated are filled in by the pipeline when it records the failure. The // message is formatted with fmt.Sprintf(format, args...); err may be nil. @@ -109,9 +140,10 @@ func NewError(code ErrorCode, err error, format string, args ...any) *PipelineEr } // TargetInfo is the resolved compute target (spec §6 "target"). Source records -// which of the four precedence sources was used. SparkVersion is the raw cluster -// runtime string the resolver read; it is folded into EnvKey (dbr/) -// and is not part of the JSON contract, kept only as intermediate resolver state. +// which precedence source was used ("cluster", "serverless", "job", or +// "bundle"). SparkVersion is the raw cluster runtime string the resolver read; +// it is folded into EnvKey (dbr/) and is not part of the JSON +// contract, kept only as intermediate resolver state. type TargetInfo struct { Source string `json:"source"` ClusterID string `json:"clusterId,omitempty"` @@ -129,7 +161,7 @@ type ResolvedInfo struct { ArtifactSource string `json:"artifactSource"` } -// Plan describes the changes a --check run would apply (spec §6.3). +// Plan describes the changes a --dry-run run would apply (spec §6.3). // ChangedRegions is retained for text output only and is not serialized. type Plan struct { WouldWrite string `json:"wouldWrite"` diff --git a/libs/localenv/result_test.go b/libs/localenv/result_test.go index 75fe60ea66d..f8f2d60c618 100644 --- a/libs/localenv/result_test.go +++ b/libs/localenv/result_test.go @@ -24,8 +24,8 @@ func TestModeString(t *testing.T) { func TestCommandName(t *testing.T) { // The --json "command" field and all help text derive from these; the - // three-part path must join to the full command a user types. - assert.Equal(t, "local-env python sync", CommandName) + // path must join to the full command a user types. + assert.Equal(t, "environments setup-local", CommandName) } func TestNewResultEmitsEmptyArraysNotNull(t *testing.T) { diff --git a/libs/localenv/target.go b/libs/localenv/target.go index cba5666d3ff..b73575cc1fa 100644 --- a/libs/localenv/target.go +++ b/libs/localenv/target.go @@ -2,6 +2,7 @@ package localenv import ( "context" + "errors" "fmt" "strings" ) @@ -10,16 +11,48 @@ import ( type ComputeClient interface { // GetClusterSparkVersion returns the Spark version string for a cluster. GetClusterSparkVersion(ctx context.Context, clusterID string) (string, error) - // GetJobSparkVersion returns either a Spark version (isServerless=false) or a - // serverless marker (isServerless=true) for a job, plus a recorded version string. - GetJobSparkVersion(ctx context.Context, jobID string) (sparkVersion string, isServerless bool, version string, err error) + // GetClusterByName resolves a cluster name to its ID and Spark version. It + // errors when the name is unknown or ambiguous (more than one cluster shares + // the name), so the caller can surface an actionable E_RESOLVE. + GetClusterByName(ctx context.Context, name string) (clusterID, sparkVersion string, err error) + // JobTaskEnvironment resolves a single job task's compute to an environment. + // + // A job can bind multiple tasks to different environment versions, so the + // target is a specific task, not the whole job. taskKey selects it; an empty + // taskKey is a request to enumerate — the method returns ErrTaskKeyRequired + // wrapping the job's task keys so the caller can prompt for one. An unknown + // taskKey returns an error naming the available keys. + // + // For a serverless task, isServerless is true and version is the task's + // recorded serverless environment version (read directly — no fallback). For + // a classic task, isServerless is false and sparkVersion is the task cluster's + // runtime. + JobTaskEnvironment(ctx context.Context, jobID, taskKey string) (sparkVersion string, isServerless bool, version string, err error) +} + +// ErrTaskKeyRequired is returned by JobTaskEnvironment when --job-task names a +// job but no task, so ResolveTarget can classify it as E_USAGE (the user must +// pick a task) rather than E_RESOLVE. It carries the job's task keys so the +// error message can list them. +type ErrTaskKeyRequired struct { + JobID string + TaskKeys []string +} + +func (e *ErrTaskKeyRequired) Error() string { + return fmt.Sprintf("job %s has multiple tasks; specify one: --job-task %s. (available: %s)", + e.JobID, e.JobID, strings.Join(e.TaskKeys, ", ")) } // TargetFlags holds the mutually-exclusive compute target flags from the CLI. type TargetFlags struct { - Cluster string - Serverless string - Job string + Cluster string + ClusterName string + Serverless string + // JobTask is "." (or a bare "", which resolves to + // an E_USAGE listing the job's task keys). A job can bind tasks to different + // environment versions, so the target is a specific task, not the whole job. + JobTask string } // BundleTarget is the three-state result of reading the bundle's configured @@ -32,20 +65,23 @@ type BundleTarget struct { // noTargetMessage is the actionable message shown when no target is selected, // matching spec §2.3. -const noTargetMessage = "No compute target is selected. Select a cluster or serverless target, or pass --cluster / --serverless / --job" +const noTargetMessage = "No compute target is selected. Select a cluster or serverless target, or pass --cluster-id / --cluster-name / --serverless-version / --job-task" -// ValidateTargetFlags returns an error if more than one of the three flags is set. +// ValidateTargetFlags returns an error if more than one of the target flags is set. // Cobra marks them mutually exclusive too; this guards the library path. func ValidateTargetFlags(f TargetFlags) error { var set []string if f.Cluster != "" { - set = append(set, "--cluster") + set = append(set, "--cluster-id") + } + if f.ClusterName != "" { + set = append(set, "--cluster-name") } if f.Serverless != "" { - set = append(set, "--serverless") + set = append(set, "--serverless-version") } - if f.Job != "" { - set = append(set, "--job") + if f.JobTask != "" { + set = append(set, "--job-task") } if len(set) > 1 { return fmt.Errorf("flags %s are mutually exclusive; specify at most one", strings.Join(set, " and ")) @@ -54,7 +90,7 @@ func ValidateTargetFlags(f TargetFlags) error { } // ResolveTarget resolves the compute target using ordered precedence: -// --cluster flag → --serverless flag → --job flag → bundle target. +// --cluster-id → --cluster-name → --serverless-version → --job-task → bundle target. // PythonVersion is left empty; it is filled later from constraint data. // // Incompatible flags are rejected up front: without this a library caller that @@ -79,7 +115,26 @@ func ResolveTarget(ctx context.Context, f TargetFlags, c ComputeClient, bt Bundl }, nil } + if f.ClusterName != "" { + // Resolve the name to an ID via the Clusters API; from there it is + // identical to --cluster-id. An unknown or ambiguous name (two clusters + // sharing it) yields an actionable E_RESOLVE. + id, v, err := c.GetClusterByName(ctx, f.ClusterName) + if err != nil { + return nil, NewError(ErrResolve, err, "resolving cluster name %q", f.ClusterName) + } + return &TargetInfo{ + Source: "cluster", + ClusterID: id, + SparkVersion: v, + EnvKey: EnvKeyForSparkVersion(v), + }, nil + } + if f.Serverless != "" { + if !ValidServerlessVersion(f.Serverless) { + return nil, NewError(ErrResolve, nil, "invalid --serverless-version %q: expected a version number like 5", f.Serverless) + } return &TargetInfo{ Source: "serverless", ServerlessVersion: NormalizeServerless(f.Serverless), @@ -87,27 +142,32 @@ func ResolveTarget(ctx context.Context, f TargetFlags, c ComputeClient, bt Bundl }, nil } - if f.Job != "" { - sparkVersion, isServerless, version, err := c.GetJobSparkVersion(ctx, f.Job) + if f.JobTask != "" { + // Split "." on the FIRST dot: task keys may themselves + // contain dots, so everything after the first separator is the task key. + // A bare "" (no dot) leaves taskKey empty, which JobTaskEnvironment + // treats as an enumerate request and reports back via ErrTaskKeyRequired. + jobID, taskKey, _ := strings.Cut(f.JobTask, ".") + sparkVersion, isServerless, version, err := c.JobTaskEnvironment(ctx, jobID, taskKey) if err != nil { - return nil, NewError(ErrResolve, err, "resolving job %s", f.Job) + // A missing task key is a usage error (the user must pick a task), + // distinct from a genuine resolve failure (unknown key, API error). + if _, ok := errors.AsType[*ErrTaskKeyRequired](err); ok { + return nil, NewError(ErrUsage, err, "specify a job task") + } + return nil, NewError(ErrResolve, err, "resolving job task %s", f.JobTask) } if isServerless { - // Use the job's recorded serverless environment version when present; - // fall back to v4 when the job did not pin one (documented stand-in from - // the original script, spec §4.3). - v := version - if v == "" { - v = "v4" - } + // The task's serverless environment version is read directly from its + // bound environment, so there is no version to guess: unlike the bundle + // path, the job-task path never applies the serverless-vN fallback. return &TargetInfo{ Source: "job", - ServerlessVersion: NormalizeServerless(v), - EnvKey: EnvKeyForServerless(v), + ServerlessVersion: NormalizeServerless(version), + EnvKey: EnvKeyForServerless(version), }, nil } - // Classic compute: the Spark version is the first return per the - // GetJobSparkVersion contract, not the recorded-version third return. + // Classic compute: sparkVersion is the task cluster's runtime. return &TargetInfo{ Source: "job", SparkVersion: sparkVersion, @@ -121,12 +181,12 @@ func ResolveTarget(ctx context.Context, f TargetFlags, c ComputeClient, bt Bundl } if bt.Serverless { - // Default to serverless-v4: the serverless env version is not recorded - // in the bundle/project (documented stand-in from the original script). + // The bundle records that the target is serverless but not which version, + // so use the default stand-in (spec §4.3). return &TargetInfo{ Source: "bundle", - ServerlessVersion: "v4", - EnvKey: EnvKeyForServerless("v4"), + ServerlessVersion: NormalizeServerless(defaultServerlessVersion), + EnvKey: EnvKeyForServerless(defaultServerlessVersion), }, nil } diff --git a/libs/localenv/target_test.go b/libs/localenv/target_test.go index 7e80dce2413..6e0d64e9abd 100644 --- a/libs/localenv/target_test.go +++ b/libs/localenv/target_test.go @@ -12,13 +12,20 @@ import ( type stubCompute struct { clusterVersion string clusterErr error + byNameID string + byNameVersion string + byNameErr error } func (s stubCompute) GetClusterSparkVersion(_ context.Context, _ string) (string, error) { return s.clusterVersion, s.clusterErr } -func (s stubCompute) GetJobSparkVersion(_ context.Context, _ string) (string, bool, string, error) { +func (s stubCompute) GetClusterByName(_ context.Context, _ string) (string, string, error) { + return s.byNameID, s.byNameVersion, s.byNameErr +} + +func (s stubCompute) JobTaskEnvironment(_ context.Context, _, _ string) (string, bool, string, error) { return "", false, "", nil } @@ -30,6 +37,24 @@ func TestResolveServerlessFlag(t *testing.T) { assert.Equal(t, "serverless/serverless-v4", ti.EnvKey) } +func TestResolveServerlessFlagBareNumber(t *testing.T) { + // The documented input is a bare number; it normalizes to the vN env key. + ti, err := ResolveTarget(t.Context(), TargetFlags{Serverless: "5"}, stubCompute{}, BundleTarget{}) + require.NoError(t, err) + assert.Equal(t, "serverless/serverless-v5", ti.EnvKey) +} + +func TestResolveServerlessFlagRejectsMalformed(t *testing.T) { + // Malformed values fail fast at resolve (E_RESOLVE) rather than resolving to + // a bogus env key that only 404s at fetch. + for _, bad := range []string{"vv5", "v", " 5", "5x", "latest"} { + _, err := ResolveTarget(t.Context(), TargetFlags{Serverless: bad}, stubCompute{}, BundleTarget{}) + var pe *PipelineError + require.ErrorAs(t, err, &pe, "input %q should error", bad) + assert.Equal(t, ErrResolve, pe.Code, "input %q", bad) + } +} + func TestResolveClusterFlag(t *testing.T) { c := stubCompute{clusterVersion: "15.4.x-scala2.12"} ti, err := ResolveTarget(t.Context(), TargetFlags{Cluster: "abc"}, c, BundleTarget{}) @@ -48,6 +73,36 @@ func TestResolveClusterFlagError(t *testing.T) { assert.Equal(t, ErrResolve, pe.Code) } +func TestResolveClusterNameFlag(t *testing.T) { + // --cluster-name resolves to an ID via the Clusters API, then behaves like + // --cluster-id: source=cluster, the resolved ID is reported, and the env key + // derives from the resolved cluster's Spark version. + c := stubCompute{byNameID: "cid-123", byNameVersion: "15.4.x-scala2.12"} + ti, err := ResolveTarget(t.Context(), TargetFlags{ClusterName: "my-cluster"}, c, BundleTarget{}) + require.NoError(t, err) + assert.Equal(t, "cluster", ti.Source) + assert.Equal(t, "cid-123", ti.ClusterID) + assert.Equal(t, "15.4.x-scala2.12", ti.SparkVersion) + assert.Equal(t, "dbr/15.4.x-scala2.12", ti.EnvKey) +} + +func TestResolveClusterNameFlagError(t *testing.T) { + // An unknown or ambiguous name surfaces as E_RESOLVE. + c := stubCompute{byNameErr: errors.New("there are 2 instances of ClusterDetails named 'dup'")} + _, err := ResolveTarget(t.Context(), TargetFlags{ClusterName: "dup"}, c, BundleTarget{}) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrResolve, pe.Code) +} + +func TestResolveClusterIdAndNameMutuallyExclusive(t *testing.T) { + // The library path rejects setting both --cluster-id and --cluster-name. + _, err := ResolveTarget(t.Context(), TargetFlags{Cluster: "abc", ClusterName: "xyz"}, stubCompute{}, BundleTarget{}) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrResolve, pe.Code) +} + func TestResolveBundleNothingSelected(t *testing.T) { _, err := ResolveTarget(t.Context(), TargetFlags{}, stubCompute{}, BundleTarget{Selected: false}) var pe *PipelineError @@ -56,59 +111,100 @@ func TestResolveBundleNothingSelected(t *testing.T) { } func TestResolveBundleServerless(t *testing.T) { + // The bundle records serverless but no version, so the default stand-in applies. ti, err := ResolveTarget(t.Context(), TargetFlags{}, stubCompute{}, BundleTarget{Selected: true, Serverless: true}) require.NoError(t, err) assert.Equal(t, "bundle", ti.Source) - assert.Equal(t, "serverless/serverless-v4", ti.EnvKey) + // Concrete literal, not "serverless-"+defaultServerlessVersion: the default + // is v5, and asserting the constant against itself would pass for any value. + assert.Equal(t, "serverless/serverless-v5", ti.EnvKey) } -// jobStubCompute returns distinct values for the first (sparkVersion) and third -// (recorded version) results of GetJobSparkVersion so the classic-compute branch -// can be checked against the documented contract (it must use the first). +// jobStubCompute stubs JobTaskEnvironment. It records the jobID and taskKey it +// was called with (so tests can assert the "." split) and +// returns configurable results, including an error for the bare-job / unknown-key +// paths. The sparkVersion/version fields are distinct so the classic branch can +// be checked against the contract (it must use the Spark-version return). type jobStubCompute struct { sparkVersion string isServerless bool version string + err error + + gotJobID string + gotTaskKey string } -func (jobStubCompute) GetClusterSparkVersion(_ context.Context, _ string) (string, error) { +func (*jobStubCompute) GetClusterSparkVersion(_ context.Context, _ string) (string, error) { return "", nil } -func (s jobStubCompute) GetJobSparkVersion(_ context.Context, _ string) (string, bool, string, error) { +func (*jobStubCompute) GetClusterByName(_ context.Context, _ string) (string, string, error) { + return "", "", nil +} + +func (s *jobStubCompute) JobTaskEnvironment(_ context.Context, jobID, taskKey string) (string, bool, string, error) { + s.gotJobID = jobID + s.gotTaskKey = taskKey + if s.err != nil { + return "", false, "", s.err + } return s.sparkVersion, s.isServerless, s.version, nil } -func TestResolveJobClassicUsesSparkVersionReturn(t *testing.T) { - // Contract: for a classic-compute job the Spark version is the FIRST return. - // The third "recorded version" return differs here to catch use of the wrong one. - c := jobStubCompute{sparkVersion: "15.4.x-scala2.12", isServerless: false, version: "wrong-recorded"} - ti, err := ResolveTarget(t.Context(), TargetFlags{Job: "42"}, c, BundleTarget{}) +func TestResolveJobTaskClassicUsesSparkVersion(t *testing.T) { + // A classic task resolves to its cluster's Spark version → dbr/ env key. + c := &jobStubCompute{sparkVersion: "15.4.x-scala2.12"} + ti, err := ResolveTarget(t.Context(), TargetFlags{JobTask: "42.ingest"}, c, BundleTarget{}) require.NoError(t, err) assert.Equal(t, "job", ti.Source) assert.Equal(t, "15.4.x-scala2.12", ti.SparkVersion) assert.Equal(t, "dbr/15.4.x-scala2.12", ti.EnvKey) + // The value is split on the first dot into job ID and task key. + assert.Equal(t, "42", c.gotJobID) + assert.Equal(t, "ingest", c.gotTaskKey) +} + +func TestResolveJobTaskSplitsOnFirstDot(t *testing.T) { + // Task keys may themselves contain dots, so only the first dot separates the + // job ID from the task key. + c := &jobStubCompute{sparkVersion: "15.4.x-scala2.12"} + _, err := ResolveTarget(t.Context(), TargetFlags{JobTask: "42.a.b.c"}, c, BundleTarget{}) + require.NoError(t, err) + assert.Equal(t, "42", c.gotJobID) + assert.Equal(t, "a.b.c", c.gotTaskKey) } -func TestResolveJobServerlessUsesRecordedVersion(t *testing.T) { - // A serverless job (isServerless=true) pins its serverless version via the - // third "recorded version" return; ResolveTarget must map it to the matching - // serverless-vN rather than the classic dbr path. - c := jobStubCompute{isServerless: true, version: "3"} - ti, err := ResolveTarget(t.Context(), TargetFlags{Job: "42"}, c, BundleTarget{}) +func TestResolveJobTaskServerlessUsesTaskVersion(t *testing.T) { + // A serverless task's environment version is read directly (no fallback): + // it maps to the matching serverless-vN, not the classic dbr path. + c := &jobStubCompute{isServerless: true, version: "3"} + ti, err := ResolveTarget(t.Context(), TargetFlags{JobTask: "42.transform"}, c, BundleTarget{}) require.NoError(t, err) assert.Equal(t, "job", ti.Source) assert.Empty(t, ti.SparkVersion) assert.Equal(t, "serverless/serverless-v3", ti.EnvKey) } -func TestResolveJobServerlessEmptyVersionFallsBackToV4(t *testing.T) { - // When the job records no serverless version, ResolveTarget defaults to v4 - // (documented stand-in), matching the bundle serverless path. - c := jobStubCompute{isServerless: true, version: ""} - ti, err := ResolveTarget(t.Context(), TargetFlags{Job: "42"}, c, BundleTarget{}) - require.NoError(t, err) - assert.Equal(t, "serverless/serverless-v4", ti.EnvKey) +func TestResolveJobTaskMissingKeyIsUsageError(t *testing.T) { + // A bare "" (no task key) is E_USAGE, not E_RESOLVE: the user must + // pick a task. The stub reports the enumerate request via ErrTaskKeyRequired. + c := &jobStubCompute{err: &ErrTaskKeyRequired{JobID: "42", TaskKeys: []string{"ingest", "transform"}}} + _, err := ResolveTarget(t.Context(), TargetFlags{JobTask: "42"}, c, BundleTarget{}) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrUsage, pe.Code) + assert.Empty(t, c.gotTaskKey, "a bare job ID yields an empty task key") +} + +func TestResolveJobTaskUnknownKeyIsResolveError(t *testing.T) { + // An unknown task key is a genuine resolve failure (E_RESOLVE), distinct from + // the missing-key usage error above. + c := &jobStubCompute{err: errors.New(`job 42 has no task "nope" (available: ingest)`)} + _, err := ResolveTarget(t.Context(), TargetFlags{JobTask: "42.nope"}, c, BundleTarget{}) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrResolve, pe.Code) } func TestValidateTargetFlagsMutuallyExclusive(t *testing.T) { diff --git a/libs/localenv/uv.go b/libs/localenv/uv.go index f2996ba29bc..81fbc8802c8 100644 --- a/libs/localenv/uv.go +++ b/libs/localenv/uv.go @@ -93,9 +93,13 @@ func (m *uvManager) EnsurePython(ctx context.Context, minor string) error { return nil } -// Provision runs `uv sync` inside projectDir to install project dependencies. -func (m *uvManager) Provision(ctx context.Context, projectDir string) error { - args := append([]string{m.bin}, m.syncArgs()...) +// Provision runs `uv sync` inside projectDir to install project dependencies, +// pinning the interpreter to pyMinor. Without --python, `uv sync` selects the +// newest installed interpreter satisfying requires-python (e.g. 3.13 for a +// ">=3.12" floor), which then fails validation against the 3.12 target; pinning +// the minor we just installed keeps the venv on the intended version. +func (m *uvManager) Provision(ctx context.Context, projectDir, pyMinor string) error { + args := append([]string{m.bin}, m.syncArgs(pyMinor)...) if err := m.runUv(ctx, args, projectDir); err != nil { return uvFailure(ErrProvision, err, "uv sync") } @@ -143,14 +147,17 @@ try: print("` + validateDBCPrefix + `" + importlib.metadata.version("databricks-connect")) except importlib.metadata.PackageNotFoundError: print("` + validateDBCPrefix + `")` - // --no-project runs the interpreter from the created .venv without re-resolving/syncing - // the project's declared dependencies, so validation observes exactly what was installed. + // Invoke the venv interpreter directly rather than `uv run`: `uv run` resolves + // the interpreter from an active VIRTUAL_ENV / CONDA_PREFIX when one is set + // (even with --no-project), which would validate whatever env the caller has + // active instead of the .venv we just provisioned. The direct path is exactly + // what was installed, so validation observes the real target. out, err := process.Background(ctx, - []string{m.bin, "run", "--no-project", "python", "-c", pyCode}, + []string{venvPython(projectDir), "-c", pyCode}, process.WithDir(projectDir), ) if err != nil { - return "", "", uvFailure(ErrValidate, err, "uv run python validation") + return "", "", uvFailure(ErrValidate, err, "venv python validation") } pyVer, ok := lineWithPrefix(out, validatePyPrefix) if !ok || pyVer == "" { @@ -180,9 +187,10 @@ func lineWithPrefix(out, prefix string) (string, bool) { return "", false } -// syncArgs returns the argument slice for `uv sync` (without the binary). -func (m *uvManager) syncArgs() []string { - return []string{"sync"} +// syncArgs returns the argument slice for `uv sync` (without the binary), +// pinning the interpreter to pyMinor via --python. +func (m *uvManager) syncArgs(pyMinor string) []string { + return []string{"sync", "--python", pyMinor} } // pythonInstallArgs returns the argument slice for `uv python install `. diff --git a/libs/localenv/uv_test.go b/libs/localenv/uv_test.go index 8678c45481b..8fcce4376a7 100644 --- a/libs/localenv/uv_test.go +++ b/libs/localenv/uv_test.go @@ -37,11 +37,24 @@ func writePipConf(t *testing.T, conf string) context.Context { func TestUvArgs(t *testing.T) { m := &uvManager{bin: "uv"} - assert.Equal(t, []string{"sync"}, m.syncArgs()) + assert.Equal(t, []string{"sync", "--python", "3.12"}, m.syncArgs("3.12")) assert.Equal(t, []string{"python", "install", "3.12"}, m.pythonInstallArgs("3.12")) assert.Equal(t, []string{"pip", "install", "pip", "--python", "/p/.venv/bin/python"}, m.pipSeedArgs("/p/.venv/bin/python")) } +func TestVenvPythonPath(t *testing.T) { + // Validate invokes this interpreter directly (not via `uv run`) so it observes + // exactly the .venv that was provisioned, ignoring any active VIRTUAL_ENV. + got := venvPython(filepath.Join("p", "proj")) + var want string + if runtime.GOOS == "windows" { + want = filepath.Join("p", "proj", ".venv", "Scripts", "python.exe") + } else { + want = filepath.Join("p", "proj", ".venv", "bin", "python") + } + assert.Equal(t, want, got) +} + func TestDiscoverUvFindsBinOnPath(t *testing.T) { dir := t.TempDir() // exec.LookPath only resolves a bare "uv" to a file with a PATHEXT extension diff --git a/libs/structs/structtag/bundletag.go b/libs/structs/structtag/bundletag.go index 9b7bb2d0ac2..f8254b53e07 100644 --- a/libs/structs/structtag/bundletag.go +++ b/libs/structs/structtag/bundletag.go @@ -11,3 +11,9 @@ func (tag BundleTag) ReadOnly() bool { func (tag BundleTag) Internal() bool { return hasOption(string(tag), "internal") } + +// Sensitive reports whether the field holds a value that must be masked +// when rendering configuration or plan output (e.g. secret values). +func (tag BundleTag) Sensitive() bool { + return hasOption(string(tag), "sensitive") +} diff --git a/libs/structs/structtag/bundletag_test.go b/libs/structs/structtag/bundletag_test.go index 3d2129ec764..ff3057cee93 100644 --- a/libs/structs/structtag/bundletag_test.go +++ b/libs/structs/structtag/bundletag_test.go @@ -8,16 +8,19 @@ import ( func TestBundleTagMethods(t *testing.T) { tests := []struct { - tag string - isReadOnly bool - isInternal bool + tag string + isReadOnly bool + isInternal bool + isSensitive bool }{ // only one annotation. {tag: "readonly", isReadOnly: true}, {tag: "internal", isInternal: true}, + {tag: "sensitive", isSensitive: true}, // multiple annotations. {tag: "readonly,internal", isReadOnly: true, isInternal: true}, + {tag: "readonly,sensitive", isReadOnly: true, isSensitive: true}, // unknown annotations are ignored. {tag: "something"}, @@ -32,6 +35,7 @@ func TestBundleTagMethods(t *testing.T) { assert.Equal(t, test.isReadOnly, tag.ReadOnly()) assert.Equal(t, test.isInternal, tag.Internal()) + assert.Equal(t, test.isSensitive, tag.Sensitive()) }) } } diff --git a/libs/structs/structwalk/redact.go b/libs/structs/structwalk/redact.go new file mode 100644 index 00000000000..d495f3b3f07 --- /dev/null +++ b/libs/structs/structwalk/redact.go @@ -0,0 +1,87 @@ +package structwalk + +import ( + "encoding/json" + "reflect" + + "github.com/databricks/cli/libs/structs/structtag" +) + +// RedactSensitiveFields returns a JSON-encoded copy of v with all string fields +// tagged `bundle:"sensitive"` replaced by [redacted]. The original value is not +// modified. This is used to scrub secrets before persisting state to disk. +// +// Only string-typed fields at any depth are redacted. The function handles +// embedded structs, pointers, slices, and maps by recursively walking the +// reflect type tree before encoding. +func RedactSensitiveFields(v any, redacted string) ([]byte, error) { + rv := reflect.ValueOf(v) + clone := deepCloneRedact(rv, redacted) + return json.Marshal(clone.Interface()) +} + +// deepCloneRedact returns a deep copy of rv with sensitive string fields replaced. +func deepCloneRedact(rv reflect.Value, redacted string) reflect.Value { + if rv.Kind() == reflect.Pointer { + if rv.IsNil() { + return reflect.Zero(rv.Type()) + } + inner := deepCloneRedact(rv.Elem(), redacted) + ptr := reflect.New(inner.Type()) + ptr.Elem().Set(inner) + return ptr + } + + switch rv.Kind() { + case reflect.Struct: + clone := reflect.New(rv.Type()).Elem() + t := rv.Type() + for i := range t.NumField() { + sf := t.Field(i) + src := rv.Field(i) + dst := clone.Field(i) + + if !sf.IsExported() { + continue + } + + btag := structtag.BundleTag(sf.Tag.Get("bundle")) + if btag.Sensitive() && sf.Type.Kind() == reflect.String { + if src.String() != "" { + dst.SetString(redacted) + } + // empty sensitive string stays empty + continue + } + + dst.Set(deepCloneRedact(src, redacted)) + } + return clone + + case reflect.Slice: + if rv.IsNil() { + return reflect.Zero(rv.Type()) + } + clone := reflect.MakeSlice(rv.Type(), rv.Len(), rv.Len()) + for i := range rv.Len() { + clone.Index(i).Set(deepCloneRedact(rv.Index(i), redacted)) + } + return clone + + case reflect.Map: + if rv.IsNil() { + return reflect.Zero(rv.Type()) + } + clone := reflect.MakeMap(rv.Type()) + for _, k := range rv.MapKeys() { + clone.SetMapIndex(k, deepCloneRedact(rv.MapIndex(k), redacted)) + } + return clone + + default: + // Scalar, interface, etc. — copy as-is. + clone := reflect.New(rv.Type()).Elem() + clone.Set(rv) + return clone + } +} diff --git a/libs/structs/structwalk/redact_test.go b/libs/structs/structwalk/redact_test.go new file mode 100644 index 00000000000..1272aec3b06 --- /dev/null +++ b/libs/structs/structwalk/redact_test.go @@ -0,0 +1,117 @@ +package structwalk_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/libs/structs/structwalk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRedactSensitiveFields_PlainString(t *testing.T) { + type Config struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + } + + src := Config{Name: "my-resource", Token: "s3cr3t"} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got Config + require.NoError(t, json.Unmarshal(b, &got)) + assert.Equal(t, "my-resource", got.Name) + assert.Equal(t, "", got.Token) +} + +func TestRedactSensitiveFields_OriginalUnchanged(t *testing.T) { + type Config struct { + Token string `json:"token" bundle:"sensitive"` + } + + src := Config{Token: "s3cr3t"} + _, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + assert.Equal(t, "s3cr3t", src.Token, "original must not be modified") +} + +func TestRedactSensitiveFields_EmptyStringUnchanged(t *testing.T) { + type Config struct { + Token string `json:"token" bundle:"sensitive"` + } + + src := Config{Token: ""} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got Config + require.NoError(t, json.Unmarshal(b, &got)) + assert.Empty(t, got.Token, "empty sensitive string should stay empty") +} + +func TestRedactSensitiveFields_NonSensitiveUnchanged(t *testing.T) { + type Config struct { + Name string `json:"name"` + } + + src := Config{Name: "hello"} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got Config + require.NoError(t, json.Unmarshal(b, &got)) + assert.Equal(t, "hello", got.Name) +} + +func TestRedactSensitiveFields_Pointer(t *testing.T) { + type Config struct { + Token string `json:"token" bundle:"sensitive"` + } + + src := &Config{Token: "s3cr3t"} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got Config + require.NoError(t, json.Unmarshal(b, &got)) + assert.Equal(t, "", got.Token) + assert.Equal(t, "s3cr3t", src.Token, "original pointer target must not be modified") +} + +func TestRedactSensitiveFields_NestedStruct(t *testing.T) { + type Inner struct { + Secret string `json:"secret" bundle:"sensitive"` + } + type Outer struct { + Name string `json:"name"` + Inner Inner `json:"inner"` + } + + src := Outer{Name: "outer", Inner: Inner{Secret: "s3cr3t"}} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got Outer + require.NoError(t, json.Unmarshal(b, &got)) + assert.Equal(t, "outer", got.Name) + assert.Equal(t, "", got.Inner.Secret) +} + +func TestRedactSensitiveFields_Slice(t *testing.T) { + type Config struct { + Token string `json:"token" bundle:"sensitive"` + Name string `json:"name"` + } + + src := []Config{{Token: "s1", Name: "a"}, {Token: "s2", Name: "b"}} + b, err := structwalk.RedactSensitiveFields(src, "") + require.NoError(t, err) + + var got []Config + require.NoError(t, json.Unmarshal(b, &got)) + assert.Equal(t, "", got[0].Token) + assert.Equal(t, "", got[1].Token) + assert.Equal(t, "a", got[0].Name) + assert.Equal(t, "b", got[1].Name) +} diff --git a/libs/telemetry/context.go b/libs/telemetry/context.go index e556e462cdf..f0eb6ad60e8 100644 --- a/libs/telemetry/context.go +++ b/libs/telemetry/context.go @@ -23,3 +23,11 @@ func fromContext(ctx context.Context) *logger { return v.(*logger) } + +// loggerFromContext returns the telemetry logger, or false if none was +// installed. Unlike fromContext it does not panic, so callers on paths that may +// run without telemetry setup can drop events instead of crashing. +func loggerFromContext(ctx context.Context) (*logger, bool) { + v, ok := ctx.Value(telemetryLoggerKey).(*logger) + return v, ok +} diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index cddc407c62e..c92b7470f3a 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -29,7 +29,16 @@ const ( ) func Log(ctx context.Context, event protos.DatabricksCliLog) { - fromContext(ctx).log(event) + // A missing logger means telemetry was never initialized on this context + // (e.g. a command invoked outside the normal cmd/root setup). Dropping the + // event is the right call: telemetry is best-effort and must never crash a + // command. + l, ok := loggerFromContext(ctx) + if !ok { + log.Debugf(ctx, "telemetry logger not found in the context; dropping event") + return + } + l.log(event) } type logger struct { diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 59633403b3d..e4ff717e6d1 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -152,3 +152,14 @@ func TestTelemetryUploadMaxRetries(t *testing.T) { assert.EqualError(t, err, "failed to upload telemetry logs after three attempts") assert.Equal(t, 3, count) } + +func TestLogWithoutLoggerDropsEvent(t *testing.T) { + // A context without a telemetry logger must not panic; the event is dropped. + assert.NotPanics(t, func() { + Log(t.Context(), protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{ + Name: protos.DummyCliEnumValue1, + }, + }) + }) +} diff --git a/libs/telemetry/protos/ssh_tunnel.go b/libs/telemetry/protos/ssh_tunnel.go index 42be8233b7f..c359002be1f 100644 --- a/libs/telemetry/protos/ssh_tunnel.go +++ b/libs/telemetry/protos/ssh_tunnel.go @@ -38,10 +38,19 @@ type SshTunnelEvent struct { // Whether the cluster was auto-started by the CLI. AutoStartCluster bool `json:"auto_start_cluster,omitempty"` + // Whether a custom base environment was set via --base-environment. + // Only the presence is recorded: the flag value can be an env.yaml path + // or display name carrying PII, so the value itself is not logged. + HasBaseEnvironment bool `json:"has_base_environment,omitempty"` + // Time in milliseconds spent starting the SSH server. // Zero if server was already running. ServerStartTimeMs int64 `json:"server_start_time_ms"` // Whether the connection was successful. IsSuccess bool `json:"is_success,omitempty"` + + // Whether a serverless usage policy was set via --usage-policy-id. + // Only the presence is recorded, not the policy ID itself. + HasUsagePolicy bool `json:"has_usage_policy,omitempty"` } diff --git a/libs/template/builtin.go b/libs/template/builtin.go index 5b10534ef54..6a86fc5d545 100644 --- a/libs/template/builtin.go +++ b/libs/template/builtin.go @@ -8,6 +8,12 @@ import ( //go:embed all:templates var builtinTemplates embed.FS +// sharedLibraryDir holds definitions shared across templates; it has no schema, so builtin() excludes it. +const sharedLibraryDir = "common" + +// sharedLibraryFS is parsed into every template's namespace by newRenderer. +var sharedLibraryFS, _ = fs.Sub(builtinTemplates, "templates/"+sharedLibraryDir+"/"+libraryDirName) + // builtinTemplate represents a template that is built into the CLI. type builtinTemplate struct { Name string @@ -32,6 +38,11 @@ func builtin() ([]builtinTemplate, error) { continue } + // The shared library dir is not a template; skip it. + if entry.Name() == sharedLibraryDir { + continue + } + templateFS, err := fs.Sub(templates, entry.Name()) if err != nil { return nil, err diff --git a/libs/template/renderer.go b/libs/template/renderer.go index 94745da2b0b..63558b8523b 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -75,20 +75,16 @@ func newRenderer( // Initialize new template, with helper functions loaded tmpl := template.New("").Funcs(helpers) - // Find user-defined templates in the library directory - matches, err := fs.Glob(templateFS, path.Join(libraryDir, "*")) + // Parse the shared library before the template's own, so a same-named + // definition in the template's library takes precedence. + tmpl, err := parseLibrary(tmpl, sharedLibraryFS, "*") if err != nil { return nil, err } - // Parse user-defined templates. - // Note: we do not call [ParseFS] with the glob directly because - // it returns an error if no files match the pattern. - if len(matches) != 0 { - tmpl, err = tmpl.ParseFS(templateFS, matches...) - if err != nil { - return nil, err - } + tmpl, err = parseLibrary(tmpl, templateFS, path.Join(libraryDir, "*")) + if err != nil { + return nil, err } srcFS, err := fs.Sub(templateFS, path.Clean(templateDir)) @@ -109,6 +105,19 @@ func newRenderer( }, nil } +// parseLibrary parses files in fsys matching pattern into tmpl, tolerating no matches +// (unlike [template.Template.ParseFS], which errors when nothing matches). +func parseLibrary(tmpl *template.Template, fsys fs.FS, pattern string) (*template.Template, error) { + matches, err := fs.Glob(fsys, pattern) + if err != nil { + return nil, err + } + if len(matches) == 0 { + return tmpl, nil + } + return tmpl.ParseFS(fsys, matches...) +} + // Executes the template by applying config on it. Returns the materialized template // as a string func (r *renderer) executeTemplate(templateDefinition string) (string, error) { diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index bb839628627..837584c452a 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -155,6 +155,34 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) { assert.Equal(t, "shreyas.goenka@databricks.com", strings.Trim(string(b), "\n\r")) } +func TestRendererSharedLibraryAndOverride(t *testing.T) { + tmpDir := t.TempDir() + + ctx := t.Context() + ctx = cmdctx.SetWorkspaceClient(ctx, nil) + helpers := loadHelpers(ctx) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/library-override/template", "./testdata/library-override/library") + require.NoError(t, err) + + err = r.walk() + require.NoError(t, err) + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(tmpDir, "out")) + require.NoError(t, err) + got := string(b) + + // agents_md is defined only in the shared library, so rendering it (a heading) + // without error proves the shared library is parsed into the template's namespace. + assert.Contains(t, got, "shared: #") + // claude_md is defined in both the shared library and this template's own + // library; the template's own definition must take precedence. + assert.Contains(t, got, "own: OWN WINS") +} + func TestRendererExecuteTemplate(t *testing.T) { templateText := `"{{.count}} items are made of {{.Material}}". {{if eq .Animal "sheep" }} diff --git a/libs/template/template.go b/libs/template/template.go index b8448936bea..cb2e1a8e3a2 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -56,6 +56,7 @@ var databricksTemplates = []Template{ { name: DefaultMinimal, description: "The minimal template, for advanced users", + aliases: []string{"empty"}, Reader: &builtinReader{name: string(DefaultMinimal)}, Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DefaultMinimal}}, }, diff --git a/libs/template/template_test.go b/libs/template/template_test.go index 4692f0acb2a..b0a5b6c641b 100644 --- a/libs/template/template_test.go +++ b/libs/template/template_test.go @@ -84,4 +84,5 @@ func TestTemplateGetDatabricksTemplate(t *testing.T) { // Assert aliases work. assert.Equal(t, MlopsStacks, GetDatabricksTemplate(TemplateName("mlops-stack")).name) + assert.Equal(t, DefaultMinimal, GetDatabricksTemplate(TemplateName("empty")).name) } diff --git a/libs/template/templates/common/library/agents.tmpl b/libs/template/templates/common/library/agents.tmpl new file mode 100644 index 00000000000..7455ab15ab7 --- /dev/null +++ b/libs/template/templates/common/library/agents.tmpl @@ -0,0 +1,45 @@ +{{- /* + * Template for the AGENTS.md / CLAUDE.md files created in new projects. + * + * Goal for this file is to point agents to Databricks AI Tools for guidance. + * A secondary goal is to offer a placeholder to users to extend with their own instructions. + * It is a non-goal to inline all instructions. + * + */ -}} + +{{- define "agents_md" -}} +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + +{{- end -}} + +{{- define "claude_md" -}} +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md +{{- end -}} diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/AGENTS.md.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/AGENTS.md.tmpl new file mode 100644 index 00000000000..4fdb4bba90c --- /dev/null +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/AGENTS.md.tmpl @@ -0,0 +1 @@ +{{template "agents_md" .}} diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/CLAUDE.md.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/CLAUDE.md.tmpl new file mode 100644 index 00000000000..197fa48b81b --- /dev/null +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/CLAUDE.md.tmpl @@ -0,0 +1 @@ +{{template "claude_md" .}} diff --git a/libs/template/templates/default-minimal/databricks_template_schema.json b/libs/template/templates/default-minimal/databricks_template_schema.json index 7faec4a5cb7..ba74e1160e2 100644 --- a/libs/template/templates/default-minimal/databricks_template_schema.json +++ b/libs/template/templates/default-minimal/databricks_template_schema.json @@ -63,13 +63,15 @@ "order": 6 }, "personal_schemas": { + "//": "Always 'yes' for default-minimal: development uses a personal schema based on the current user name", + "skip_prompt_if": {}, "type": "string", - "description": "Use a personal schema for each user working on this project\n(this is recommended, your personal schema will be '{{.default_catalog}}.{{short_name}}')", "default": "yes", "enum": [ "yes", "no, I will customize the schema configuration later in databricks.yml" ], + "description": "Use a personal schema for each user working on this project", "order": 7 }, "language_choice": { @@ -79,15 +81,15 @@ "enum": [ "python", "sql", - "other" + "skip" ], "order": 8 }, "language": { - "//": "Derived from language_choice: for Python, pyproject.toml and tests are included; for others they are excluded", + "//": "Mirrors language_choice: 'python' includes pyproject.toml and tests, 'sql' includes SQL sources, 'skip' includes neither", "skip_prompt_if": {}, "type": "string", - "default": "{{if eq .language_choice \"python\"}}python{{else}}sql{{end}}", + "default": "{{.language_choice}}", "description": "Derived language property for template logic", "order": 9 }, diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/AGENTS.md.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/AGENTS.md.tmpl new file mode 100644 index 00000000000..4fdb4bba90c --- /dev/null +++ b/libs/template/templates/default-scala/template/{{.project_name}}/AGENTS.md.tmpl @@ -0,0 +1 @@ +{{template "agents_md" .}} diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/CLAUDE.md.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/CLAUDE.md.tmpl new file mode 100644 index 00000000000..197fa48b81b --- /dev/null +++ b/libs/template/templates/default-scala/template/{{.project_name}}/CLAUDE.md.tmpl @@ -0,0 +1 @@ +{{template "claude_md" .}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/AGENTS.md.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/AGENTS.md.tmpl new file mode 100644 index 00000000000..4fdb4bba90c --- /dev/null +++ b/libs/template/templates/default-sql/template/{{.project_name}}/AGENTS.md.tmpl @@ -0,0 +1 @@ +{{template "agents_md" .}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/CLAUDE.md.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/CLAUDE.md.tmpl new file mode 100644 index 00000000000..197fa48b81b --- /dev/null +++ b/libs/template/templates/default-sql/template/{{.project_name}}/CLAUDE.md.tmpl @@ -0,0 +1 @@ +{{template "claude_md" .}} diff --git a/libs/template/templates/default/library/versions.tmpl b/libs/template/templates/default/library/versions.tmpl index 3fd128b4701..7b75820bf8a 100644 --- a/libs/template/templates/default/library/versions.tmpl +++ b/libs/template/templates/default/library/versions.tmpl @@ -47,4 +47,4 @@ 3.12 {{- end}} -{{define "latest_databricks_bundles_version" -}}1.8.0{{- end}} +{{define "latest_databricks_bundles_version" -}}1.10.0{{- end}} diff --git a/libs/template/templates/default/template/__preamble.tmpl b/libs/template/templates/default/template/__preamble.tmpl index b3f1d25039c..5bbdc888e2b 100644 --- a/libs/template/templates/default/template/__preamble.tmpl +++ b/libs/template/templates/default/template/__preamble.tmpl @@ -8,6 +8,7 @@ This file only contains template directives; it is skipped for the actual output {{$notebook_job := eq .include_job "yes"}} {{$python_package := eq .include_python "yes"}} {{$sql_language := eq .language "sql"}} +{{$python_language := eq .language "python"}} {{$lakeflow_only := eq .lakeflow_only "yes"}} {{$pydabs := eq .enable_pydabs "yes"}} {{$has_python_package_dir := and (or $python_package $notebook_job $pipeline) (not $lakeflow_only)}} @@ -35,15 +36,16 @@ This file only contains template directives; it is skipped for the actual output {{skip "{{.project_name}}/src/{{.project_name}}_etl/transformations/*.sql"}} {{end}} -{{if or $sql_language $lakeflow_only}} +{{if or (not $python_language) $lakeflow_only}} {{skip "{{.project_name}}/tests"}} {{skip "{{.project_name}}/fixtures"}} {{else if not (or $python_package $notebook_job $pipeline)}} {{skip "{{.project_name}}/tests/sample_*.py"}} {{end}} -{{if $sql_language}} +{{if not $python_language}} {{skip "{{.project_name}}/pyproject.toml"}} + {{skip "{{.project_name}}/.vscode/__builtins__.pyi"}} {{end}} {{if $pydabs}} diff --git a/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl b/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl index e566c51f740..e400f0256cc 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/.gitignore.tmpl @@ -1,9 +1,11 @@ .databricks/ +{{- if eq .language "python"}} build/ dist/ __pycache__/ *.egg-info .venv/ +{{- end}} scratch/** !scratch/README.md **/explorations/** diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl b/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl new file mode 100644 index 00000000000..f1cf722e7fc --- /dev/null +++ b/libs/template/templates/default/template/{{.project_name}}/.vscode/extensions.json.tmpl @@ -0,0 +1,9 @@ +{ + "recommendations": [ +{{- if eq .language "python"}} + "charliermarsh.ruff", +{{- end}} + "databricks.databricks", + "redhat.vscode-yaml" + ] +} diff --git a/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json b/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl similarity index 94% rename from libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json rename to libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl index d73c73b5705..e1dae324628 100644 --- a/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json +++ b/libs/template/templates/default/template/{{.project_name}}/.vscode/settings.json.tmpl @@ -1,4 +1,5 @@ { +{{- if eq .language "python"}} "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", "python.testing.pytestArgs": [ @@ -10,9 +11,11 @@ ".pytest_cache": true, "dist": true, }, +{{- end}} "files.associations": { "**/.gitkeep": "markdown" }, +{{- if eq .language "python"}} // Pylance settings (VS Code) // Set typeCheckingMode to "basic" to enable type checking! @@ -36,4 +39,5 @@ "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, }, +{{- end}} } diff --git a/libs/template/templates/default/template/{{.project_name}}/AGENTS.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/AGENTS.md.tmpl new file mode 100644 index 00000000000..4fdb4bba90c --- /dev/null +++ b/libs/template/templates/default/template/{{.project_name}}/AGENTS.md.tmpl @@ -0,0 +1 @@ +{{template "agents_md" .}} diff --git a/libs/template/templates/default/template/{{.project_name}}/CLAUDE.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/CLAUDE.md.tmpl new file mode 100644 index 00000000000..197fa48b81b --- /dev/null +++ b/libs/template/templates/default/template/{{.project_name}}/CLAUDE.md.tmpl @@ -0,0 +1 @@ +{{template "claude_md" .}} diff --git a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl index bca68edc2fb..486df502f4e 100644 --- a/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/README.md.tmpl @@ -1,12 +1,12 @@ # {{.project_name}} -{{- $skip_tests := or (eq .language "sql") (eq .lakeflow_only "yes")}} +{{- $skip_tests := or (ne .language "python") (eq .lakeflow_only "yes")}} {{- $has_python_package_dir := and (or (eq .include_python "yes") (eq .include_job "yes") (eq .include_pipeline "yes")) (not (eq .lakeflow_only "yes"))}} {{- $is_lakeflow := eq .lakeflow_only "yes"}} The '{{.project_name}}' project was generated by using the {{.template_name}} template. -* `src/`: {{if eq .language "sql"}}SQL{{else}}Python{{end}} source code for this project. +* `src/`: {{if eq .language "sql"}}SQL{{else if eq .language "python"}}Python{{else}}Any{{end}} source code for this project. {{- if $has_python_package_dir}} * `src/{{.project_name}}/`: Shared Python code that can be used by jobs and pipelines. {{- end}} @@ -28,7 +28,7 @@ Choose how you want to work on this project: (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html -{{- if not $is_lakeflow}} +{{- if and (eq .language "python") (not $is_lakeflow)}} If you're developing with an IDE, dependencies for this project should be installed using uv: diff --git a/libs/template/testdata/library-override/library/override.tmpl b/libs/template/testdata/library-override/library/override.tmpl new file mode 100644 index 00000000000..aeb54aea7f0 --- /dev/null +++ b/libs/template/testdata/library-override/library/override.tmpl @@ -0,0 +1 @@ +{{define "claude_md"}}OWN WINS{{end}} diff --git a/libs/template/testdata/library-override/template/out.tmpl b/libs/template/testdata/library-override/template/out.tmpl new file mode 100644 index 00000000000..8da557e7af8 --- /dev/null +++ b/libs/template/testdata/library-override/template/out.tmpl @@ -0,0 +1,2 @@ +own: {{template "claude_md" .}} +shared: {{template "agents_md" .}} diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 99382986d17..e767584e6ec 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -186,11 +186,67 @@ func (s *FakeWorkspace) AppsStop(_ Request, name string) Response { State: "UNAVAILABLE", Message: appStatusUnavailableMessage, } + // The backend clears both deployments on stop for the apps these fixtures use, + // so the deploy-only fields read back empty. Match that so drift tests are realistic. + app.ActiveDeployment = nil + app.PendingDeployment = nil s.Apps[name] = app return Response{Body: app} } +// AppsGet returns the app, keeping DELETING resources visible so callers can +// observe transient state (matches the cloud DELETE lifecycle). +func (s *FakeWorkspace) AppsGet(name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{"message": fmt.Sprintf("Resource apps.App not found: %v", name)}, + } + } + + return Response{Body: app} +} + +// AppsDelete simulates the real Apps DELETE lifecycle: the first DELETE flips +// the app into DELETING state (without removing it), and a second DELETE while +// still in DELETING returns 400 with the exact cloud error message. +func (s *FakeWorkspace) AppsDelete(name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + if app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateDeleting { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": fmt.Sprintf( + "Cannot delete app %s as it is not terminal with state DELETING, "+ + "and was updated less than 20 minutes ago. Please wait before trying again.", name), + }, + } + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateDeleting, + Message: "App is being deleted.", + } + app.AppStatus = &apps.ApplicationStatus{ + State: "UNAVAILABLE", + Message: appStatusUnavailableMessage, + } + s.Apps[name] = app + + return Response{} +} + func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { var app apps.App diff --git a/libs/testserver/experiments.go b/libs/testserver/experiments.go index e35967deed1..c7c3fa6c77d 100644 --- a/libs/testserver/experiments.go +++ b/libs/testserver/experiments.go @@ -71,6 +71,8 @@ func (s *FakeWorkspace) ExperimentCreate(req Request) Response { ArtifactLocation: experiment.ArtifactLocation, Tags: append(experiment.Tags, appendTags...), LifecycleStage: "active", + // Echo back like the real GetExperiment; omitting this immutable field triggers a spurious recreate. + TraceLocation: experiment.TraceLocation, } s.Experiments[experimentId] = ml.GetExperimentResponse{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index df9499c7e91..8d6e8ee0dd3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -185,6 +185,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails + InstancePools map[string]compute.GetInstancePool Catalogs map[string]catalog.CatalogInfo ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo @@ -393,6 +394,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, + InstancePools: map[string]compute.GetInstancePool{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a51a13b4afe..f98dd944d1f 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -55,6 +55,12 @@ func AddDefaultHandlers(server *Server) { }, } }) + server.Handle("POST", "/api/2.0/instance-pools/create", func(req Request) any { return req.Workspace.InstancePoolsCreate(req) }) + server.Handle("POST", "/api/2.0/instance-pools/edit", func(req Request) any { return req.Workspace.InstancePoolsEdit(req) }) + server.Handle("POST", "/api/2.0/instance-pools/delete", func(req Request) any { return req.Workspace.InstancePoolsDelete(req) }) + server.Handle("GET", "/api/2.0/instance-pools/get", func(req Request) any { + return req.Workspace.InstancePoolsGet(req, req.URL.Query().Get("instance_pool_id")) + }) server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ @@ -219,6 +225,15 @@ func AddDefaultHandlers(server *Server) { defer req.Workspace.LockUnlock()() + // The API rejects creating a directory where a file already exists; it is + // idempotent only over directories. Mirror that so callers observe the 409. + if _, isFile := req.Workspace.files[dirPath]; isFile { + return Response{ + StatusCode: 409, + Body: map[string]string{"message": "The given path points to an existing file. This API does not support operations on files."}, + } + } + // Create directory and all parent directories. for dir := dirPath; dir != "/" && dir != ""; dir = path.Dir(dir) { if _, exists := req.Workspace.directories[dir]; !exists { @@ -464,7 +479,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("GET", "/api/2.0/apps/{name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.Apps, req.Vars["name"]) + return req.Workspace.AppsGet(req.Vars["name"]) }) server.Handle("POST", "/api/2.0/apps", func(req Request) any { @@ -476,7 +491,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("DELETE", "/api/2.0/apps/{name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.Apps, req.Vars["name"]) + return req.Workspace.AppsDelete(req.Vars["name"]) }) // Schemas: diff --git a/libs/testserver/instance_pools.go b/libs/testserver/instance_pools.go new file mode 100644 index 00000000000..f6279fbfc3f --- /dev/null +++ b/libs/testserver/instance_pools.go @@ -0,0 +1,81 @@ +package testserver + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) InstancePoolsCreate(req Request) any { + // Unmarshal into the stored (GET) type directly: CreateInstancePool and + // GetInstancePool share JSON field names, so every config field is carried over. + var pool compute.GetInstancePool + if err := json.Unmarshal(req.Body, &pool); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + id := nextUUID() + pool.InstancePoolId = id + s.InstancePools[id] = pool + + return Response{Body: compute.CreateInstancePoolResponse{InstancePoolId: id}} +} + +func (s *FakeWorkspace) InstancePoolsGet(req Request, instancePoolId string) any { + defer s.LockUnlock()() + + pool, ok := s.InstancePools[instancePoolId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: pool} +} + +func (s *FakeWorkspace) InstancePoolsEdit(req Request) any { + var request compute.EditInstancePool + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + pool, ok := s.InstancePools[request.InstancePoolId] + if !ok { + return Response{StatusCode: 404} + } + + // Edit only accepts a subset of fields; overwrite those and keep the rest + // (immutable fields like aws_attributes, disk_spec) as stored. + pool.InstancePoolName = request.InstancePoolName + pool.NodeTypeId = request.NodeTypeId + pool.MinIdleInstances = request.MinIdleInstances + pool.MaxCapacity = request.MaxCapacity + pool.IdleInstanceAutoterminationMinutes = request.IdleInstanceAutoterminationMinutes + pool.CustomTags = request.CustomTags + pool.RemoteDiskThroughput = request.RemoteDiskThroughput + pool.TotalInitialRemoteDiskSize = request.TotalInitialRemoteDiskSize + s.InstancePools[request.InstancePoolId] = pool + + return Response{} +} + +func (s *FakeWorkspace) InstancePoolsDelete(req Request) any { + var request compute.DeleteInstancePool + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.InstancePools[request.InstancePoolId]; !ok { + return Response{StatusCode: 404} + } + + delete(s.InstancePools, request.InstancePoolId) + + return Response{} +} diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 357618172c6..8c929997d4c 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -150,8 +150,6 @@ func (s *FakeWorkspace) PostgresProjectCreate(req Request, projectID string) Res project.Status.DefaultEndpointSettings.SuspendTimeoutDuration = project.Spec.DefaultEndpointSettings.SuspendTimeoutDuration } } - // Clear spec so it's not returned in response (API only returns status) - project.Spec = nil } s.PostgresProjects[name] = project @@ -213,22 +211,32 @@ func (s *FakeWorkspace) PostgresProjectUpdate(req Request, name string) Response } } - // Apply updates from spec to status + // Apply updates to both spec and status. GET echoes the spec verbatim, so + // the stored spec must track updates alongside the materialized status. if updateProject.Spec != nil { + if project.Spec == nil { + project.Spec = &postgres.ProjectSpec{} + } if project.Status == nil { project.Status = &postgres.ProjectStatus{} } if updateProject.Spec.DisplayName != "" { + project.Spec.DisplayName = updateProject.Spec.DisplayName project.Status.DisplayName = updateProject.Spec.DisplayName } if updateProject.Spec.DefaultEndpointSettings != nil { + if project.Spec.DefaultEndpointSettings == nil { + project.Spec.DefaultEndpointSettings = &postgres.ProjectDefaultEndpointSettings{} + } if project.Status.DefaultEndpointSettings == nil { project.Status.DefaultEndpointSettings = &postgres.ProjectDefaultEndpointSettings{} } if updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMinCu != 0 { + project.Spec.DefaultEndpointSettings.AutoscalingLimitMinCu = updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMinCu project.Status.DefaultEndpointSettings.AutoscalingLimitMinCu = updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMinCu } if updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMaxCu != 0 { + project.Spec.DefaultEndpointSettings.AutoscalingLimitMaxCu = updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMaxCu project.Status.DefaultEndpointSettings.AutoscalingLimitMaxCu = updateProject.Spec.DefaultEndpointSettings.AutoscalingLimitMaxCu } } @@ -350,13 +358,11 @@ func (s *FakeWorkspace) PostgresBranchCreate(req Request, parent, branchID strin } // Apply user-provided spec fields to status (where input fields are surfaced). + // The spec itself is preserved and echoed verbatim on GET. if branch.Spec != nil { branch.Status.IsProtected = branch.Spec.IsProtected } - // Clear spec - API only returns status - branch.Spec = nil - s.PostgresBranches[name] = branch // Each branch implicitly provisions a primary read-write endpoint. Only create @@ -436,11 +442,16 @@ func (s *FakeWorkspace) PostgresBranchUpdate(req Request, name string) Response } } - // Apply updates from spec to status + // Apply updates to both spec and status. GET echoes the spec verbatim, so + // the stored spec must track updates alongside the materialized status. if updateBranch.Spec != nil { + if branch.Spec == nil { + branch.Spec = &postgres.BranchSpec{} + } if branch.Status == nil { branch.Status = &postgres.BranchStatus{} } + branch.Spec.IsProtected = updateBranch.Spec.IsProtected branch.Status.IsProtected = updateBranch.Spec.IsProtected } @@ -587,9 +598,6 @@ func (s *FakeWorkspace) PostgresEndpointCreate(req Request, parent, endpointID s endpoint.Status.Disabled = endpoint.Spec.Disabled } - // Clear spec - API only returns status - endpoint.Spec = nil - s.PostgresEndpoints[name] = endpoint return Response{ @@ -672,20 +680,28 @@ func (s *FakeWorkspace) PostgresEndpointUpdate(req Request, name string) Respons } } - // Apply updates from spec to status + // Apply updates to both spec and status. GET echoes the spec verbatim, so + // the stored spec must track updates alongside the materialized status. if updateEndpoint.Spec != nil { + if endpoint.Spec == nil { + endpoint.Spec = &postgres.EndpointSpec{} + } if endpoint.Status == nil { endpoint.Status = &postgres.EndpointStatus{} } if updateEndpoint.Spec.AutoscalingLimitMinCu != 0 { + endpoint.Spec.AutoscalingLimitMinCu = updateEndpoint.Spec.AutoscalingLimitMinCu endpoint.Status.AutoscalingLimitMinCu = updateEndpoint.Spec.AutoscalingLimitMinCu } if updateEndpoint.Spec.AutoscalingLimitMaxCu != 0 { + endpoint.Spec.AutoscalingLimitMaxCu = updateEndpoint.Spec.AutoscalingLimitMaxCu endpoint.Status.AutoscalingLimitMaxCu = updateEndpoint.Spec.AutoscalingLimitMaxCu } if updateEndpoint.Spec.SuspendTimeoutDuration != nil { + endpoint.Spec.SuspendTimeoutDuration = updateEndpoint.Spec.SuspendTimeoutDuration endpoint.Status.SuspendTimeoutDuration = updateEndpoint.Spec.SuspendTimeoutDuration } + endpoint.Spec.Disabled = updateEndpoint.Spec.Disabled endpoint.Status.Disabled = updateEndpoint.Spec.Disabled } @@ -775,7 +791,7 @@ func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID s } existing.UpdateTime = now existing.Status = status - existing.Spec = nil + existing.Spec = database.Spec s.PostgresDatabases[name] = existing return Response{ @@ -789,14 +805,13 @@ func (s *FakeWorkspace) PostgresDatabaseCreate(req Request, parent, databaseID s database.CreateTime = now database.UpdateTime = now - // Mirror spec onto status; the real API only echoes Status on GET. + // Mirror spec onto status; GET echoes both the spec (verbatim) and status. status := &postgres.DatabaseDatabaseStatus{ DatabaseId: databaseID, PostgresDatabase: database.Spec.PostgresDatabase, Role: database.Spec.Role, } database.Status = status - database.Spec = nil s.PostgresDatabases[name] = database @@ -890,6 +905,13 @@ func (s *FakeWorkspace) PostgresDatabaseUpdate(req Request, name string) Respons } database.Status = applyDatabaseSpecMask(database.Status, databaseStatusFromSpec(updateDatabase.Spec), paths) database.Status.DatabaseId = databaseID + + // GET echoes the spec verbatim, so keep the stored spec in sync with the + // masked status update. + database.Spec = &postgres.DatabaseDatabaseSpec{ + PostgresDatabase: database.Status.PostgresDatabase, + Role: database.Status.Role, + } } database.UpdateTime = nowTime() @@ -1120,7 +1142,7 @@ func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string, r existing.UpdateTime = now existing.Status = roleStatusFromSpec(role.Spec) existing.Status.RoleId = roleID - existing.Spec = nil + existing.Spec = role.Spec s.PostgresRoles[name] = existing return Response{ @@ -1136,7 +1158,6 @@ func (s *FakeWorkspace) PostgresRoleCreate(req Request, parent, roleID string, r role.Status = roleStatusFromSpec(role.Spec) role.Status.RoleId = roleID - role.Spec = nil s.PostgresRoles[name] = role @@ -1233,6 +1254,11 @@ func (s *FakeWorkspace) PostgresRoleUpdate(req Request, name string) Response { } role.Status = applyRoleSpecMask(role.Status, roleStatusFromSpec(updateRole.Spec), paths) role.Status.RoleId = roleID + + // GET echoes the spec verbatim, so keep the stored spec in sync while + // respecting the update_mask: only masked fields are taken from the + // request, so out-of-mask body values don't leak into the echoed spec. + role.Spec = applyRoleSpecMaskToSpec(role.Spec, updateRole.Spec, paths) } role.UpdateTime = nowTime() @@ -1279,6 +1305,36 @@ func applyRoleSpecMask(existing, desired *postgres.RoleRoleStatus, paths []strin return &result } +// applyRoleSpecMaskToSpec applies the masked fields from desired onto existing at +// the spec level, so the spec echoed on GET tracks exactly the fields the +// update_mask named (out-of-mask body values are ignored). It mirrors +// [applyRoleSpecMask] but preserves the verbatim spec shape rather than the +// materialized status. An empty paths slice replaces everything. +func applyRoleSpecMaskToSpec(existing, desired *postgres.RoleRoleSpec, paths []string) *postgres.RoleRoleSpec { + if len(paths) == 0 || existing == nil { + return desired + } + result := *existing + for _, p := range paths { + field, _, _ := strings.Cut(strings.TrimPrefix(p, "spec."), ".") + switch field { + case "spec": + result = *desired + case "postgres_role": + result.PostgresRole = desired.PostgresRole + case "auth_method": + result.AuthMethod = desired.AuthMethod + case "identity_type": + result.IdentityType = desired.IdentityType + case "membership_roles": + result.MembershipRoles = desired.MembershipRoles + case "attributes": + result.Attributes = desired.Attributes + } + } + return &result +} + // PostgresRoleDelete deletes a postgres role. func (s *FakeWorkspace) PostgresRoleDelete(name string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/postgres_test.go b/libs/testserver/postgres_test.go index 1a414cf6762..e782eb30fe1 100644 --- a/libs/testserver/postgres_test.go +++ b/libs/testserver/postgres_test.go @@ -19,8 +19,9 @@ func TestPostgresProjectCRUD(t *testing.T) { client := &http.Client{} baseURL := server.URL - // Create project - createReq, _ := http.NewRequest(http.MethodPost, baseURL+"/api/2.0/postgres/projects?project_id=test-project", nil) + // Create project with a spec so the GET below can assert the spec is echoed. + createBody := `{"spec":{"display_name":"Test Project","pg_version":16}}` + createReq, _ := http.NewRequest(http.MethodPost, baseURL+"/api/2.0/postgres/projects?project_id=test-project", strings.NewReader(createBody)) createReq.Header.Set("Authorization", "Bearer test-token") createReq.Header.Set("Content-Type", "application/json") createResp, err := client.Do(createReq) @@ -42,6 +43,10 @@ func TestPostgresProjectCRUD(t *testing.T) { var project postgres.Project require.NoError(t, json.NewDecoder(getResp.Body).Decode(&project)) assert.Equal(t, "projects/test-project", project.Name) + // The backend echoes the spec verbatim on GET (only the fields the user set). + require.NotNil(t, project.Spec) + assert.Equal(t, "Test Project", project.Spec.DisplayName) + assert.Equal(t, 16, project.Spec.PgVersion) getResp.Body.Close() // List projects @@ -407,6 +412,11 @@ func TestPostgresDatabaseUpdateMaskPreservesUnmaskedFields(t *testing.T) { require.NotNil(t, database.Status) assert.Equal(t, "renamed_db", database.Status.PostgresDatabase, "masked field should be updated") assert.Equal(t, "projects/mask-db-project/branches/main/roles/owner", database.Status.Role, "field absent from update_mask should be preserved") + + // GET echoes the spec verbatim; it must track the masked update alongside status. + require.NotNil(t, database.Spec) + assert.Equal(t, "renamed_db", database.Spec.PostgresDatabase, "spec must reflect the masked update") + assert.Equal(t, "projects/mask-db-project/branches/main/roles/owner", database.Spec.Role, "spec must preserve the field absent from update_mask") } func TestPostgresDatabaseCreateDuplicateReturns400(t *testing.T) { @@ -596,6 +606,14 @@ func TestPostgresRoleUpdateMaskPreservesUnmaskedFields(t *testing.T) { require.NotNil(t, role.Status.Attributes) assert.True(t, role.Status.Attributes.Createdb, "masked field should be updated") assert.Equal(t, "app_role", role.Status.PostgresRole, "field absent from update_mask should be preserved") + + // GET echoes the spec verbatim; it must track the update alongside status, + // and honor the update_mask — the out-of-mask postgres_role in the body must + // not leak into the echoed spec. + require.NotNil(t, role.Spec) + require.NotNil(t, role.Spec.Attributes) + assert.True(t, role.Spec.Attributes.Createdb, "spec must reflect the updated attribute") + assert.Equal(t, "app_role", role.Spec.PostgresRole, "spec must not apply the field absent from update_mask") } func TestPostgresRoleNotFoundWhenBranchNotExists(t *testing.T) { diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index abf90080200..0cdd10e78c4 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -177,6 +177,8 @@ func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vect return nil } return &vectorsearch.DeltaSyncVectorIndexSpecResponse{ + ColumnsToIndex: req.ColumnsToIndex, + ColumnsToSync: req.ColumnsToSync, EmbeddingSourceColumns: req.EmbeddingSourceColumns, EmbeddingVectorColumns: req.EmbeddingVectorColumns, EmbeddingWritebackTable: req.EmbeddingWritebackTable, diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index be19bd31e75..a1bf973801f 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -33,6 +33,7 @@ var resourceURLPatterns = map[string]string{ "vector_search_indexes": "explore/data/%s", "volumes": "explore/data/volumes/%s", "warehouses": "sql/warehouses/%s", + "instance_pools": "compute/instance-pools/%s", } // resourceAliases lets callers use bundle-config plural names as synonyms for diff --git a/python/README.md b/python/README.md index 88489d1d2d0..6c6a218e621 100644 --- a/python/README.md +++ b/python/README.md @@ -13,7 +13,7 @@ Reference documentation is available at https://databricks.github.io/cli/python/ To use `databricks-bundles`, you must first: -1. Install the [Databricks CLI](https://github.com/databricks/cli), version 1.8.0 or above +1. Install the [Databricks CLI](https://github.com/databricks/cli), version 1.10.0 or above 2. Authenticate to your Databricks workspace if you have not done so already: ```bash diff --git a/python/databricks/bundles/jobs/__init__.py b/python/databricks/bundles/jobs/__init__.py index 4a9ff6d3872..d215d5ce889 100644 --- a/python/databricks/bundles/jobs/__init__.py +++ b/python/databricks/bundles/jobs/__init__.py @@ -78,6 +78,8 @@ "DbtTask", "DbtTaskDict", "DbtTaskParam", + "DependencyMode", + "DependencyModeParam", "DeploymentSpec", "DeploymentSpecDict", "DeploymentSpecParam", @@ -444,6 +446,10 @@ DbtPlatformTaskParam, ) from databricks.bundles.jobs._models.dbt_task import DbtTask, DbtTaskDict, DbtTaskParam +from databricks.bundles.jobs._models.dependency_mode import ( + DependencyMode, + DependencyModeParam, +) from databricks.bundles.jobs._models.deployment_spec import ( DeploymentSpec, DeploymentSpecDict, diff --git a/python/databricks/bundles/jobs/_models/ai_runtime_task.py b/python/databricks/bundles/jobs/_models/ai_runtime_task.py index 1125797f9fe..592936f9509 100644 --- a/python/databricks/bundles/jobs/_models/ai_runtime_task.py +++ b/python/databricks/bundles/jobs/_models/ai_runtime_task.py @@ -47,14 +47,9 @@ class AiRuntimeTask: """ :meta private: [EXPERIMENTAL] - [Private Preview] Optional workspace or UC volume path of the uploaded code-source - archive. The CLI packages the user's local code directory into an - archive and populates this. Customers calling the Jobs API directly - should upload their archive to the workspace or a UC volume first and - supply the resulting path here. - - When set, the training node exposes the value via the `$CODE_SOURCE` - environment variable. + [Private Preview] Workspace or UC volume path of the code-source archive, unpacked on + each node and exposed through `$CODE_SOURCE`. Set by first-party + tooling; not for direct callers. """ deployments: VariableOrList[DeploymentSpec] = field(default_factory=list) @@ -112,14 +107,9 @@ class AiRuntimeTaskDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] Optional workspace or UC volume path of the uploaded code-source - archive. The CLI packages the user's local code directory into an - archive and populates this. Customers calling the Jobs API directly - should upload their archive to the workspace or a UC volume first and - supply the resulting path here. - - When set, the training node exposes the value via the `$CODE_SOURCE` - environment variable. + [Private Preview] Workspace or UC volume path of the code-source archive, unpacked on + each node and exposed through `$CODE_SOURCE`. Set by first-party + tooling; not for direct callers. """ deployments: VariableOrList[DeploymentSpecParam] diff --git a/python/databricks/bundles/jobs/_models/azure_attributes.py b/python/databricks/bundles/jobs/_models/azure_attributes.py index a89bbc9e875..05b9add8fb0 100644 --- a/python/databricks/bundles/jobs/_models/azure_attributes.py +++ b/python/databricks/bundles/jobs/_models/azure_attributes.py @@ -32,7 +32,7 @@ class AzureAttributes: capacity_reservation_group: VariableOrOptional[str] = None """ - [Public Preview] The Azure capacity reservation group resource ID to use for launching VMs. + The Azure capacity reservation group resource ID to use for launching VMs. When specified, VMs will be launched using the provided capacity reservation. Capacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not @@ -89,7 +89,7 @@ class AzureAttributesDict(TypedDict, total=False): capacity_reservation_group: VariableOrOptional[str] """ - [Public Preview] The Azure capacity reservation group resource ID to use for launching VMs. + The Azure capacity reservation group resource ID to use for launching VMs. When specified, VMs will be launched using the provided capacity reservation. Capacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not diff --git a/python/databricks/bundles/jobs/_models/cluster_spec.py b/python/databricks/bundles/jobs/_models/cluster_spec.py index bc3e59afd9d..e4b3646ac93 100644 --- a/python/databricks/bundles/jobs/_models/cluster_spec.py +++ b/python/databricks/bundles/jobs/_models/cluster_spec.py @@ -28,6 +28,10 @@ DataSecurityMode, DataSecurityModeParam, ) +from databricks.bundles.jobs._models.dependency_mode import ( + DependencyMode, + DependencyModeParam, +) from databricks.bundles.jobs._models.docker_image import DockerImage, DockerImageParam from databricks.bundles.jobs._models.gcp_attributes import ( GcpAttributes, @@ -141,6 +145,11 @@ class ClusterSpec: * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. """ + dependency_mode: VariableOrOptional[DependencyMode] = None + """ + [Beta] Controls dependency configuration for the cluster. + """ + docker_image: VariableOrOptional[DockerImage] = None """ Custom docker image BYOC @@ -412,6 +421,11 @@ class ClusterSpecDict(TypedDict, total=False): * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. """ + dependency_mode: VariableOrOptional[DependencyModeParam] + """ + [Beta] Controls dependency configuration for the cluster. + """ + docker_image: VariableOrOptional[DockerImageParam] """ Custom docker image BYOC diff --git a/python/databricks/bundles/jobs/_models/compute_spec_accelerator_type.py b/python/databricks/bundles/jobs/_models/compute_spec_accelerator_type.py index 4c0e09cc3dd..2fddf70fc2a 100644 --- a/python/databricks/bundles/jobs/_models/compute_spec_accelerator_type.py +++ b/python/databricks/bundles/jobs/_models/compute_spec_accelerator_type.py @@ -6,9 +6,9 @@ class ComputeSpecAcceleratorType(Enum): """ :meta private: [EXPERIMENTAL] - Customer-facing AcceleratorType: hardware accelerator type for the - AiRuntime workload. Per-node accelerator count is encoded in the value - name (e.g. `GPU_8xH100` means 8 H100s per node). + Hardware accelerator type for the AiRuntime workload. Per-node + accelerator count is encoded in the value name (e.g. `GPU_8xH100` means + 8 H100s per node). """ GPU_1X_A10 = "GPU_1xA10" diff --git a/python/databricks/bundles/jobs/_models/dependency_mode.py b/python/databricks/bundles/jobs/_models/dependency_mode.py new file mode 100644 index 00000000000..6612562f3dd --- /dev/null +++ b/python/databricks/bundles/jobs/_models/dependency_mode.py @@ -0,0 +1,26 @@ +from enum import Enum +from typing import Literal + + +class DependencyMode(Enum): + """ + Controls dependency configuration for the cluster. + + * `DEPENDENCY_MODE_AUTO`: Databricks will choose the most appropriate dependency mode based on your compute configuration. + * `DEPENDENCY_MODE_ENVIRONMENTS`: Enables a unified dependency management experience across classic and serverless, resulting in increased stability and performance. Supported only on DBR 19+ in Standard access mode. + * `DEPENDENCY_MODE_CLUSTER_LIBRARIES`: Legacy mode: dependencies come from cluster libraries and init scripts. + """ + + DEPENDENCY_MODE_ENVIRONMENTS = "DEPENDENCY_MODE_ENVIRONMENTS" + DEPENDENCY_MODE_CLUSTER_LIBRARIES = "DEPENDENCY_MODE_CLUSTER_LIBRARIES" + DEPENDENCY_MODE_AUTO = "DEPENDENCY_MODE_AUTO" + + +DependencyModeParam = ( + Literal[ + "DEPENDENCY_MODE_ENVIRONMENTS", + "DEPENDENCY_MODE_CLUSTER_LIBRARIES", + "DEPENDENCY_MODE_AUTO", + ] + | DependencyMode +) diff --git a/python/databricks/bundles/jobs/_models/deployment_spec.py b/python/databricks/bundles/jobs/_models/deployment_spec.py index 23a07f9adaa..123994dfe84 100644 --- a/python/databricks/bundles/jobs/_models/deployment_spec.py +++ b/python/databricks/bundles/jobs/_models/deployment_spec.py @@ -30,10 +30,21 @@ class DeploymentSpec: """ :meta private: [EXPERIMENTAL] - [Private Preview] Workspace path of the bash script to execute on each node in this - deployment. The CLI uploads the user's script and populates this. - Customers calling the Jobs API directly should upload their script to - the workspace first and supply the resulting path here. + [Private Preview] Workspace path of the script to run on each node in this deployment. + Upload the script to this path and supply the path here. When the task + runs, the file at this path is run on each node; if it fails, the task + fails with its exit code. + + Example script contents: + + # Plain Python: + python train.py --epochs 10 + + # Multi-GPU via accelerate: + accelerate launch train.py --config config.yaml + + # Distributed via torchrun: + torchrun --nproc_per_node=8 train.py """ compute: VariableOr[ComputeSpec] @@ -68,10 +79,21 @@ class DeploymentSpecDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] Workspace path of the bash script to execute on each node in this - deployment. The CLI uploads the user's script and populates this. - Customers calling the Jobs API directly should upload their script to - the workspace first and supply the resulting path here. + [Private Preview] Workspace path of the script to run on each node in this deployment. + Upload the script to this path and supply the path here. When the task + runs, the file at this path is run on each node; if it fails, the task + fails with its exit code. + + Example script contents: + + # Plain Python: + python train.py --epochs 10 + + # Multi-GPU via accelerate: + accelerate launch train.py --config config.yaml + + # Distributed via torchrun: + torchrun --nproc_per_node=8 train.py """ compute: VariableOr[ComputeSpecParam] diff --git a/python/databricks/bundles/jobs/_models/job_cluster.py b/python/databricks/bundles/jobs/_models/job_cluster.py index a23bb51f99f..7e78fa161c0 100644 --- a/python/databricks/bundles/jobs/_models/job_cluster.py +++ b/python/databricks/bundles/jobs/_models/job_cluster.py @@ -3,7 +3,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOr +from databricks.bundles.core._variable import VariableOr, VariableOrOptional from databricks.bundles.jobs._models.cluster_spec import ClusterSpec, ClusterSpecParam if TYPE_CHECKING: @@ -25,6 +25,15 @@ class JobCluster: If new_cluster, a description of a cluster that is created for each task. """ + serverless_compute_id: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The ID of the serverless compute object to bind this cluster to. At most one + JobCluster per job may set this field; the rate limit defined on the referenced + serverless compute applies across all tasks bound to this cluster. + """ + @classmethod def from_dict(cls, value: "JobClusterDict") -> "Self": return _transform(cls, value) @@ -47,5 +56,14 @@ class JobClusterDict(TypedDict, total=False): If new_cluster, a description of a cluster that is created for each task. """ + serverless_compute_id: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The ID of the serverless compute object to bind this cluster to. At most one + JobCluster per job may set this field; the rate limit defined on the referenced + serverless compute applies across all tasks bound to this cluster. + """ + JobClusterParam = JobClusterDict | JobCluster diff --git a/python/databricks/bundles/jobs/_models/periodic_trigger_configuration_time_unit.py b/python/databricks/bundles/jobs/_models/periodic_trigger_configuration_time_unit.py index a24d023a78e..c049a9788bb 100644 --- a/python/databricks/bundles/jobs/_models/periodic_trigger_configuration_time_unit.py +++ b/python/databricks/bundles/jobs/_models/periodic_trigger_configuration_time_unit.py @@ -6,8 +6,9 @@ class PeriodicTriggerConfigurationTimeUnit(Enum): HOURS = "HOURS" DAYS = "DAYS" WEEKS = "WEEKS" + MINUTES = "MINUTES" PeriodicTriggerConfigurationTimeUnitParam = ( - Literal["HOURS", "DAYS", "WEEKS"] | PeriodicTriggerConfigurationTimeUnit + Literal["HOURS", "DAYS", "WEEKS", "MINUTES"] | PeriodicTriggerConfigurationTimeUnit ) diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index bac69263b87..8992142276e 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -71,6 +71,9 @@ "GoogleAdsConfig", "GoogleAdsConfigDict", "GoogleAdsConfigParam", + "GoogleAdsCustomReportOptions", + "GoogleAdsCustomReportOptionsDict", + "GoogleAdsCustomReportOptionsParam", "GoogleAdsOptions", "GoogleAdsOptionsDict", "GoogleAdsOptionsParam", @@ -87,6 +90,9 @@ "IngestionGatewayPipelineDefinitionParam", "IngestionPipelineDefinition", "IngestionPipelineDefinitionDict", + "IngestionPipelineDefinitionFanoutOptions", + "IngestionPipelineDefinitionFanoutOptionsDict", + "IngestionPipelineDefinitionFanoutOptionsParam", "IngestionPipelineDefinitionParam", "IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig", "IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigDict", @@ -120,6 +126,9 @@ "MavenLibraryParam", "MetaMarketingOptions", "MetaMarketingOptionsDict", + "MetaMarketingOptionsMetaMarketingCustomReportOptions", + "MetaMarketingOptionsMetaMarketingCustomReportOptionsDict", + "MetaMarketingOptionsMetaMarketingCustomReportOptionsParam", "MetaMarketingOptionsParam", "NotebookLibrary", "NotebookLibraryDict", @@ -168,6 +177,12 @@ "PostgresSlotConfig", "PostgresSlotConfigDict", "PostgresSlotConfigParam", + "RedditAdsOptions", + "RedditAdsOptionsDict", + "RedditAdsOptionsParam", + "RedditAdsOptionsRedditAdsCustomReportOptions", + "RedditAdsOptionsRedditAdsCustomReportOptionsDict", + "RedditAdsOptionsRedditAdsCustomReportOptionsParam", "ReportSpec", "ReportSpecDict", "ReportSpecParam", @@ -208,6 +223,13 @@ "TikTokAdsOptions", "TikTokAdsOptionsDict", "TikTokAdsOptionsParam", + "TikTokAdsOptionsTikTokAdsCustomReportOptions", + "TikTokAdsOptionsTikTokAdsCustomReportOptionsDict", + "TikTokAdsOptionsTikTokAdsCustomReportOptionsParam", + "TikTokAdsOptionsTikTokDataLevel", + "TikTokAdsOptionsTikTokDataLevelParam", + "TikTokAdsOptionsTikTokReportType", + "TikTokAdsOptionsTikTokReportTypeParam", "Transformer", "TransformerDict", "TransformerFormat", @@ -348,6 +370,11 @@ GoogleAdsConfigDict, GoogleAdsConfigParam, ) +from databricks.bundles.pipelines._models.google_ads_custom_report_options import ( + GoogleAdsCustomReportOptions, + GoogleAdsCustomReportOptionsDict, + GoogleAdsCustomReportOptionsParam, +) from databricks.bundles.pipelines._models.google_ads_options import ( GoogleAdsOptions, GoogleAdsOptionsDict, @@ -377,6 +404,11 @@ IngestionPipelineDefinitionDict, IngestionPipelineDefinitionParam, ) +from databricks.bundles.pipelines._models.ingestion_pipeline_definition_fanout_options import ( + IngestionPipelineDefinitionFanoutOptions, + IngestionPipelineDefinitionFanoutOptionsDict, + IngestionPipelineDefinitionFanoutOptionsParam, +) from databricks.bundles.pipelines._models.ingestion_pipeline_definition_table_specific_config_query_based_connector_config import ( IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig, IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigDict, @@ -432,6 +464,11 @@ MetaMarketingOptionsDict, MetaMarketingOptionsParam, ) +from databricks.bundles.pipelines._models.meta_marketing_options_meta_marketing_custom_report_options import ( + MetaMarketingOptionsMetaMarketingCustomReportOptions, + MetaMarketingOptionsMetaMarketingCustomReportOptionsDict, + MetaMarketingOptionsMetaMarketingCustomReportOptionsParam, +) from databricks.bundles.pipelines._models.notebook_library import ( NotebookLibrary, NotebookLibraryDict, @@ -513,6 +550,16 @@ PostgresSlotConfigDict, PostgresSlotConfigParam, ) +from databricks.bundles.pipelines._models.reddit_ads_options import ( + RedditAdsOptions, + RedditAdsOptionsDict, + RedditAdsOptionsParam, +) +from databricks.bundles.pipelines._models.reddit_ads_options_reddit_ads_custom_report_options import ( + RedditAdsOptionsRedditAdsCustomReportOptions, + RedditAdsOptionsRedditAdsCustomReportOptionsDict, + RedditAdsOptionsRedditAdsCustomReportOptionsParam, +) from databricks.bundles.pipelines._models.report_spec import ( ReportSpec, ReportSpecDict, @@ -577,6 +624,19 @@ TikTokAdsOptionsDict, TikTokAdsOptionsParam, ) +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_ads_custom_report_options import ( + TikTokAdsOptionsTikTokAdsCustomReportOptions, + TikTokAdsOptionsTikTokAdsCustomReportOptionsDict, + TikTokAdsOptionsTikTokAdsCustomReportOptionsParam, +) +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_data_level import ( + TikTokAdsOptionsTikTokDataLevel, + TikTokAdsOptionsTikTokDataLevelParam, +) +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_report_type import ( + TikTokAdsOptionsTikTokReportType, + TikTokAdsOptionsTikTokReportTypeParam, +) from databricks.bundles.pipelines._models.transformer import ( Transformer, TransformerDict, diff --git a/python/databricks/bundles/pipelines/_models/azure_attributes.py b/python/databricks/bundles/pipelines/_models/azure_attributes.py index 2a2367e47e6..c03d336f3b2 100644 --- a/python/databricks/bundles/pipelines/_models/azure_attributes.py +++ b/python/databricks/bundles/pipelines/_models/azure_attributes.py @@ -32,7 +32,7 @@ class AzureAttributes: capacity_reservation_group: VariableOrOptional[str] = None """ - [Public Preview] The Azure capacity reservation group resource ID to use for launching VMs. + The Azure capacity reservation group resource ID to use for launching VMs. When specified, VMs will be launched using the provided capacity reservation. Capacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not @@ -89,7 +89,7 @@ class AzureAttributesDict(TypedDict, total=False): capacity_reservation_group: VariableOrOptional[str] """ - [Public Preview] The Azure capacity reservation group resource ID to use for launching VMs. + The Azure capacity reservation group resource ID to use for launching VMs. When specified, VMs will be launched using the provided capacity reservation. Capacity reservations can only be specified when the workspace uses injected vnet (i.e. customer defined vnet not diff --git a/python/databricks/bundles/pipelines/_models/connector_options.py b/python/databricks/bundles/pipelines/_models/connector_options.py index ba03b1619fb..0ed8fe28732 100644 --- a/python/databricks/bundles/pipelines/_models/connector_options.py +++ b/python/databricks/bundles/pipelines/_models/connector_options.py @@ -32,6 +32,10 @@ OutlookOptions, OutlookOptionsParam, ) +from databricks.bundles.pipelines._models.reddit_ads_options import ( + RedditAdsOptions, + RedditAdsOptionsParam, +) from databricks.bundles.pipelines._models.sharepoint_options import ( SharepointOptions, SharepointOptionsParam, @@ -87,9 +91,7 @@ class ConnectorOptions: kafka_options: VariableOrOptional[KafkaOptions] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] + [Beta] """ meta_ads_options: VariableOrOptional[MetaMarketingOptions] = None @@ -104,6 +106,13 @@ class ConnectorOptions: [Private Preview] Outlook specific options for ingestion """ + reddit_ads_options: VariableOrOptional[RedditAdsOptions] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Reddit Ads specific options for ingestion + """ + sharepoint_options: VariableOrOptional[SharepointOptions] = None """ :meta private: [EXPERIMENTAL] @@ -171,9 +180,7 @@ class ConnectorOptionsDict(TypedDict, total=False): kafka_options: VariableOrOptional[KafkaOptionsParam] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] + [Beta] """ meta_ads_options: VariableOrOptional[MetaMarketingOptionsParam] @@ -188,6 +195,13 @@ class ConnectorOptionsDict(TypedDict, total=False): [Private Preview] Outlook specific options for ingestion """ + reddit_ads_options: VariableOrOptional[RedditAdsOptionsParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Reddit Ads specific options for ingestion + """ + sharepoint_options: VariableOrOptional[SharepointOptionsParam] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/file_ingestion_options_schema_evolution_mode.py b/python/databricks/bundles/pipelines/_models/file_ingestion_options_schema_evolution_mode.py index ac29b94d0cb..191e0cdf994 100644 --- a/python/databricks/bundles/pipelines/_models/file_ingestion_options_schema_evolution_mode.py +++ b/python/databricks/bundles/pipelines/_models/file_ingestion_options_schema_evolution_mode.py @@ -4,8 +4,6 @@ class FileIngestionOptionsSchemaEvolutionMode(Enum): """ - :meta private: [EXPERIMENTAL] - Based on https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#how-does-auto-loader-schema-evolution-work """ diff --git a/python/databricks/bundles/pipelines/_models/google_ads_custom_report_options.py b/python/databricks/bundles/pipelines/_models/google_ads_custom_report_options.py new file mode 100644 index 00000000000..6cab19509e4 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/google_ads_custom_report_options.py @@ -0,0 +1,117 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr, VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class GoogleAdsCustomReportOptions: + """ + :meta private: [EXPERIMENTAL] + + User-defined custom report for the Google Ads connector. Mirrors the + resource + fields + segments + metrics model that Google Ads GAQL exposes. + The customer account this report runs against is supplied by the source + schema (namespace), not by this message. The whole message is gated by + the parent GoogleAdsOptions.custom_report_options stage; per-field stage + annotations are intentionally omitted. + Only supported on table-type objects: a custom report requires a + destination table, so it cannot be specified at the schema/source level. + """ + + resource: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Required) Google Ads resource to query (e.g. "ad_group_ad", + "keyword_view", "search_term_view"). Must be a resource that has + metrics. Values are validated against Google Ads' field-service + catalog at pipeline plan time. + """ + + metrics: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Metric fields to select (e.g. "metrics.clicks", + "metrics.cost_micros"). Multiple values are joined into the GAQL + SELECT clause. + """ + + resource_fields: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Resource fields to select, in fully-qualified GAQL form + (e.g. "ad_group_ad.ad.id", "ad_group_ad.status"). Multiple values are + joined into the GAQL SELECT clause. + """ + + segments: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Segment fields to select (e.g. "segments.date", + "segments.device"). Must include at least one of segments.date, + segments.week, or segments.month — that segment is used as the + incremental cursor for the table. + """ + + @classmethod + def from_dict(cls, value: "GoogleAdsCustomReportOptionsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "GoogleAdsCustomReportOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class GoogleAdsCustomReportOptionsDict(TypedDict, total=False): + """""" + + resource: VariableOr[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Required) Google Ads resource to query (e.g. "ad_group_ad", + "keyword_view", "search_term_view"). Must be a resource that has + metrics. Values are validated against Google Ads' field-service + catalog at pipeline plan time. + """ + + metrics: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Metric fields to select (e.g. "metrics.clicks", + "metrics.cost_micros"). Multiple values are joined into the GAQL + SELECT clause. + """ + + resource_fields: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Resource fields to select, in fully-qualified GAQL form + (e.g. "ad_group_ad.ad.id", "ad_group_ad.status"). Multiple values are + joined into the GAQL SELECT clause. + """ + + segments: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Segment fields to select (e.g. "segments.date", + "segments.device"). Must include at least one of segments.date, + segments.week, or segments.month — that segment is used as the + incremental cursor for the table. + """ + + +GoogleAdsCustomReportOptionsParam = ( + GoogleAdsCustomReportOptionsDict | GoogleAdsCustomReportOptions +) diff --git a/python/databricks/bundles/pipelines/_models/google_ads_options.py b/python/databricks/bundles/pipelines/_models/google_ads_options.py index 3af2893e5a5..822122b4b67 100644 --- a/python/databricks/bundles/pipelines/_models/google_ads_options.py +++ b/python/databricks/bundles/pipelines/_models/google_ads_options.py @@ -4,6 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOr, VariableOrOptional +from databricks.bundles.pipelines._models.google_ads_custom_report_options import ( + GoogleAdsCustomReportOptions, + GoogleAdsCustomReportOptionsParam, +) if TYPE_CHECKING: from typing_extensions import Self @@ -28,6 +32,16 @@ class GoogleAdsOptions: Overrides GoogleAdsConfig.manager_account_id from source_configurations when set. """ + custom_report_options: VariableOrOptional[GoogleAdsCustomReportOptions] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined Google Ads custom report: the connector synthesizes a GAQL + query from the resource, fields, segments, and metrics specified here. + When unset, the table must match one of the connector's prebuilt sources. + """ + lookback_window_days: VariableOrOptional[int] = None """ :meta private: [EXPERIMENTAL] @@ -65,6 +79,16 @@ class GoogleAdsOptionsDict(TypedDict, total=False): Overrides GoogleAdsConfig.manager_account_id from source_configurations when set. """ + custom_report_options: VariableOrOptional[GoogleAdsCustomReportOptionsParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined Google Ads custom report: the connector synthesizes a GAQL + query from the resource, fields, segments, and metrics specified here. + When unset, the table must match one of the connector's prebuilt sources. + """ + lookback_window_days: VariableOrOptional[int] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/ingestion_config.py b/python/databricks/bundles/pipelines/_models/ingestion_config.py index b9276540f8a..4ccd935e819 100644 --- a/python/databricks/bundles/pipelines/_models/ingestion_config.py +++ b/python/databricks/bundles/pipelines/_models/ingestion_config.py @@ -4,7 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional -from databricks.bundles.pipelines._models.report_spec import ReportSpec, ReportSpecParam +from databricks.bundles.pipelines._models.report_spec import ( + ReportSpec, + ReportSpecParam, +) from databricks.bundles.pipelines._models.schema_spec import SchemaSpec, SchemaSpecParam from databricks.bundles.pipelines._models.table_spec import TableSpec, TableSpecParam diff --git a/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition_fanout_options.py b/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition_fanout_options.py new file mode 100644 index 00000000000..74183042358 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/ingestion_pipeline_definition_fanout_options.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.transformer import ( + Transformer, + TransformerParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class IngestionPipelineDefinitionFanoutOptions: + """ + :meta private: [EXPERIMENTAL] + + Fanout configuration for multi-table routing from streaming sources. + Routes each input record to a destination table based on a routing + key derived from the record. The key value becomes the table name + suffix: {destination_catalog}.{destination_schema}.{key_value}. + """ + + fanout_by: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Column path or SQL expression whose value determines the destination table. + Supports dotted paths (e.g. "value.event_name") and expressions + (e.g. "value:event_name::string"). + """ + + transforms: VariableOrList[Transformer] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Optional transforms applied to each route's DataFrame before writing + to the destination table. + """ + + @classmethod + def from_dict(cls, value: "IngestionPipelineDefinitionFanoutOptionsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "IngestionPipelineDefinitionFanoutOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class IngestionPipelineDefinitionFanoutOptionsDict(TypedDict, total=False): + """""" + + fanout_by: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Column path or SQL expression whose value determines the destination table. + Supports dotted paths (e.g. "value.event_name") and expressions + (e.g. "value:event_name::string"). + """ + + transforms: VariableOrList[TransformerParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Optional transforms applied to each route's DataFrame before writing + to the destination table. + """ + + +IngestionPipelineDefinitionFanoutOptionsParam = ( + IngestionPipelineDefinitionFanoutOptionsDict + | IngestionPipelineDefinitionFanoutOptions +) diff --git a/python/databricks/bundles/pipelines/_models/json_transformer_options.py b/python/databricks/bundles/pipelines/_models/json_transformer_options.py index 22c8d37956b..88bd9c2fa8e 100644 --- a/python/databricks/bundles/pipelines/_models/json_transformer_options.py +++ b/python/databricks/bundles/pipelines/_models/json_transformer_options.py @@ -15,45 +15,33 @@ @dataclass(kw_only=True) class JsonTransformerOptions: - """ - :meta private: [EXPERIMENTAL] - """ + """""" as_variant: VariableOrOptional[bool] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Parse the entire value as a single Variant column. + [Beta] Parse the entire value as a single Variant column. """ schema: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Inline schema string for JSON parsing (Spark DDL format). + [Beta] Inline schema string for JSON parsing (Spark DDL format). """ schema_evolution_mode: VariableOrOptional[ FileIngestionOptionsSchemaEvolutionMode ] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Schema evolution mode for schema inference. + [Beta] (Optional) Schema evolution mode for schema inference. """ schema_file_path: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Path to a schema file (.ddl). + [Beta] Path to a schema file (.ddl). """ schema_hints: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Schema hints as a comma-separated string of "column_name type" pairs. + [Beta] (Optional) Schema hints as a comma-separated string of "column_name type" pairs. """ @classmethod @@ -69,39 +57,29 @@ class JsonTransformerOptionsDict(TypedDict, total=False): as_variant: VariableOrOptional[bool] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Parse the entire value as a single Variant column. + [Beta] Parse the entire value as a single Variant column. """ schema: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Inline schema string for JSON parsing (Spark DDL format). + [Beta] Inline schema string for JSON parsing (Spark DDL format). """ schema_evolution_mode: VariableOrOptional[ FileIngestionOptionsSchemaEvolutionModeParam ] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Schema evolution mode for schema inference. + [Beta] (Optional) Schema evolution mode for schema inference. """ schema_file_path: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Path to a schema file (.ddl). + [Beta] Path to a schema file (.ddl). """ schema_hints: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Schema hints as a comma-separated string of "column_name type" pairs. + [Beta] (Optional) Schema hints as a comma-separated string of "column_name type" pairs. """ diff --git a/python/databricks/bundles/pipelines/_models/kafka_options.py b/python/databricks/bundles/pipelines/_models/kafka_options.py index b37566a291b..61cd437c607 100644 --- a/python/databricks/bundles/pipelines/_models/kafka_options.py +++ b/python/databricks/bundles/pipelines/_models/kafka_options.py @@ -19,9 +19,7 @@ @dataclass(kw_only=True) class KafkaOptions: - """ - :meta private: [EXPERIMENTAL] - """ + """""" client_config: VariableOrDict[str] = field(default_factory=dict) """ @@ -34,9 +32,7 @@ class KafkaOptions: key_transformer: VariableOrOptional[Transformer] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Transformer for the message key. + [Beta] (Optional) Transformer for the message key. If not specified, the key is left as raw bytes. """ @@ -49,33 +45,25 @@ class KafkaOptions: starting_offset: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Where to begin reading when no checkpoint exists. + [Beta] (Optional) Where to begin reading when no checkpoint exists. Valid values: "latest" and "earliest". Defaults to "latest". """ topic_pattern: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Java regex pattern to subscribe to matching topics. + [Beta] Java regex pattern to subscribe to matching topics. Only one of topics or topic_pattern must be specified. """ topics: VariableOrList[str] = field(default_factory=list) """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Topics to subscribe to. + [Beta] Topics to subscribe to. Only one of topics or topic_pattern must be specified. """ value_transformer: VariableOrOptional[Transformer] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Transformer for the message value. + [Beta] (Optional) Transformer for the message value. If not specified, the value is left as raw bytes. """ @@ -101,9 +89,7 @@ class KafkaOptionsDict(TypedDict, total=False): key_transformer: VariableOrOptional[TransformerParam] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Transformer for the message key. + [Beta] (Optional) Transformer for the message key. If not specified, the key is left as raw bytes. """ @@ -116,33 +102,25 @@ class KafkaOptionsDict(TypedDict, total=False): starting_offset: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Where to begin reading when no checkpoint exists. + [Beta] (Optional) Where to begin reading when no checkpoint exists. Valid values: "latest" and "earliest". Defaults to "latest". """ topic_pattern: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Java regex pattern to subscribe to matching topics. + [Beta] Java regex pattern to subscribe to matching topics. Only one of topics or topic_pattern must be specified. """ topics: VariableOrList[str] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Topics to subscribe to. + [Beta] Topics to subscribe to. Only one of topics or topic_pattern must be specified. """ value_transformer: VariableOrOptional[TransformerParam] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] (Optional) Transformer for the message value. + [Beta] (Optional) Transformer for the message value. If not specified, the value is left as raw bytes. """ diff --git a/python/databricks/bundles/pipelines/_models/meta_marketing_options.py b/python/databricks/bundles/pipelines/_models/meta_marketing_options.py index d77005c7b40..9378ee27ec2 100644 --- a/python/databricks/bundles/pipelines/_models/meta_marketing_options.py +++ b/python/databricks/bundles/pipelines/_models/meta_marketing_options.py @@ -4,6 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.meta_marketing_options_meta_marketing_custom_report_options import ( + MetaMarketingOptionsMetaMarketingCustomReportOptions, + MetaMarketingOptionsMetaMarketingCustomReportOptionsParam, +) if TYPE_CHECKING: from typing_extensions import Self @@ -43,6 +47,17 @@ class MetaMarketingOptions: updated conversion data from the API, shared by prebuilt and custom reports. """ + custom_report_options: VariableOrOptional[ + MetaMarketingOptionsMetaMarketingCustomReportOptions + ] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Per-table custom report definition. When set, defines the shape of the insights + call for this table (level/fields/breakdowns/action_breakdowns/etc.). Supersedes the deprecated + flat report-shape fields above. + """ + level: VariableOrOptional[str] = None """ [DEPRECATED] [Beta] (Optional, DEPRECATED — use custom_report_options.level) Granularity of data to pull @@ -100,6 +115,17 @@ class MetaMarketingOptionsDict(TypedDict, total=False): updated conversion data from the API, shared by prebuilt and custom reports. """ + custom_report_options: VariableOrOptional[ + MetaMarketingOptionsMetaMarketingCustomReportOptionsParam + ] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Per-table custom report definition. When set, defines the shape of the insights + call for this table (level/fields/breakdowns/action_breakdowns/etc.). Supersedes the deprecated + flat report-shape fields above. + """ + level: VariableOrOptional[str] """ [DEPRECATED] [Beta] (Optional, DEPRECATED — use custom_report_options.level) Granularity of data to pull diff --git a/python/databricks/bundles/pipelines/_models/meta_marketing_options_meta_marketing_custom_report_options.py b/python/databricks/bundles/pipelines/_models/meta_marketing_options_meta_marketing_custom_report_options.py new file mode 100644 index 00000000000..4b19c3afc42 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/meta_marketing_options_meta_marketing_custom_report_options.py @@ -0,0 +1,123 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MetaMarketingOptionsMetaMarketingCustomReportOptions: + """ + :meta private: [EXPERIMENTAL] + + Defines the shape of a single Meta Ads custom report (one /insights call shape). + start_date, custom_insights_lookback_window live on MetaMarketingOptions, not here. + Metrics are not customer-selectable; the connector returns a fixed standard metric set. + """ + + action_attribution_windows: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + """ + + action_breakdowns: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Action breakdowns to configure for data aggregation + """ + + action_report_time: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + """ + + breakdowns: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Breakdowns to configure for data aggregation + """ + + level: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Granularity of data to pull (account, ad, adset, campaign) + """ + + time_increment: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Value in string by which to aggregate statistics (all_days, monthly or number of days) + """ + + @classmethod + def from_dict( + cls, value: "MetaMarketingOptionsMetaMarketingCustomReportOptionsDict" + ) -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MetaMarketingOptionsMetaMarketingCustomReportOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class MetaMarketingOptionsMetaMarketingCustomReportOptionsDict(TypedDict, total=False): + """""" + + action_attribution_windows: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + """ + + action_breakdowns: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Action breakdowns to configure for data aggregation + """ + + action_report_time: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + """ + + breakdowns: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Breakdowns to configure for data aggregation + """ + + level: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Granularity of data to pull (account, ad, adset, campaign) + """ + + time_increment: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Value in string by which to aggregate statistics (all_days, monthly or number of days) + """ + + +MetaMarketingOptionsMetaMarketingCustomReportOptionsParam = ( + MetaMarketingOptionsMetaMarketingCustomReportOptionsDict + | MetaMarketingOptionsMetaMarketingCustomReportOptions +) diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index 6a972356157..6437977d959 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -25,7 +25,10 @@ IngestionPipelineDefinition, IngestionPipelineDefinitionParam, ) -from databricks.bundles.pipelines._models.lifecycle import Lifecycle, LifecycleParam +from databricks.bundles.pipelines._models.lifecycle import ( + Lifecycle, + LifecycleParam, +) from databricks.bundles.pipelines._models.notifications import ( Notifications, NotificationsParam, diff --git a/python/databricks/bundles/pipelines/_models/reddit_ads_options.py b/python/databricks/bundles/pipelines/_models/reddit_ads_options.py new file mode 100644 index 00000000000..c166e9da1cf --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/reddit_ads_options.py @@ -0,0 +1,94 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.reddit_ads_options_reddit_ads_custom_report_options import ( + RedditAdsOptionsRedditAdsCustomReportOptions, + RedditAdsOptionsRedditAdsCustomReportOptionsParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class RedditAdsOptions: + """ + :meta private: [EXPERIMENTAL] + + Reddit Ads specific options for ingestion + """ + + custom_report_options: VariableOrOptional[ + RedditAdsOptionsRedditAdsCustomReportOptions + ] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined Reddit Ads custom report. When unset, the table must match + one of the connector's prebuilt sources. + """ + + lookback_window_days: VariableOrOptional[int] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Number of days to look back for report tables during incremental sync + to capture late-arriving conversions and attribution data. + If not specified, defaults to 30 days. + """ + + sync_start_date: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Start date for the initial sync of report tables in YYYY-MM-DD format. + This determines the earliest date from which to sync historical data. + If not specified, defaults to 2 years ago. + """ + + @classmethod + def from_dict(cls, value: "RedditAdsOptionsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "RedditAdsOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class RedditAdsOptionsDict(TypedDict, total=False): + """""" + + custom_report_options: VariableOrOptional[ + RedditAdsOptionsRedditAdsCustomReportOptionsParam + ] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined Reddit Ads custom report. When unset, the table must match + one of the connector's prebuilt sources. + """ + + lookback_window_days: VariableOrOptional[int] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Number of days to look back for report tables during incremental sync + to capture late-arriving conversions and attribution data. + If not specified, defaults to 30 days. + """ + + sync_start_date: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Start date for the initial sync of report tables in YYYY-MM-DD format. + This determines the earliest date from which to sync historical data. + If not specified, defaults to 2 years ago. + """ + + +RedditAdsOptionsParam = RedditAdsOptionsDict | RedditAdsOptions diff --git a/python/databricks/bundles/pipelines/_models/reddit_ads_options_reddit_ads_custom_report_options.py b/python/databricks/bundles/pipelines/_models/reddit_ads_options_reddit_ads_custom_report_options.py new file mode 100644 index 00000000000..777fe0f2fed --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/reddit_ads_options_reddit_ads_custom_report_options.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class RedditAdsOptionsRedditAdsCustomReportOptions: + """ + :meta private: [EXPERIMENTAL] + + User-defined custom report for the Reddit Ads connector. + Applies only to the custom_report table — prebuilt tables ignore this. + """ + + breakdowns: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Breakdown dimensions to group report data by. + Examples: CAMPAIGN_ID, DATE, COUNTRY, REGION, AD_ID. + Must include at least one time dimension (DATE or HOUR). + """ + + fields: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Fields to include in the report (maps to the Reddit Ads API `fields` parameter). + Examples: IMPRESSIONS, CLICKS, SPEND, CPC, CTR. + """ + + @classmethod + def from_dict( + cls, value: "RedditAdsOptionsRedditAdsCustomReportOptionsDict" + ) -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "RedditAdsOptionsRedditAdsCustomReportOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class RedditAdsOptionsRedditAdsCustomReportOptionsDict(TypedDict, total=False): + """""" + + breakdowns: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Breakdown dimensions to group report data by. + Examples: CAMPAIGN_ID, DATE, COUNTRY, REGION, AD_ID. + Must include at least one time dimension (DATE or HOUR). + """ + + fields: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Fields to include in the report (maps to the Reddit Ads API `fields` parameter). + Examples: IMPRESSIONS, CLICKS, SPEND, CPC, CTR. + """ + + +RedditAdsOptionsRedditAdsCustomReportOptionsParam = ( + RedditAdsOptionsRedditAdsCustomReportOptionsDict + | RedditAdsOptionsRedditAdsCustomReportOptions +) diff --git a/python/databricks/bundles/pipelines/_models/schema_spec.py b/python/databricks/bundles/pipelines/_models/schema_spec.py index a00c3b9334a..84c9a0e5a0e 100644 --- a/python/databricks/bundles/pipelines/_models/schema_spec.py +++ b/python/databricks/bundles/pipelines/_models/schema_spec.py @@ -8,6 +8,10 @@ ConnectorOptions, ConnectorOptionsParam, ) +from databricks.bundles.pipelines._models.ingestion_pipeline_definition_fanout_options import ( + IngestionPipelineDefinitionFanoutOptions, + IngestionPipelineDefinitionFanoutOptionsParam, +) from databricks.bundles.pipelines._models.table_specific_config import ( TableSpecificConfig, TableSpecificConfigParam, @@ -41,6 +45,16 @@ class SchemaSpec: [Public Preview] (Optional) Source Specific Connector Options """ + fanout_options: VariableOrOptional[IngestionPipelineDefinitionFanoutOptions] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Fanout options for multi-table routing from streaming sources. + When set, records are routed to destination tables based on a + per-record routing key. The key value becomes the table name: + {destination_catalog}.{destination_schema}.{key_value}. + """ + source_catalog: VariableOrOptional[str] = None """ [Public Preview] The source catalog name. Might be optional depending on the type of source. @@ -82,6 +96,16 @@ class SchemaSpecDict(TypedDict, total=False): [Public Preview] (Optional) Source Specific Connector Options """ + fanout_options: VariableOrOptional[IngestionPipelineDefinitionFanoutOptionsParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Fanout options for multi-table routing from streaming sources. + When set, records are routed to destination tables based on a + per-record routing key. The key value becomes the table name: + {destination_catalog}.{destination_schema}.{key_value}. + """ + source_catalog: VariableOrOptional[str] """ [Public Preview] The source catalog name. Might be optional depending on the type of source. diff --git a/python/databricks/bundles/pipelines/_models/table_specific_config.py b/python/databricks/bundles/pipelines/_models/table_specific_config.py index 4489e78906c..24a3936100a 100644 --- a/python/databricks/bundles/pipelines/_models/table_specific_config.py +++ b/python/databricks/bundles/pipelines/_models/table_specific_config.py @@ -121,6 +121,14 @@ class TableSpecificConfig: [Public Preview] The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. """ + source_metadata_column: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Name of the struct column added to each ingested record to hold per row source + metadata. + """ + table_properties: VariableOrDict[str] = field(default_factory=dict) """ [Beta] Table properties to set on the destination table. @@ -237,6 +245,14 @@ class TableSpecificConfigDict(TypedDict, total=False): [Public Preview] The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. """ + source_metadata_column: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Name of the struct column added to each ingested record to hold per row source + metadata. + """ + table_properties: VariableOrDict[str] """ [Beta] Table properties to set on the destination table. diff --git a/python/databricks/bundles/pipelines/_models/tik_tok_ads_options.py b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options.py index c1fd6811a52..89ea94cef13 100644 --- a/python/databricks/bundles/pipelines/_models/tik_tok_ads_options.py +++ b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options.py @@ -4,6 +4,10 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_ads_custom_report_options import ( + TikTokAdsOptionsTikTokAdsCustomReportOptions, + TikTokAdsOptionsTikTokAdsCustomReportOptionsParam, +) if TYPE_CHECKING: from typing_extensions import Self @@ -17,6 +21,19 @@ class TikTokAdsOptions: TikTok Ads specific options for ingestion """ + custom_report_options: VariableOrOptional[ + TikTokAdsOptionsTikTokAdsCustomReportOptions + ] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined TikTok Ads custom report: the connector synthesizes a report + request from the dimensions, metrics, report type, and data level specified + here. Supersedes the deprecated top-level dimensions/metrics/report_type/ + data_level/query_lifetime fields above. + """ + lookback_window_days: VariableOrOptional[int] = None """ :meta private: [EXPERIMENTAL] @@ -44,6 +61,19 @@ def as_dict(self) -> "TikTokAdsOptionsDict": class TikTokAdsOptionsDict(TypedDict, total=False): """""" + custom_report_options: VariableOrOptional[ + TikTokAdsOptionsTikTokAdsCustomReportOptionsParam + ] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Custom report definition. When set, the table is treated as a + user-defined TikTok Ads custom report: the connector synthesizes a report + request from the dimensions, metrics, report type, and data level specified + here. Supersedes the deprecated top-level dimensions/metrics/report_type/ + data_level/query_lifetime fields above. + """ + lookback_window_days: VariableOrOptional[int] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_ads_custom_report_options.py b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_ads_custom_report_options.py new file mode 100644 index 00000000000..47c2ba1f9d4 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_ads_custom_report_options.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_data_level import ( + TikTokAdsOptionsTikTokDataLevel, + TikTokAdsOptionsTikTokDataLevelParam, +) +from databricks.bundles.pipelines._models.tik_tok_ads_options_tik_tok_report_type import ( + TikTokAdsOptionsTikTokReportType, + TikTokAdsOptionsTikTokReportTypeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class TikTokAdsOptionsTikTokAdsCustomReportOptions: + """ + :meta private: [EXPERIMENTAL] + + User-defined custom report for the TikTok Ads connector. Groups the + dimensions + metrics + report type + data level that define a TikTok Ads + custom report request. + """ + + data_level: VariableOrOptional[TikTokAdsOptionsTikTokDataLevel] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Data level for the report. + If not specified, defaults to AUCTION_CAMPAIGN. + """ + + dimensions: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Dimensions to include in the report (e.g. "campaign_id", + "adgroup_id", "ad_id", "stat_time_day", "stat_time_hour"). + """ + + metrics: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Metrics to include in the report (e.g. "spend", "impressions", + "clicks", "conversion", "cpc"). + """ + + query_lifetime: VariableOrOptional[bool] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Whether to request lifetime metrics (all-time aggregated data). + When true, the report returns all-time data. + If not specified, defaults to false. + """ + + report_type: VariableOrOptional[TikTokAdsOptionsTikTokReportType] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Report type for the TikTok Ads API. + If not specified, defaults to BASIC. + """ + + @classmethod + def from_dict( + cls, value: "TikTokAdsOptionsTikTokAdsCustomReportOptionsDict" + ) -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "TikTokAdsOptionsTikTokAdsCustomReportOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class TikTokAdsOptionsTikTokAdsCustomReportOptionsDict(TypedDict, total=False): + """""" + + data_level: VariableOrOptional[TikTokAdsOptionsTikTokDataLevelParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Data level for the report. + If not specified, defaults to AUCTION_CAMPAIGN. + """ + + dimensions: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Dimensions to include in the report (e.g. "campaign_id", + "adgroup_id", "ad_id", "stat_time_day", "stat_time_hour"). + """ + + metrics: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Metrics to include in the report (e.g. "spend", "impressions", + "clicks", "conversion", "cpc"). + """ + + query_lifetime: VariableOrOptional[bool] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Whether to request lifetime metrics (all-time aggregated data). + When true, the report returns all-time data. + If not specified, defaults to false. + """ + + report_type: VariableOrOptional[TikTokAdsOptionsTikTokReportTypeParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] (Optional) Report type for the TikTok Ads API. + If not specified, defaults to BASIC. + """ + + +TikTokAdsOptionsTikTokAdsCustomReportOptionsParam = ( + TikTokAdsOptionsTikTokAdsCustomReportOptionsDict + | TikTokAdsOptionsTikTokAdsCustomReportOptions +) diff --git a/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_data_level.py b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_data_level.py new file mode 100644 index 00000000000..11f6159bcf8 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_data_level.py @@ -0,0 +1,21 @@ +from enum import Enum +from typing import Literal + + +class TikTokAdsOptionsTikTokDataLevel(Enum): + """ + :meta private: [EXPERIMENTAL] + + Data level for TikTok Ads report aggregation. + """ + + AUCTION_ADVERTISER = "AUCTION_ADVERTISER" + AUCTION_CAMPAIGN = "AUCTION_CAMPAIGN" + AUCTION_ADGROUP = "AUCTION_ADGROUP" + AUCTION_AD = "AUCTION_AD" + + +TikTokAdsOptionsTikTokDataLevelParam = ( + Literal["AUCTION_ADVERTISER", "AUCTION_CAMPAIGN", "AUCTION_ADGROUP", "AUCTION_AD"] + | TikTokAdsOptionsTikTokDataLevel +) diff --git a/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_report_type.py b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_report_type.py new file mode 100644 index 00000000000..272d36ff0c0 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/tik_tok_ads_options_tik_tok_report_type.py @@ -0,0 +1,23 @@ +from enum import Enum +from typing import Literal + + +class TikTokAdsOptionsTikTokReportType(Enum): + """ + :meta private: [EXPERIMENTAL] + + Report type for TikTok Ads API. + """ + + BASIC = "BASIC" + AUDIENCE = "AUDIENCE" + PLAYABLE_AD = "PLAYABLE_AD" + DSA = "DSA" + BUSINESS_CENTER = "BUSINESS_CENTER" + GMV_MAX = "GMV_MAX" + + +TikTokAdsOptionsTikTokReportTypeParam = ( + Literal["BASIC", "AUDIENCE", "PLAYABLE_AD", "DSA", "BUSINESS_CENTER", "GMV_MAX"] + | TikTokAdsOptionsTikTokReportType +) diff --git a/python/databricks/bundles/pipelines/_models/transformer.py b/python/databricks/bundles/pipelines/_models/transformer.py index 84af0aa9d98..a913e0e6cee 100644 --- a/python/databricks/bundles/pipelines/_models/transformer.py +++ b/python/databricks/bundles/pipelines/_models/transformer.py @@ -20,23 +20,17 @@ @dataclass(kw_only=True) class Transformer: """ - :meta private: [EXPERIMENTAL] - Specifies how to transform binary data into structured data. """ format: VariableOrOptional[TransformerFormat] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Required: the wire format of the data. + [Beta] Required: the wire format of the data. """ json_options: VariableOrOptional[JsonTransformerOptions] = None """ - :meta private: [EXPERIMENTAL] - - [Private Preview] + [Beta] """ @classmethod @@ -52,16 +46,12 @@ class TransformerDict(TypedDict, total=False): format: VariableOrOptional[TransformerFormatParam] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] Required: the wire format of the data. + [Beta] Required: the wire format of the data. """ json_options: VariableOrOptional[JsonTransformerOptionsParam] """ - :meta private: [EXPERIMENTAL] - - [Private Preview] + [Beta] """ diff --git a/python/databricks/bundles/pipelines/_models/transformer_format.py b/python/databricks/bundles/pipelines/_models/transformer_format.py index 5682c94d794..5c2a771bbeb 100644 --- a/python/databricks/bundles/pipelines/_models/transformer_format.py +++ b/python/databricks/bundles/pipelines/_models/transformer_format.py @@ -3,10 +3,6 @@ class TransformerFormat(Enum): - """ - :meta private: [EXPERIMENTAL] - """ - STRING = "STRING" JSON = "JSON" diff --git a/python/databricks/bundles/version.py b/python/databricks/bundles/version.py index 29654eec0a4..fcfdf3836e2 100644 --- a/python/databricks/bundles/version.py +++ b/python/databricks/bundles/version.py @@ -1 +1 @@ -__version__ = "1.8.0" +__version__ = "1.10.0" diff --git a/python/pyproject.toml b/python/pyproject.toml index 43eb14e4d26..dbf49bceb5b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "databricks-bundles" description = "Python support for Declarative Automation Bundles" -version = "1.8.0" +version = "1.10.0" authors = [ { name = "Gleb Kanterov", email = "gleb.kanterov@databricks.com" }, diff --git a/python/uv.lock b/python/uv.lock index b2c631fd66b..47e2817600b 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -166,7 +166,7 @@ toml = [ [[package]] name = "databricks-bundles" -version = "1.8.0" +version = "1.10.0" source = { editable = "." } [package.dev-dependencies] diff --git a/tools/gh_parse.py b/tools/gh_parse.py index d6b5a299917..b64f4dcddc0 100755 --- a/tools/gh_parse.py +++ b/tools/gh_parse.py @@ -245,12 +245,15 @@ def load_known_failures(): def cleanup_env(name): """ - >>> cleanup_env("test-output-aws-prod-is-linux-ubuntu-latest") + >>> cleanup_env("test-output-aws-cli-is-linux-ubuntu-latest") 'aws linux' - >>> cleanup_env("test-output-gcp-prod-is-windows-server-latest") + >>> cleanup_env("test-output-gcp-cli-is-windows-server-latest") 'gcp windows' + >>> cleanup_env("test-output-aws-prod-is-linux-ubuntu-latest") + 'aws linux' + >>> cleanup_env("test-output-azure-prod-ucws-is-linux-ubuntu-latest") 'azure-ucws linux' """ @@ -259,6 +262,7 @@ def cleanup_env(name): name = name.removeprefix("test-output-") name = name.replace("-prod-ucws-is-", "-ucws-") name = name.replace("-prod-is-", "-") + name = name.replace("-cli-is-", "-") name = name.replace("-linux-ubuntu-latest", " linux") name = name.replace("-windows-server-latest", " windows") return name diff --git a/tools/go.mod b/tools/go.mod index f0c2a3dcba7..4fa5d6c7625 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -7,7 +7,7 @@ toolchain go1.26.5 require github.com/stretchr/testify v1.11.1 require ( - golang.org/x/mod v0.37.0 + golang.org/x/mod v0.38.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -217,12 +217,12 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.44.0 // indirect - golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/vuln v1.3.0 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index 8f19e0e06d2..d07e2f621b8 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -700,8 +700,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -736,8 +736,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -757,8 +757,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -803,10 +803,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -872,8 +872,8 @@ golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0t golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= diff --git a/tools/summarize_failed_tests.py b/tools/summarize_failed_tests.py new file mode 100755 index 00000000000..dfcbb564acf --- /dev/null +++ b/tools/summarize_failed_tests.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# /// +""" +Summarize failed tests from a gotestsum --jsonfile output for a GitHub job summary. + +`task test` prints a lot of output that GitHub is slow to render, so this surfaces +just the failed test names as a Markdown table. It reads the concatenated +gotestsum --jsonfile output (go test -json events, one JSON object per line) and +prints Markdown to stdout; the workflow tees that into $GITHUB_STEP_SUMMARY. +""" + +import argparse +import json + +# go test -json result actions. Only these mark the terminal state of a test or +# package; intermediate actions like "run" and "output" are ignored. +RESULT_ACTIONS = ("pass", "fail", "skip") + + +def load_events(path): + """Yield the result events (parsed JSON objects) from a gotestsum jsonfile.""" + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + event = json.loads(line) + if event.get("Action") in RESULT_ACTIONS: + yield event + + +def last_action_by_key(events, key): + """Map each key to the Action of its last event.""" + last = {} + for event in events: + last[key(event)] = event["Action"] + return last + + +def failed_tests(events): + """Return sorted (package, test) pairs whose last result was a failure. + + A test can appear more than once because of --rerun-fails, so we group by + package+test and keep only those whose last result was a failure (recovered + flakes end on "pass"). + """ + test_events = [e for e in events if e.get("Test") is not None] + last = last_action_by_key(test_events, lambda e: (e["Package"], e["Test"])) + return sorted(key for key, action in last.items() if action == "fail") + + +def failed_packages_without_test(events, failed_test_packages): + """Return sorted packages that failed with no failing test (build error / panic). + + A build error or panic surfaces as a package-level "fail" event (Test == null) + with no failing test in the package. go test -json also emits a package-level + "fail" for every package that merely has a failing test, so exclude packages + already reported via failed_tests; otherwise those get double-reported here as + spurious build errors. + """ + package_events = [e for e in events if e.get("Test") is None] + last = last_action_by_key(package_events, lambda e: e["Package"]) + return sorted( + package for package, action in last.items() if action == "fail" and package not in failed_test_packages + ) + + +def render(tests, packages): + lines = ["## Failed tests"] + if tests: + lines += ["", "| Package | Test |", "| --- | --- |"] + lines += [f"| {package} | `{test}` |" for package, test in tests] + if packages: + lines += ["", "### Packages that failed without a test (build error / panic)", "", "```"] + lines += packages + lines += ["```"] + if not tests and not packages: + lines += ["", "No failed tests found in test-output.json (the failure may be outside the test run)."] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("jsonfile", help="Path to the gotestsum --jsonfile output") + args = parser.parse_args() + + try: + events = list(load_events(args.jsonfile)) + except FileNotFoundError: + print(f"{args.jsonfile} not found; skipping failed-test summary") + return + + tests = failed_tests(events) + packages = failed_packages_without_test(events, {package for package, _ in tests}) + print(render(tests, packages)) + + +if __name__ == "__main__": + main() diff --git a/yamlfmt.yml b/yamlfmt.yml index 6eee333e8d4..25ffe28dd53 100644 --- a/yamlfmt.yml +++ b/yamlfmt.yml @@ -3,6 +3,7 @@ exclude: - acceptance/selftest/bundleconfig # https://github.com/google/yamlfmt/issues/254 - acceptance/bundle/artifacts/nil_artifacts/databricks.yml # https://github.com/google/yamlfmt/issues/253 - bundle/direct/dresources/apitypes.yml # remove comments + - acceptance/bundle/resources/permissions/jobs/update/databricks.yml # comment indentation is significant: it is uncommented into a top-level permissions: [] key formatter: type: basic retain_line_breaks_single: true