diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md new file mode 100644 index 00000000000..43c60d9cd61 --- /dev/null +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -0,0 +1 @@ +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the field is only accepted on create and is never returned by the serving endpoints GET API, so every subsequent `bundle plan` reported the endpoint as a perpetual update. Changing `telemetry_config` on a deployed endpoint no longer plans an update that cannot be applied, since the API has no method to update it. diff --git a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl new file mode 100644 index 00000000000..21cc542afe9 --- /dev/null +++ b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + model_serving_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index d652e02679c..9df6a941a97 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -23,6 +23,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index c9dda900f76..d4caaf86c82 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -22,6 +22,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a53a93b84a6..6e6e50dfd81 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -48,6 +48,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml new file mode 100644 index 00000000000..5b40d000814 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + endpoint1: + name: test-endpoint + config: + served_entities: + - name: prod + external_model: + name: gpt-4o-mini + provider: openai + task: llm/v1/chat + openai_config: + openai_api_key_plaintext: sk-test-plaintext-key + traffic_config: + routes: + - served_model_name: prod + traffic_percentage: 100 + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt new file mode 100644 index 00000000000..6107891c48e --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -0,0 +1,95 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op even though telemetry_config does not round-trip +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== telemetry_config is skipped (confirms the matched rules) +>>> [CLI] bundle plan --output json +{ + "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "action": "skip", + "reason": "input_only", + "old": "sk-test-plaintext-key", + "new": "sk-test-plaintext-key", + "remote": "" + }, + "description": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config": { + "action": "skip", + "reason": "input_only", + "old": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + }, + "new": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} + +=== Changing telemetry_config plans nothing: it is create-only +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no config/ai-gateway/tags update calls) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //serving-endpoints +{ + "method": "POST", + "path": "/api/2.0/serving-endpoints", + "body": { + "config": { + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key_plaintext": "sk-test-plaintext-key" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ], + "traffic_config": { + "routes": [ + { + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } + }, + "name": "test-endpoint", + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script new file mode 100644 index 00000000000..e4f0ed5bbf8 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op even though telemetry_config does not round-trip" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "telemetry_config is skipped (confirms the matched rules)" +trace $CLI bundle plan --output json | jq '.plan[].changes' + +title "Changing telemetry_config plans nothing: it is create-only" +update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" +trace $CLI bundle deploy +trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml new file mode 100644 index 00000000000..a43d26a50dd --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -0,0 +1,4 @@ +RecordRequests = true + +# Scope this regression to the direct engine where the classification lives. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index d1d420b9ffe..c2a3c6706b3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -279,6 +279,9 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only + # Remote always reads back empty, so without this it never converges. + - field: telemetry_config + reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext reason: input_only @@ -309,6 +312,9 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented + # Only exists on CreateServingEndpoint; no update API accepts it. + - field: telemetry_config + reason: no_update_api backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed()