From 61ee2a4f51fb247e4275773137ea447e787aaf33 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 12:58:55 +0000 Subject: [PATCH 1/2] Fix model serving telemetry_config never converging and silently dropping updates telemetry_config is only carried by CreateServingEndpoint. The serving endpoints GET API never echoes it back, and no update API accepts it, so classify it as input_only for remote changes and no_update_api for local ones. --- ...odel-serving-telemetry-config-converges.md | 1 + ...serving_endpoint_telemetry_config.yml.tmpl | 10 ++ .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + .../drift/telemetry_config/databricks.yml | 25 +++++ .../drift/telemetry_config/out.test.toml | 3 + .../drift/telemetry_config/output.txt | 95 +++++++++++++++++++ .../drift/telemetry_config/script | 18 ++++ .../drift/telemetry_config/test.toml | 4 + bundle/direct/dresources/resources.yml | 11 +++ 14 files changed, 173 insertions(+) create mode 100644 .nextchanges/bundles/model-serving-telemetry-config-converges.md create mode 100644 acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script create mode 100644 acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml diff --git a/.nextchanges/bundles/model-serving-telemetry-config-converges.md b/.nextchanges/bundles/model-serving-telemetry-config-converges.md new file mode 100644 index 00000000000..43c60d9cd61 --- /dev/null +++ b/.nextchanges/bundles/model-serving-telemetry-config-converges.md @@ -0,0 +1 @@ +Fixed `telemetry_config` on model serving endpoints never converging on the direct engine: the field is only accepted on create and is never returned by the serving endpoints GET API, so every subsequent `bundle plan` reported the endpoint as a perpetual update. Changing `telemetry_config` on a deployed endpoint no longer plans an update that cannot be applied, since the API has no method to update it. diff --git a/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl new file mode 100644 index 00000000000..21cc542afe9 --- /dev/null +++ b/acceptance/bundle/invariant/configs/model_serving_endpoint_telemetry_config.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + model_serving_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index d652e02679c..9df6a941a97 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -23,6 +23,7 @@ EnvMatrix.INPUT_CONFIG = [ "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index c9dda900f76..d4caaf86c82 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -22,6 +22,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 45cdef9cc7c..c440507d525 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a53a93b84a6..6e6e50dfd81 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -48,6 +48,7 @@ EnvMatrix.INPUT_CONFIG = [ "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry_config.yml.tmpl", "pipeline.yml.tmpl", "pipeline_allow_duplicate_names.yml.tmpl", "pipeline_apply_policy_default_values.yml.tmpl", diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml new file mode 100644 index 00000000000..703699db5ed --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + endpoint1: + name: test-endpoint + config: + served_entities: + - name: prod + external_model: + name: gpt-4o-mini + provider: openai + task: llm/v1/chat + openai_config: + openai_api_key_plaintext: sk-test-plaintext-key + traffic_config: + routes: + - served_model_name: prod + traffic_percentage: 100 + # Accepted by the create API, never returned on GET, and there is no + # update API for it. + telemetry_config: + inference_table_config: + sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt new file mode 100644 index 00000000000..28bd2b2ef12 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -0,0 +1,95 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op even though telemetry_config does not round-trip +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== telemetry_config is skipped rather than planned as an update the API cannot make +>>> [CLI] bundle plan --output json +{ + "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { + "action": "skip", + "reason": "input_only", + "old": "sk-test-plaintext-key", + "new": "sk-test-plaintext-key", + "remote": "" + }, + "description": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "route_optimized": { + "action": "skip", + "reason": "empty", + "remote": false + }, + "telemetry_config": { + "action": "skip", + "reason": "input_only", + "old": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + }, + "new": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} + +=== Changing telemetry_config does not plan an update: it can only be set at create time +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no config/ai-gateway/tags update calls) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //serving-endpoints +{ + "method": "POST", + "path": "/api/2.0/serving-endpoints", + "body": { + "config": { + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key_plaintext": "sk-test-plaintext-key" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ], + "traffic_config": { + "routes": [ + { + "served_model_name": "prod", + "traffic_percentage": 100 + } + ] + } + }, + "name": "test-endpoint", + "telemetry_config": { + "inference_table_config": { + "sampling_fraction": 0.5 + } + } + } +} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script new file mode 100644 index 00000000000..59a8e2ffbd8 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op even though telemetry_config does not round-trip" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "telemetry_config is skipped rather than planned as an update the API cannot make" +trace $CLI bundle plan --output json | jq '.plan[].changes' + +title "Changing telemetry_config does not plan an update: it can only be set at create time" +update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" +trace $CLI bundle deploy +trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml new file mode 100644 index 00000000000..a43d26a50dd --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/test.toml @@ -0,0 +1,4 @@ +RecordRequests = true + +# Scope this regression to the direct engine where the classification lives. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index d1d420b9ffe..ea3a206c1e5 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -279,6 +279,11 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only + # Accepted by the create API but GET never echoes it back, so the remote + # always reports empty and every deploy would otherwise replan the same + # update forever. + - field: telemetry_config + reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. - field: config.served_entities[*].external_model.ai21labs_config.ai21labs_api_key_plaintext reason: input_only @@ -309,6 +314,12 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented + # telemetry_config can only be set at create time: there is no update API + # for it on ServingEndpointsAPI, and no update request body in the OpenAPI + # spec carries the field. Without this rule the plan promises an update + # that DoUpdate has no call to make, so the change is silently dropped. + - field: telemetry_config + reason: no_update_api backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/serving/resource_model_serving.go#L383 # common.CustomizeSchemaPath(m, "config", "served_entities", "name").SetComputed() From 75c676571e122ddab0c7558c617e9509e260c6ae Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 13:09:43 +0000 Subject: [PATCH 2/2] Shorten comments --- .../drift/telemetry_config/databricks.yml | 2 -- .../drift/telemetry_config/output.txt | 4 ++-- .../drift/telemetry_config/script | 4 ++-- bundle/direct/dresources/resources.yml | 9 ++------- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml index 703699db5ed..5b40d000814 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/databricks.yml @@ -18,8 +18,6 @@ resources: routes: - served_model_name: prod traffic_percentage: 100 - # Accepted by the create API, never returned on GET, and there is no - # update API for it. telemetry_config: inference_table_config: sampling_fraction: 0.5 diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt index 28bd2b2ef12..6107891c48e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/output.txt @@ -10,7 +10,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== telemetry_config is skipped rather than planned as an update the API cannot make +=== telemetry_config is skipped (confirms the matched rules) >>> [CLI] bundle plan --output json { "config.served_entities[0].external_model.openai_config.openai_api_key_plaintext": { @@ -46,7 +46,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } -=== Changing telemetry_config does not plan an update: it can only be set at create time +=== Changing telemetry_config plans nothing: it is create-only >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script index 59a8e2ffbd8..e4f0ed5bbf8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config/script @@ -6,10 +6,10 @@ trace $CLI bundle deploy title "Plan is a no-op even though telemetry_config does not round-trip" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" -title "telemetry_config is skipped rather than planned as an update the API cannot make" +title "telemetry_config is skipped (confirms the matched rules)" trace $CLI bundle plan --output json | jq '.plan[].changes' -title "Changing telemetry_config does not plan an update: it can only be set at create time" +title "Changing telemetry_config plans nothing: it is create-only" update_file.py databricks.yml "sampling_fraction: 0.5" "sampling_fraction: 0.9" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index ea3a206c1e5..c2a3c6706b3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -279,9 +279,7 @@ resources: # Accepted on write but not returned by GET. - field: config.served_entities[*].burst_scaling_enabled reason: input_only - # Accepted by the create API but GET never echoes it back, so the remote - # always reports empty and every deploy would otherwise replan the same - # update forever. + # Remote always reads back empty, so without this it never converges. - field: telemetry_config reason: input_only # Write-only secrets: the backend stores them and returns the reference field, not the plaintext. @@ -314,10 +312,7 @@ resources: reason: no_update_api - field: rate_limits reason: not_implemented - # telemetry_config can only be set at create time: there is no update API - # for it on ServingEndpointsAPI, and no update request body in the OpenAPI - # spec carries the field. Without this rule the plan promises an update - # that DoUpdate has no call to make, so the change is silently dropped. + # Only exists on CreateServingEndpoint; no update API accepts it. - field: telemetry_config reason: no_update_api backend_defaults: