From 121abc94d9fa5b48e7a3060574f7d511a15a3c9d Mon Sep 17 00:00:00 2001 From: Niklas Semmler <1880943+niklassemmler@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:47:55 +0200 Subject: [PATCH 1/5] docs(api): document metadata is set / is not set filter operators (#1946) Regenerated langfuse/api from the upstream Fern definition: baseline langfuse/langfuse 619dbe3 (last sync, #1866) plus langfuse/langfuse#17357 and #18404. Split out of the bundled API-spec bot PR #1943. Co-authored-by: Claude Opus 5.5 (1M context) --- langfuse/api/__init__.py | 3 ++ langfuse/api/evaluation_commons/__init__.py | 3 ++ .../api/evaluation_commons/types/__init__.py | 5 +++ .../types/evaluation_rule_filter.py | 25 ++++++----- ...tion_rule_string_object_filter_operator.py | 42 +++++++++++++++++++ .../string_object_evaluation_rule_filter.py | 15 +++++-- langfuse/api/legacy/observations_v1/client.py | 4 +- .../api/legacy/observations_v1/raw_client.py | 4 +- langfuse/api/observations/client.py | 4 +- langfuse/api/observations/raw_client.py | 4 +- langfuse/api/scores/client.py | 4 +- langfuse/api/scores/raw_client.py | 4 +- langfuse/api/trace/client.py | 4 +- langfuse/api/trace/raw_client.py | 4 +- 14 files changed, 95 insertions(+), 30 deletions(-) create mode 100644 langfuse/api/evaluation_commons/types/evaluation_rule_string_object_filter_operator.py diff --git a/langfuse/api/__init__.py b/langfuse/api/__init__.py index ec4d25b20..275fdff7c 100644 --- a/langfuse/api/__init__.py +++ b/langfuse/api/__init__.py @@ -187,6 +187,7 @@ EvaluationRuleReadFilterBase, EvaluationRuleReadFilterWithKey, EvaluationRuleStringFilterOperator, + EvaluationRuleStringObjectFilterOperator, EvaluatorOutputDefinition, EvaluatorOutputDefinitionBase, EvaluatorOutputDefinition_Boolean, @@ -607,6 +608,7 @@ "EvaluationRuleReadFilterBase": ".evaluation_commons", "EvaluationRuleReadFilterWithKey": ".evaluation_commons", "EvaluationRuleStringFilterOperator": ".evaluation_commons", + "EvaluationRuleStringObjectFilterOperator": ".evaluation_commons", "EvaluationRulesPage": ".evaluation_rules", "EvaluationUnauthorizedError": ".evaluation_errors", "Evaluator": ".evaluators", @@ -1062,6 +1064,7 @@ def __dir__(): "EvaluationRuleReadFilterBase", "EvaluationRuleReadFilterWithKey", "EvaluationRuleStringFilterOperator", + "EvaluationRuleStringObjectFilterOperator", "EvaluationRulesPage", "EvaluationUnauthorizedError", "Evaluator", diff --git a/langfuse/api/evaluation_commons/__init__.py b/langfuse/api/evaluation_commons/__init__.py index 00912e7ce..e16b31a7c 100644 --- a/langfuse/api/evaluation_commons/__init__.py +++ b/langfuse/api/evaluation_commons/__init__.py @@ -32,6 +32,7 @@ EvaluationRuleReadFilterBase, EvaluationRuleReadFilterWithKey, EvaluationRuleStringFilterOperator, + EvaluationRuleStringObjectFilterOperator, EvaluatorOutputDefinition, EvaluatorOutputDefinitionBase, EvaluatorOutputDefinition_Boolean, @@ -85,6 +86,7 @@ "EvaluationRuleReadFilterBase": ".types", "EvaluationRuleReadFilterWithKey": ".types", "EvaluationRuleStringFilterOperator": ".types", + "EvaluationRuleStringObjectFilterOperator": ".types", "EvaluatorOutputDefinition": ".types", "EvaluatorOutputDefinitionBase": ".types", "EvaluatorOutputDefinition_Boolean": ".types", @@ -167,6 +169,7 @@ def __dir__(): "EvaluationRuleReadFilterBase", "EvaluationRuleReadFilterWithKey", "EvaluationRuleStringFilterOperator", + "EvaluationRuleStringObjectFilterOperator", "EvaluatorOutputDefinition", "EvaluatorOutputDefinitionBase", "EvaluatorOutputDefinition_Boolean", diff --git a/langfuse/api/evaluation_commons/types/__init__.py b/langfuse/api/evaluation_commons/types/__init__.py index d04011e88..7c5018f54 100644 --- a/langfuse/api/evaluation_commons/types/__init__.py +++ b/langfuse/api/evaluation_commons/types/__init__.py @@ -45,6 +45,9 @@ from .evaluation_rule_string_filter_operator import ( EvaluationRuleStringFilterOperator, ) + from .evaluation_rule_string_object_filter_operator import ( + EvaluationRuleStringObjectFilterOperator, + ) from .evaluator_output_definition import ( EvaluatorOutputDefinition, EvaluatorOutputDefinition_Boolean, @@ -101,6 +104,7 @@ "EvaluationRuleReadFilterBase": ".evaluation_rule_read_filter_base", "EvaluationRuleReadFilterWithKey": ".evaluation_rule_read_filter_with_key", "EvaluationRuleStringFilterOperator": ".evaluation_rule_string_filter_operator", + "EvaluationRuleStringObjectFilterOperator": ".evaluation_rule_string_object_filter_operator", "EvaluatorOutputDefinition": ".evaluator_output_definition", "EvaluatorOutputDefinitionBase": ".evaluator_output_definition_base", "EvaluatorOutputDefinition_Boolean": ".evaluator_output_definition", @@ -183,6 +187,7 @@ def __dir__(): "EvaluationRuleReadFilterBase", "EvaluationRuleReadFilterWithKey", "EvaluationRuleStringFilterOperator", + "EvaluationRuleStringObjectFilterOperator", "EvaluatorOutputDefinition", "EvaluatorOutputDefinitionBase", "EvaluatorOutputDefinition_Boolean", diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py index f281dbb17..160026a20 100644 --- a/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py @@ -16,6 +16,9 @@ from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator +from .evaluation_rule_string_object_filter_operator import ( + EvaluationRuleStringObjectFilterOperator, +) class EvaluationRuleFilter_Datetime(UniversalBaseModel): @@ -34,7 +37,7 @@ class EvaluationRuleFilter_Datetime(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -97,7 +100,7 @@ class EvaluationRuleFilter_String(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -160,7 +163,7 @@ class EvaluationRuleFilter_Number(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -223,7 +226,7 @@ class EvaluationRuleFilter_StringOptions(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -286,7 +289,7 @@ class EvaluationRuleFilter_CategoryOptions(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -350,7 +353,7 @@ class EvaluationRuleFilter_ArrayOptions(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -413,7 +416,7 @@ class EvaluationRuleFilter_StringObject(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -453,7 +456,7 @@ class EvaluationRuleFilter_StringObject(UniversalBaseModel): type: typing.Literal["stringObject"] = "stringObject" column: str key: str - operator: EvaluationRuleStringFilterOperator + operator: EvaluationRuleStringObjectFilterOperator value: str model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( @@ -477,7 +480,7 @@ class EvaluationRuleFilter_NumberObject(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -541,7 +544,7 @@ class EvaluationRuleFilter_Boolean(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` @@ -604,7 +607,7 @@ class EvaluationRuleFilter_Null(UniversalBaseModel): - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - `stringOptions`: `any of`, `none of` - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` + - `stringObject`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`, `is set`, `is not set` (use `is set` / `is not set` for key presence; an empty value for `contains`, `starts with`, or `ends with` is treated as `is set`) - `boolean`: `"="`, `"<>"` - `null`: `is null`, `is not null` diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_string_object_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_string_object_filter_operator.py new file mode 100644 index 000000000..edf675442 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_string_object_filter_operator.py @@ -0,0 +1,42 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleStringObjectFilterOperator(enum.StrEnum): + EQUALS = "=" + CONTAINS = "contains" + DOES_NOT_CONTAIN = "does not contain" + STARTS_WITH = "starts with" + ENDS_WITH = "ends with" + IS_SET = "is set" + IS_NOT_SET = "is not set" + + def visit( + self, + equals: typing.Callable[[], T_Result], + contains: typing.Callable[[], T_Result], + does_not_contain: typing.Callable[[], T_Result], + starts_with: typing.Callable[[], T_Result], + ends_with: typing.Callable[[], T_Result], + is_set: typing.Callable[[], T_Result], + is_not_set: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleStringObjectFilterOperator.EQUALS: + return equals() + if self is EvaluationRuleStringObjectFilterOperator.CONTAINS: + return contains() + if self is EvaluationRuleStringObjectFilterOperator.DOES_NOT_CONTAIN: + return does_not_contain() + if self is EvaluationRuleStringObjectFilterOperator.STARTS_WITH: + return starts_with() + if self is EvaluationRuleStringObjectFilterOperator.ENDS_WITH: + return ends_with() + if self is EvaluationRuleStringObjectFilterOperator.IS_SET: + return is_set() + if self is EvaluationRuleStringObjectFilterOperator.IS_NOT_SET: + return is_not_set() diff --git a/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py index d291153f0..c0aed0846 100644 --- a/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py +++ b/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py @@ -4,7 +4,9 @@ import pydantic from ...core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator +from .evaluation_rule_string_object_filter_operator import ( + EvaluationRuleStringObjectFilterOperator, +) class StringObjectEvaluationRuleFilter(UniversalBaseModel): @@ -18,8 +20,15 @@ class StringObjectEvaluationRuleFilter(UniversalBaseModel): Top-level key inside the object-valued column to filter on. """ - operator: EvaluationRuleStringFilterOperator - value: str + operator: EvaluationRuleStringObjectFilterOperator = pydantic.Field() + """ + Use `is set` / `is not set` to filter on key presence. + """ + + value: str = pydantic.Field() + """ + Value to compare against. Ignored for `is set` / `is not set`; send `""`. + """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/legacy/observations_v1/client.py b/langfuse/api/legacy/observations_v1/client.py index 871191ee5..46ee7772a 100644 --- a/langfuse/api/legacy/observations_v1/client.py +++ b/langfuse/api/legacy/observations_v1/client.py @@ -150,7 +150,7 @@ def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" @@ -414,7 +414,7 @@ async def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" diff --git a/langfuse/api/legacy/observations_v1/raw_client.py b/langfuse/api/legacy/observations_v1/raw_client.py index 73343c130..ee1792052 100644 --- a/langfuse/api/legacy/observations_v1/raw_client.py +++ b/langfuse/api/legacy/observations_v1/raw_client.py @@ -211,7 +211,7 @@ def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" @@ -588,7 +588,7 @@ async def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" diff --git a/langfuse/api/observations/client.py b/langfuse/api/observations/client.py index 0bab55257..a44bd0707 100644 --- a/langfuse/api/observations/client.py +++ b/langfuse/api/observations/client.py @@ -153,7 +153,7 @@ def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" @@ -437,7 +437,7 @@ async def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" diff --git a/langfuse/api/observations/raw_client.py b/langfuse/api/observations/raw_client.py index a4d6b82e7..24a78cc02 100644 --- a/langfuse/api/observations/raw_client.py +++ b/langfuse/api/observations/raw_client.py @@ -151,7 +151,7 @@ def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" @@ -494,7 +494,7 @@ async def get_many( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - boolean: "=", "<>" // - null: "is null", "is not null" diff --git a/langfuse/api/scores/client.py b/langfuse/api/scores/client.py index d035dc0bd..95f94389a 100644 --- a/langfuse/api/scores/client.py +++ b/langfuse/api/scores/client.py @@ -231,7 +231,7 @@ def get_many( Comma-separated list of field groups to include in the response. Available field groups: 'score' (core score fields), 'trace' (trace properties: userId, tags, environment, sessionId). If not specified, both 'score' and 'trace' are returned by default. Example: 'score' to exclude trace data, 'score,trace' to include both. Note: When filtering by trace properties (using userId or traceTags parameters), the 'trace' field group must be included, otherwise a 400 error will be returned. filter : typing.Optional[str] - A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with. + A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with, is set, is not set. Use is set / is not set to filter on metadata key presence. An empty value for contains, starts with, or ends with is treated as is set (key presence). request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -544,7 +544,7 @@ async def get_many( Comma-separated list of field groups to include in the response. Available field groups: 'score' (core score fields), 'trace' (trace properties: userId, tags, environment, sessionId). If not specified, both 'score' and 'trace' are returned by default. Example: 'score' to exclude trace data, 'score,trace' to include both. Note: When filtering by trace properties (using userId or traceTags parameters), the 'trace' field group must be included, otherwise a 400 error will be returned. filter : typing.Optional[str] - A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with. + A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with, is set, is not set. Use is set / is not set to filter on metadata key presence. An empty value for contains, starts with, or ends with is treated as is set (key presence). request_options : typing.Optional[RequestOptions] Request-specific configuration. diff --git a/langfuse/api/scores/raw_client.py b/langfuse/api/scores/raw_client.py index b87521d7a..e4295f233 100644 --- a/langfuse/api/scores/raw_client.py +++ b/langfuse/api/scores/raw_client.py @@ -297,7 +297,7 @@ def get_many( Comma-separated list of field groups to include in the response. Available field groups: 'score' (core score fields), 'trace' (trace properties: userId, tags, environment, sessionId). If not specified, both 'score' and 'trace' are returned by default. Example: 'score' to exclude trace data, 'score,trace' to include both. Note: When filtering by trace properties (using userId or traceTags parameters), the 'trace' field group must be included, otherwise a 400 error will be returned. filter : typing.Optional[str] - A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with. + A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with, is set, is not set. Use is set / is not set to filter on metadata key presence. An empty value for contains, starts with, or ends with is treated as is set (key presence). request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -789,7 +789,7 @@ async def get_many( Comma-separated list of field groups to include in the response. Available field groups: 'score' (core score fields), 'trace' (trace properties: userId, tags, environment, sessionId). If not specified, both 'score' and 'trace' are returned by default. Example: 'score' to exclude trace data, 'score,trace' to include both. Note: When filtering by trace properties (using userId or traceTags parameters), the 'trace' field group must be included, otherwise a 400 error will be returned. filter : typing.Optional[str] - A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with. + A JSON stringified array of filter objects. Each object requires type, column, operator, and value. Supports filtering by score metadata using the stringObject type. Example: [{"type":"stringObject","column":"metadata","key":"user_id","operator":"=","value":"abc123"}]. Supported types: stringObject (metadata key-value filtering), string, number, datetime, stringOptions, arrayOptions. Supported operators for stringObject: =, contains, does not contain, starts with, ends with, is set, is not set. Use is set / is not set to filter on metadata key presence. An empty value for contains, starts with, or ends with is treated as is set (key presence). request_options : typing.Optional[RequestOptions] Request-specific configuration. diff --git a/langfuse/api/trace/client.py b/langfuse/api/trace/client.py index 2cf76a4a0..5674177c1 100644 --- a/langfuse/api/trace/client.py +++ b/langfuse/api/trace/client.py @@ -198,7 +198,7 @@ def list( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - booleanObject: "=", "<>" // - boolean: "=", "<>" @@ -574,7 +574,7 @@ async def list( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - booleanObject: "=", "<>" // - boolean: "=", "<>" diff --git a/langfuse/api/trace/raw_client.py b/langfuse/api/trace/raw_client.py index 7cb22a519..0797b9919 100644 --- a/langfuse/api/trace/raw_client.py +++ b/langfuse/api/trace/raw_client.py @@ -326,7 +326,7 @@ def list( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - booleanObject: "=", "<>" // - boolean: "=", "<>" @@ -938,7 +938,7 @@ async def list( // - categoryOptions: "any of", "none of" // - arrayOptions: "any of", "none of", "all of" // - number: "=", ">", "<", ">=", "<=" - // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" + // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "is set", "is not set" // - numberObject: "=", ">", "<", ">=", "<=" // - booleanObject: "=", "<>" // - boolean: "=", "<>" From 195face2023e2525c3966f971f137657694a748f Mon Sep 17 00:00:00 2001 From: Niklas Semmler <1880943+niklassemmler@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:00:52 +0200 Subject: [PATCH 2/5] fix(api): type Dataset JSON-schema response fields as objects (#1949) Regenerated langfuse/api from the upstream Fern definition: baseline langfuse/langfuse 619dbe3 (last sync, #1866) plus langfuse/langfuse#17892 and the datasets.yml/opentelemetry.yml part of #18406. Split out of the bundled API-spec bot PR #1943. Co-authored-by: Claude Opus 5.5 (1M context) --- langfuse/api/commons/types/dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/langfuse/api/commons/types/dataset.py b/langfuse/api/commons/types/dataset.py index d312b291a..b7e1d6bfe 100644 --- a/langfuse/api/commons/types/dataset.py +++ b/langfuse/api/commons/types/dataset.py @@ -23,14 +23,16 @@ class Dataset(UniversalBaseModel): """ input_schema: typing_extensions.Annotated[ - typing.Optional[typing.Any], FieldMetadata(alias="inputSchema") + typing.Optional[typing.Dict[str, typing.Any]], + FieldMetadata(alias="inputSchema"), ] = pydantic.Field(default=None) """ JSON Schema for validating dataset item inputs """ expected_output_schema: typing_extensions.Annotated[ - typing.Optional[typing.Any], FieldMetadata(alias="expectedOutputSchema") + typing.Optional[typing.Dict[str, typing.Any]], + FieldMetadata(alias="expectedOutputSchema"), ] = pydantic.Field(default=None) """ JSON Schema for validating dataset item expected outputs From a01aa242ff1fc3880d6d292150c63efb723350b8 Mon Sep 17 00:00:00 2001 From: Niklas Semmler <1880943+niklassemmler@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:05:14 +0200 Subject: [PATCH 3/5] refactor(api): share cursor pagination meta via CursorMetaResponse (#1948) Regenerated langfuse/api from the upstream Fern definition: baseline langfuse/langfuse 619dbe3 (last sync, #1866) plus langfuse/langfuse#17888. Split out of the bundled API-spec bot PR #1943. Co-authored-by: Claude Opus 5.5 (1M context) --- langfuse/api/evaluators/types/cursor_meta.py | 7 +++---- .../types/experiments_response_meta.py | 7 +++---- .../observations/types/observations_v2meta.py | 11 +++------- .../api/scores_v3/types/get_scores_v3meta.py | 8 ++----- langfuse/api/utils/__init__.py | 5 +++-- langfuse/api/utils/pagination/__init__.py | 9 +++++--- .../api/utils/pagination/types/__init__.py | 8 +++++-- .../pagination/types/cursor_meta_response.py | 21 +++++++++++++++++++ 8 files changed, 47 insertions(+), 29 deletions(-) create mode 100644 langfuse/api/utils/pagination/types/cursor_meta_response.py diff --git a/langfuse/api/evaluators/types/cursor_meta.py b/langfuse/api/evaluators/types/cursor_meta.py index 3ee6e1a0a..7e4c64a67 100644 --- a/langfuse/api/evaluators/types/cursor_meta.py +++ b/langfuse/api/evaluators/types/cursor_meta.py @@ -3,13 +3,12 @@ import typing import pydantic -from ...core.pydantic_utilities import UniversalBaseModel +from ...utils.pagination.types.cursor_meta_response import CursorMetaResponse -class CursorMeta(UniversalBaseModel): - cursor: typing.Optional[str] = pydantic.Field(default=None) +class CursorMeta(CursorMetaResponse): """ - Opaque cursor for the next page. Omitted when there is no next page. + Cursor pagination metadata. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/experiments/types/experiments_response_meta.py b/langfuse/api/experiments/types/experiments_response_meta.py index 038bcc45d..61f2de4ea 100644 --- a/langfuse/api/experiments/types/experiments_response_meta.py +++ b/langfuse/api/experiments/types/experiments_response_meta.py @@ -3,13 +3,12 @@ import typing import pydantic -from ...core.pydantic_utilities import UniversalBaseModel +from ...utils.pagination.types.cursor_meta_response import CursorMetaResponse -class ExperimentsResponseMeta(UniversalBaseModel): - cursor: typing.Optional[str] = pydantic.Field(default=None) +class ExperimentsResponseMeta(CursorMetaResponse): """ - Versioned base64url cursor for retrieving the next page. Absent when there are no more results. + Metadata for cursor-based pagination. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/observations/types/observations_v2meta.py b/langfuse/api/observations/types/observations_v2meta.py index 8f86a6512..46d9f42aa 100644 --- a/langfuse/api/observations/types/observations_v2meta.py +++ b/langfuse/api/observations/types/observations_v2meta.py @@ -3,17 +3,12 @@ import typing import pydantic -from ...core.pydantic_utilities import UniversalBaseModel +from ...utils.pagination.types.cursor_meta_response import CursorMetaResponse -class ObservationsV2Meta(UniversalBaseModel): +class ObservationsV2Meta(CursorMetaResponse): """ - Metadata for cursor-based pagination - """ - - cursor: typing.Optional[str] = pydantic.Field(default=None) - """ - Base64-encoded cursor to use for retrieving the next page. If not present, there are no more results. + Metadata for cursor-based pagination. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/scores_v3/types/get_scores_v3meta.py b/langfuse/api/scores_v3/types/get_scores_v3meta.py index 7dfcfe0e1..832793364 100644 --- a/langfuse/api/scores_v3/types/get_scores_v3meta.py +++ b/langfuse/api/scores_v3/types/get_scores_v3meta.py @@ -3,15 +3,11 @@ import typing import pydantic -from ...core.pydantic_utilities import UniversalBaseModel +from ...utils.pagination.types.cursor_meta_response import CursorMetaResponse -class GetScoresV3Meta(UniversalBaseModel): +class GetScoresV3Meta(CursorMetaResponse): limit: int - cursor: typing.Optional[str] = pydantic.Field(default=None) - """ - URL-safe base64 (base64url) cursor for the next page. Absent when there are no more results. - """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/utils/__init__.py b/langfuse/api/utils/__init__.py index b272f64b5..1690ee475 100644 --- a/langfuse/api/utils/__init__.py +++ b/langfuse/api/utils/__init__.py @@ -7,8 +7,9 @@ if typing.TYPE_CHECKING: from . import pagination - from .pagination import MetaResponse + from .pagination import CursorMetaResponse, MetaResponse _dynamic_imports: typing.Dict[str, str] = { + "CursorMetaResponse": ".pagination", "MetaResponse": ".pagination", "pagination": ".pagination", } @@ -41,4 +42,4 @@ def __dir__(): return sorted(lazy_attrs) -__all__ = ["MetaResponse", "pagination"] +__all__ = ["CursorMetaResponse", "MetaResponse", "pagination"] diff --git a/langfuse/api/utils/pagination/__init__.py b/langfuse/api/utils/pagination/__init__.py index 50821832d..59cdf0321 100644 --- a/langfuse/api/utils/pagination/__init__.py +++ b/langfuse/api/utils/pagination/__init__.py @@ -6,8 +6,11 @@ from importlib import import_module if typing.TYPE_CHECKING: - from .types import MetaResponse -_dynamic_imports: typing.Dict[str, str] = {"MetaResponse": ".types"} + from .types import CursorMetaResponse, MetaResponse +_dynamic_imports: typing.Dict[str, str] = { + "CursorMetaResponse": ".types", + "MetaResponse": ".types", +} def __getattr__(attr_name: str) -> typing.Any: @@ -37,4 +40,4 @@ def __dir__(): return sorted(lazy_attrs) -__all__ = ["MetaResponse"] +__all__ = ["CursorMetaResponse", "MetaResponse"] diff --git a/langfuse/api/utils/pagination/types/__init__.py b/langfuse/api/utils/pagination/types/__init__.py index 5c0d83028..85d13aee0 100644 --- a/langfuse/api/utils/pagination/types/__init__.py +++ b/langfuse/api/utils/pagination/types/__init__.py @@ -6,8 +6,12 @@ from importlib import import_module if typing.TYPE_CHECKING: + from .cursor_meta_response import CursorMetaResponse from .meta_response import MetaResponse -_dynamic_imports: typing.Dict[str, str] = {"MetaResponse": ".meta_response"} +_dynamic_imports: typing.Dict[str, str] = { + "CursorMetaResponse": ".cursor_meta_response", + "MetaResponse": ".meta_response", +} def __getattr__(attr_name: str) -> typing.Any: @@ -37,4 +41,4 @@ def __dir__(): return sorted(lazy_attrs) -__all__ = ["MetaResponse"] +__all__ = ["CursorMetaResponse", "MetaResponse"] diff --git a/langfuse/api/utils/pagination/types/cursor_meta_response.py b/langfuse/api/utils/pagination/types/cursor_meta_response.py new file mode 100644 index 000000000..eb30740ac --- /dev/null +++ b/langfuse/api/utils/pagination/types/cursor_meta_response.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ....core.pydantic_utilities import UniversalBaseModel + + +class CursorMetaResponse(UniversalBaseModel): + """ + Metadata for cursor-based pagination. + """ + + cursor: typing.Optional[str] = pydantic.Field(default=None) + """ + Opaque cursor for the next page. Omitted when there is no next page. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) From a4cbc213f667cee8b24de437d5e0370747bc5679 Mon Sep 17 00:00:00 2001 From: Niklas Semmler <1880943+niklassemmler@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:10:09 +0200 Subject: [PATCH 4/5] docs(api): document fromStartTime as required on experiments endpoints (#1947) * docs(api): document fromStartTime as required on experiments endpoints Regenerated langfuse/api from the upstream Fern definition: baseline langfuse/langfuse 619dbe3 (last sync, #1866) plus langfuse/langfuse#17603. Split out of the bundled API-spec bot PR #1943. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(api): use language-neutral wording for experiments time window Regenerated from langfuse/langfuse branch docs/experiments-from-start-time-neutral-wording. The fromStartTime docstrings named `toStartTime`, but the Python parameter is `to_start_time`; they now say "an upper bound" instead. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- langfuse/api/experiments/client.py | 20 ++++++++++++++++---- langfuse/api/experiments/raw_client.py | 20 ++++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/langfuse/api/experiments/client.py b/langfuse/api/experiments/client.py index a3d469217..66a90d95b 100644 --- a/langfuse/api/experiments/client.py +++ b/langfuse/api/experiments/client.py @@ -47,7 +47,10 @@ def list( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiments on or after this datetime. + Required. Retrieve only experiments with events on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -145,7 +148,10 @@ def list_items( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiment items started on or after this datetime. + Required. Retrieve only experiment items started on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -263,7 +269,10 @@ async def list( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiments on or after this datetime. + Required. Retrieve only experiments with events on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -368,7 +377,10 @@ async def list_items( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiment items started on or after this datetime. + Required. Retrieve only experiment items started on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: diff --git a/langfuse/api/experiments/raw_client.py b/langfuse/api/experiments/raw_client.py index 8736786df..189c826ba 100644 --- a/langfuse/api/experiments/raw_client.py +++ b/langfuse/api/experiments/raw_client.py @@ -45,7 +45,10 @@ def list( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiments on or after this datetime. + Required. Retrieve only experiments with events on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -205,7 +208,10 @@ def list_items( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiment items started on or after this datetime. + Required. Retrieve only experiment items started on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -374,7 +380,10 @@ async def list( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiments on or after this datetime. + Required. Retrieve only experiments with events on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: @@ -534,7 +543,10 @@ async def list_items( Parameters ---------- from_start_time : dt.datetime - Retrieve only experiment items started on or after this datetime. + Required. Retrieve only experiment items started on or after this + datetime. A lower bound is required so the query stays fast on + large projects; pair it with an upper bound to select a specific + window. fields : typing.Optional[str] Comma-separated list of field groups to include. Available groups: From bf11ec121479145923ff27b32ec2ad1d06fd2d51 Mon Sep 17 00:00:00 2001 From: Niklas Semmler <1880943+niklassemmler@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:14:50 +0200 Subject: [PATCH 5/5] fix(api): type read-side metadata as optional objects (#1950) * fix(api): type read-side metadata as optional objects Regenerated langfuse/api from the upstream Fern definition: baseline langfuse/langfuse 619dbe3 (last sync, #1866) plus langfuse/langfuse#18122 and the commons.yml part of #18406. Split out of the bundled API-spec bot PR #1943. Co-Authored-By: Claude Opus 5.5 (1M context) * test(api): pin read-side metadata parsing on generated models Cover omitted, null, empty and nested-JSON metadata plus rejection of non-object metadata for Trace, Observation, ObservationV2, BaseScore, BaseScoreV1 and every Score / ScoreV1 / GetScoresResponseData variant. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- langfuse/api/commons/types/base_score.py | 4 +- langfuse/api/commons/types/base_score_v1.py | 4 +- langfuse/api/commons/types/observation.py | 4 +- langfuse/api/commons/types/observation_v2.py | 4 +- langfuse/api/commons/types/score.py | 10 +- langfuse/api/commons/types/score_v1.py | 8 +- langfuse/api/commons/types/trace.py | 6 +- .../scores/types/get_scores_response_data.py | 10 +- tests/unit/test_api_metadata_parsing.py | 166 ++++++++++++++++++ 9 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 tests/unit/test_api_metadata_parsing.py diff --git a/langfuse/api/commons/types/base_score.py b/langfuse/api/commons/types/base_score.py index 44e09033c..f46240dae 100644 --- a/langfuse/api/commons/types/base_score.py +++ b/langfuse/api/commons/types/base_score.py @@ -61,7 +61,9 @@ class BaseScore(UniversalBaseModel): Comment on the score """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Metadata associated with the score """ diff --git a/langfuse/api/commons/types/base_score_v1.py b/langfuse/api/commons/types/base_score_v1.py index 881b10a3b..13845d997 100644 --- a/langfuse/api/commons/types/base_score_v1.py +++ b/langfuse/api/commons/types/base_score_v1.py @@ -41,7 +41,9 @@ class BaseScoreV1(UniversalBaseModel): Comment on the score """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Metadata associated with the score """ diff --git a/langfuse/api/commons/types/observation.py b/langfuse/api/commons/types/observation.py index 91bd50643..76c019b0d 100644 --- a/langfuse/api/commons/types/observation.py +++ b/langfuse/api/commons/types/observation.py @@ -77,7 +77,9 @@ class Observation(UniversalBaseModel): The version of the observation """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Additional metadata of the observation """ diff --git a/langfuse/api/commons/types/observation_v2.py b/langfuse/api/commons/types/observation_v2.py index f034d8ace..9c6d9e3e6 100644 --- a/langfuse/api/commons/types/observation_v2.py +++ b/langfuse/api/commons/types/observation_v2.py @@ -152,7 +152,9 @@ class ObservationV2(UniversalBaseModel): The output data of the observation """ - metadata: typing.Optional[typing.Any] = pydantic.Field(default=None) + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Additional metadata of the observation """ diff --git a/langfuse/api/commons/types/score.py b/langfuse/api/commons/types/score.py index 7b31ae39a..e0e98c9ae 100644 --- a/langfuse/api/commons/types/score.py +++ b/langfuse/api/commons/types/score.py @@ -54,7 +54,7 @@ class Score_Numeric(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -100,7 +100,7 @@ class Score_Categorical(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -146,7 +146,7 @@ class Score_Boolean(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -192,7 +192,7 @@ class Score_Correction(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -237,7 +237,7 @@ class Score_Text(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/langfuse/api/commons/types/score_v1.py b/langfuse/api/commons/types/score_v1.py index e17e61b89..32e0a6b4e 100644 --- a/langfuse/api/commons/types/score_v1.py +++ b/langfuse/api/commons/types/score_v1.py @@ -35,7 +35,7 @@ class ScoreV1_Numeric(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -73,7 +73,7 @@ class ScoreV1_Categorical(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -111,7 +111,7 @@ class ScoreV1_Boolean(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -148,7 +148,7 @@ class ScoreV1_Text(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/langfuse/api/commons/types/trace.py b/langfuse/api/commons/types/trace.py index 6eafa9fcd..1b489c03c 100644 --- a/langfuse/api/commons/types/trace.py +++ b/langfuse/api/commons/types/trace.py @@ -59,9 +59,11 @@ class Trace(UniversalBaseModel): The user identifier associated with the trace """ - metadata: typing.Optional[typing.Any] = pydantic.Field(default=None) + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ - The metadata associated with the trace. Can be any JSON. + The metadata associated with the trace. Values can be any JSON; non-object metadata sent at ingestion is returned under the `metadata` key. """ tags: typing.List[str] = pydantic.Field() diff --git a/langfuse/api/scores/types/get_scores_response_data.py b/langfuse/api/scores/types/get_scores_response_data.py index f85cd471c..84b8abcc4 100644 --- a/langfuse/api/scores/types/get_scores_response_data.py +++ b/langfuse/api/scores/types/get_scores_response_data.py @@ -45,7 +45,7 @@ class GetScoresResponseData_Numeric(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -92,7 +92,7 @@ class GetScoresResponseData_Categorical(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -139,7 +139,7 @@ class GetScoresResponseData_Boolean(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -186,7 +186,7 @@ class GetScoresResponseData_Correction(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -232,7 +232,7 @@ class GetScoresResponseData_Text(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/tests/unit/test_api_metadata_parsing.py b/tests/unit/test_api_metadata_parsing.py new file mode 100644 index 000000000..6be15dfba --- /dev/null +++ b/tests/unit/test_api_metadata_parsing.py @@ -0,0 +1,166 @@ +"""Response parsing of read-side ``metadata`` on generated API models. + +The server returns read-side metadata as a JSON object, ``null``, or omits it. +These tests pin that contract so a future regeneration that makes the field +required or narrows its values is caught. +""" + +import typing + +import pydantic +import pytest + +from langfuse.api import ( + BaseScore, + BaseScoreV1, + GetScoresResponseData, + Observation, + ObservationV2, + Score, + ScoreV1, + Trace, +) +from langfuse.api.core import parse_obj_as + +TIMESTAMP = "2026-01-01T00:00:00.000Z" + +SCORE_BASE: typing.Dict[str, typing.Any] = { + "id": "score-1", + "name": "quality", + "source": "API", + "timestamp": TIMESTAMP, + "createdAt": TIMESTAMP, + "updatedAt": TIMESTAMP, + "environment": "default", +} +SCORE_V1_BASE = {**SCORE_BASE, "traceId": "trace-1"} + +SCORE_VARIANTS: typing.Dict[str, typing.Dict[str, typing.Any]] = { + "NUMERIC": {"value": 0.5}, + "CATEGORICAL": {"value": 1, "stringValue": "good"}, + "BOOLEAN": {"value": 1, "stringValue": "True"}, + "CORRECTION": {"value": 0, "stringValue": "corrected output"}, + "TEXT": {"stringValue": "free text"}, +} +SCORE_V1_DATA_TYPES = ["NUMERIC", "CATEGORICAL", "BOOLEAN", "TEXT"] + + +def _score_payload( + base: typing.Dict[str, typing.Any], data_type: str +) -> typing.Dict[str, typing.Any]: + return {**base, "dataType": data_type, **SCORE_VARIANTS[data_type]} + + +CASES: typing.List[typing.Any] = [ + pytest.param( + Trace, + { + "id": "trace-1", + "timestamp": TIMESTAMP, + "tags": [], + "public": False, + "environment": "default", + }, + id="Trace", + ), + pytest.param( + Observation, + { + "id": "obs-1", + "type": "SPAN", + "startTime": TIMESTAMP, + "modelParameters": {}, + "input": None, + "output": None, + "usage": {"input": 0, "output": 0, "total": 0}, + "level": "DEFAULT", + "usageDetails": {}, + "costDetails": {}, + "environment": "default", + }, + id="Observation", + ), + pytest.param( + ObservationV2, + { + "id": "obs-1", + "startTime": TIMESTAMP, + "projectId": "project-1", + "type": "SPAN", + }, + id="ObservationV2", + ), + pytest.param(BaseScore, SCORE_BASE, id="BaseScore"), + pytest.param(BaseScoreV1, SCORE_V1_BASE, id="BaseScoreV1"), + *[ + pytest.param(Score, _score_payload(SCORE_BASE, dt), id=f"Score-{dt}") + for dt in SCORE_VARIANTS + ], + *[ + pytest.param( + GetScoresResponseData, + _score_payload(SCORE_BASE, dt), + id=f"GetScoresResponseData-{dt}", + ) + for dt in SCORE_VARIANTS + ], + *[ + pytest.param(ScoreV1, _score_payload(SCORE_V1_BASE, dt), id=f"ScoreV1-{dt}") + for dt in SCORE_V1_DATA_TYPES + ], +] + +NESTED_METADATA = { + "str": "value", + "int": 1, + "float": 1.5, + "bool": True, + "null": None, + "list": [1, "two", {"three": 3}], + "nested": {"deeper": {"key": ["a", "b"]}}, +} + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_omitted_parses_as_none(type_, payload): + assert "metadata" not in payload + + parsed = parse_obj_as(type_, payload) + + assert parsed.metadata is None + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_null_parses_as_none(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": None}) + + assert parsed.metadata is None + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_empty_object_parses(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": {}}) + + assert parsed.metadata == {} + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_object_with_nested_json_values_parses(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": NESTED_METADATA}) + + assert parsed.metadata == NESTED_METADATA + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +@pytest.mark.parametrize( + "metadata", + ["a string", 1, 1.5, True, ["a", "list"]], + ids=["str", "int", "float", "bool", "list"], +) +def test_metadata_non_object_is_rejected(type_, payload, metadata): + with pytest.raises(pydantic.ValidationError) as exc_info: + parse_obj_as(type_, {**payload, "metadata": metadata}) + + # The discriminated unions prefix the location with the variant tag. + assert exc_info.value.errors() + assert all("metadata" in error["loc"] for error in exc_info.value.errors())