diff --git a/openapi/README.md b/openapi/README.md index 0420ffb4..bafcaac4 100644 --- a/openapi/README.md +++ b/openapi/README.md @@ -1,42 +1,46 @@ # Pinned Braintrust OpenAPI specification -`spec.json` is a committed snapshot of the public specification from +`spec.json` is a committed snapshot of [`braintrustdata/braintrust-openapi`](https://github.com/braintrustdata/braintrust-openapi). -`config.json` pins the full upstream commit, snapshot SHA-256, generator tools, generator flags, and -selected endpoint tags. The generator scripts live in `py/scripts/`. Builds and package installation -use the committed generated source and never fetch or run code generation. +`config.json` pins the upstream commit, snapshot hash, generator versions and flags, selected endpoint +tags, and retry-policy allowlists. Builds use committed generated source and never fetch the spec or +run code generation. -From `py/`, validate and regenerate the private models offline with: +## Generate and check + +Run from `py/`: ```bash make generate-api-client make check-api-client-codegen ``` -The check regenerates into a temporary directory and does not modify the worktree. Endpoint bindings -are rolled out explicitly through `endpoint_generator.generated_tags`. The current rollout supports -the Projects and Experiments tags and emits one operation registry/resource class per tag. Reachable -models used by one resource live in that resource's model module; models shared by multiple resources -live once in `models/common.py` and are imported explicitly. Unreachable models are omitted. Public -resource method and inline response type names are derived mechanically from each `operationId`, and -generated methods forward request fields and parameters without implicit defaults. Logical POST reads -that are safe to retry are listed in `endpoint_generator.safe_reads`, while verified idempotent writes -are listed in `endpoint_generator.idempotent_writes`; GET/HEAD reads and all other writes use -mechanical retry defaults. +The check regenerates in a temporary directory and reports drift without changing the worktree. +Currently selected tags are Projects, Experiments, and Datasets. Each tag produces one resource and +operation registry. Models used by one resource stay in that resource's model module; shared models +live in `models/common.py`; unreachable models are omitted. + +Method and inline-response names come directly from normalized OpenAPI `operationId` values. Generated +models preserve exact wire keys, including leading underscores, and methods do not add implicit request +defaults. GET and HEAD operations use the safe-read retry policy. +Logical POST reads and verified idempotent writes must be listed explicitly in `safe_reads` and +`idempotent_writes`; all other writes are non-retrying. + +## Refresh the snapshot -To fetch the configured upstream commit explicitly: +Fetch the configured upstream commit: ```bash make fetch-openapi-spec ``` -For an existing local checkout, set `BRAINTRUST_OPENAPI_ROOT` to its root. The checkout must be at the -commit pinned in `config.json`, and its specification must have the pinned hash: +To fetch from a local checkout instead: ```bash BRAINTRUST_OPENAPI_ROOT=../../braintrust-openapi make fetch-openapi-spec ``` -To update the snapshot, first update the commit and SHA-256 in `config.json`, then fetch, regenerate, -and review both the upstream spec diff and generated model diff. Only operations selected through -`endpoint_generator.generated_tags` and their reachable schemas are validated and generated. +The checkout must be at the commit pinned in `config.json`, and its spec must match the pinned hash. +To update the snapshot, update the commit and hash in `config.json`, fetch, regenerate, and review both +the upstream spec diff and generated-source diff. Validation and generation apply only to selected tags +and their transitively reachable schemas. diff --git a/openapi/config.json b/openapi/config.json index 877be3a4..df785e65 100644 --- a/openapi/config.json +++ b/openapi/config.json @@ -31,13 +31,16 @@ "schema_version": 1, "generated_tags": [ "Projects", - "Experiments" + "Experiments", + "Datasets" ], "safe_reads": [ - "postExperimentIdFetch" + "postExperimentIdFetch", + "postDatasetIdFetch" ], "idempotent_writes": [ - "postProject" + "postProject", + "postDataset" ], "supported_success_statuses": [ "200", diff --git a/py/scripts/openapi_codegen.py b/py/scripts/openapi_codegen.py index f25a4f69..2d096230 100644 --- a/py/scripts/openapi_codegen.py +++ b/py/scripts/openapi_codegen.py @@ -508,9 +508,30 @@ def _prune_empty_directories(root: Path) -> None: shutil.rmtree(directory) +def _leading_underscore_field_aliases(value: Any) -> Dict[str, str]: + aliases: Dict[str, str] = {} + + def visit(node: Any) -> None: + if isinstance(node, dict): + properties = node.get("properties") + if isinstance(properties, dict): + for name in properties: + if isinstance(name, str) and name.startswith("_") and name.isidentifier(): + aliases[name] = name + for child in node.values(): + visit(child) + elif isinstance(node, list): + for child in node: + visit(child) + + visit(value) + return dict(sorted(aliases.items())) + + def _generate_models(spec_path: Path, output_path: Path, config: Mapping[str, Any]) -> None: output_path.parent.mkdir(parents=True, exist_ok=True) header = _generated_header(config, "CONTENT_HASH_PLACEHOLDER").rstrip() + aliases = _leading_underscore_field_aliases(json.loads(spec_path.read_text(encoding="utf-8"))) command = [ sys.executable, "-m", @@ -520,6 +541,7 @@ def _generate_models(spec_path: Path, output_path: Path, config: Mapping[str, An "--output", str(output_path), *_model_flags(config), + *(["--aliases", json.dumps(aliases, sort_keys=True)] if aliases else []), "--custom-file-header", header, ] diff --git a/py/src/braintrust/api/_generated/datasets.py b/py/src/braintrust/api/_generated/datasets.py new file mode 100644 index 00000000..00c6a46e --- /dev/null +++ b/py/src/braintrust/api/_generated/datasets.py @@ -0,0 +1,480 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: 39bb40ebc26c57f57f09d2a309b8bb6702b737b0d6dbd07a1faca97e7ea5be56 + +"""Generated Datasets REST operations and resource.""" + +from typing import cast + +from .._service import Operation, Parameter, ResourceAPI +from ..policies import RetryMode +from .models.common import ( + AppLimitParam, + EndingBefore, + FeedbackResponseSchema, + FetchEventsRequest, + FetchLimitParam, + Ids, + InsertEventsResponse, + MaxRootSpanId, + MaxXactId, + OrgName, + ProjectIdQuery, + ProjectName, + StartingAfter, + Version, +) +from .models.datasets import ( + CreateDataset, + Dataset, + DatasetIdParam, + DatasetName, + FeedbackDatasetEventRequest, + FetchDatasetEventsResponse, + GetDatasetResponse, + InsertDatasetEventRequest, + PatchDataset, + SummarizeData, + SummarizeDatasetResponse, +) + + +POST_DATASET = Operation( + operation_id="postDataset", + method="POST", + path="/v1/dataset", + parameters=(), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.IDEMPOTENT_WRITE, +) + + +GET_DATASET = Operation( + operation_id="getDataset", + method="GET", + path="/v1/dataset", + parameters=( + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="starting_after", + name="starting_after", + location="query", + required=False, + ), + Parameter( + argument_name="ending_before", + name="ending_before", + location="query", + required=False, + ), + Parameter( + argument_name="ids", + name="ids", + location="query", + required=False, + ), + Parameter( + argument_name="dataset_name", + name="dataset_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_name", + name="project_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_id", + name="project_id", + location="query", + required=False, + ), + Parameter( + argument_name="org_name", + name="org_name", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_DATASET_ID = Operation( + operation_id="getDatasetId", + method="GET", + path="/v1/dataset/{dataset_id}", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +PATCH_DATASET_ID = Operation( + operation_id="patchDatasetId", + method="PATCH", + path="/v1/dataset/{dataset_id}", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +DELETE_DATASET_ID = Operation( + operation_id="deleteDatasetId", + method="DELETE", + path="/v1/dataset/{dataset_id}", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +POST_DATASET_ID_INSERT = Operation( + operation_id="postDatasetIdInsert", + method="POST", + path="/v1/dataset/{dataset_id}/insert", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +POST_DATASET_ID_FETCH = Operation( + operation_id="postDatasetIdFetch", + method="POST", + path="/v1/dataset/{dataset_id}/fetch", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_DATASET_ID_FETCH = Operation( + operation_id="getDatasetIdFetch", + method="GET", + path="/v1/dataset/{dataset_id}/fetch", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="max_xact_id", + name="max_xact_id", + location="query", + required=False, + ), + Parameter( + argument_name="max_root_span_id", + name="max_root_span_id", + location="query", + required=False, + ), + Parameter( + argument_name="version", + name="version", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +POST_DATASET_ID_FEEDBACK = Operation( + operation_id="postDatasetIdFeedback", + method="POST", + path="/v1/dataset/{dataset_id}/feedback", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +GET_DATASET_ID_SUMMARIZE = Operation( + operation_id="getDatasetIdSummarize", + method="GET", + path="/v1/dataset/{dataset_id}/summarize", + parameters=( + Parameter( + argument_name="dataset_id", + name="dataset_id", + location="path", + required=True, + ), + Parameter( + argument_name="summarize_data", + name="summarize_data", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +OPERATIONS = { + "postDataset": POST_DATASET, + "getDataset": GET_DATASET, + "getDatasetId": GET_DATASET_ID, + "patchDatasetId": PATCH_DATASET_ID, + "deleteDatasetId": DELETE_DATASET_ID, + "postDatasetIdInsert": POST_DATASET_ID_INSERT, + "postDatasetIdFetch": POST_DATASET_ID_FETCH, + "getDatasetIdFetch": GET_DATASET_ID_FETCH, + "postDatasetIdFeedback": POST_DATASET_ID_FEEDBACK, + "getDatasetIdSummarize": GET_DATASET_ID_SUMMARIZE, +} + + +class DatasetsAPI(ResourceAPI): + """Generated Datasets REST API.""" + + def post_dataset( + self, + *, + body: "CreateDataset | None" = None, + ) -> "Dataset": + return cast( + "Dataset", + self.execute( + POST_DATASET, + body=body, + ), + ) + + def get_dataset( + self, + *, + limit: "AppLimitParam | None" = None, + starting_after: "StartingAfter | None" = None, + ending_before: "EndingBefore | None" = None, + ids: "Ids | None" = None, + dataset_name: "DatasetName | None" = None, + project_name: "ProjectName | None" = None, + project_id: "ProjectIdQuery | None" = None, + org_name: "OrgName | None" = None, + ) -> "GetDatasetResponse": + return cast( + "GetDatasetResponse", + self.execute( + GET_DATASET, + query_parameters={ + "limit": limit, + "starting_after": starting_after, + "ending_before": ending_before, + "ids": ids, + "dataset_name": dataset_name, + "project_name": project_name, + "project_id": project_id, + "org_name": org_name, + }, + ), + ) + + def get_dataset_id( + self, + dataset_id: "DatasetIdParam", + ) -> "Dataset": + return cast( + "Dataset", + self.execute( + GET_DATASET_ID, + path_parameters={"dataset_id": dataset_id}, + ), + ) + + def patch_dataset_id( + self, + dataset_id: "DatasetIdParam", + *, + body: "PatchDataset | None" = None, + ) -> "Dataset": + return cast( + "Dataset", + self.execute( + PATCH_DATASET_ID, + path_parameters={"dataset_id": dataset_id}, + body=body, + ), + ) + + def delete_dataset_id( + self, + dataset_id: "DatasetIdParam", + ) -> "Dataset": + return cast( + "Dataset", + self.execute( + DELETE_DATASET_ID, + path_parameters={"dataset_id": dataset_id}, + ), + ) + + def post_dataset_id_insert( + self, + dataset_id: "DatasetIdParam", + *, + body: "InsertDatasetEventRequest | None" = None, + ) -> "InsertEventsResponse": + return cast( + "InsertEventsResponse", + self.execute( + POST_DATASET_ID_INSERT, + path_parameters={"dataset_id": dataset_id}, + body=body, + ), + ) + + def post_dataset_id_fetch( + self, + dataset_id: "DatasetIdParam", + *, + body: "FetchEventsRequest | None" = None, + ) -> "FetchDatasetEventsResponse": + return cast( + "FetchDatasetEventsResponse", + self.execute( + POST_DATASET_ID_FETCH, + path_parameters={"dataset_id": dataset_id}, + body=body, + ), + ) + + def get_dataset_id_fetch( + self, + dataset_id: "DatasetIdParam", + *, + limit: "FetchLimitParam | None" = None, + max_xact_id: "MaxXactId | None" = None, + max_root_span_id: "MaxRootSpanId | None" = None, + version: "Version | None" = None, + ) -> "FetchDatasetEventsResponse": + return cast( + "FetchDatasetEventsResponse", + self.execute( + GET_DATASET_ID_FETCH, + path_parameters={"dataset_id": dataset_id}, + query_parameters={ + "limit": limit, + "max_xact_id": max_xact_id, + "max_root_span_id": max_root_span_id, + "version": version, + }, + ), + ) + + def post_dataset_id_feedback( + self, + dataset_id: "DatasetIdParam", + *, + body: "FeedbackDatasetEventRequest | None" = None, + ) -> "FeedbackResponseSchema": + return cast( + "FeedbackResponseSchema", + self.execute( + POST_DATASET_ID_FEEDBACK, + path_parameters={"dataset_id": dataset_id}, + body=body, + ), + ) + + def get_dataset_id_summarize( + self, + dataset_id: "DatasetIdParam", + *, + summarize_data: "SummarizeData | None" = None, + ) -> "SummarizeDatasetResponse": + return cast( + "SummarizeDatasetResponse", + self.execute( + GET_DATASET_ID_SUMMARIZE, + path_parameters={"dataset_id": dataset_id}, + query_parameters={"summarize_data": summarize_data}, + ), + ) diff --git a/py/src/braintrust/api/_generated/experiments.py b/py/src/braintrust/api/_generated/experiments.py index 1462cc6e..7623e5c2 100644 --- a/py/src/braintrust/api/_generated/experiments.py +++ b/py/src/braintrust/api/_generated/experiments.py @@ -4,7 +4,7 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: c3bcd5de70f3b9110426fd4557e7c056637cfda9f219ef6c18f23b17ee66b6c0 +# Content SHA-256: 539406e3c181decdc8f29755e5b60d79b61798b27371fa34cd905f1542671985 """Generated Experiments REST operations and resource.""" @@ -12,7 +12,21 @@ from .._service import Operation, Parameter, ResourceAPI from ..policies import RetryMode -from .models.common import EndingBefore, Ids, OrgName, ProjectName, StartingAfter +from .models.common import ( + EndingBefore, + FeedbackResponseSchema, + FetchEventsRequest, + FetchLimitParam, + Ids, + InsertEventsResponse, + MaxRootSpanId, + MaxXactId, + OrgName, + ProjectIdQuery, + ProjectName, + StartingAfter, + Version, +) from .models.experiments import ( AppLimitWithDefaultParam, ComparisonExperimentId, @@ -21,20 +35,12 @@ ExperimentIdParam, ExperimentName, FeedbackExperimentEventRequest, - FeedbackResponseSchema, - FetchEventsRequest, FetchExperimentEventsResponse, - FetchLimitParam, GetExperimentResponse, - InsertEventsResponse, InsertExperimentEventRequest, - MaxRootSpanId, - MaxXactId, PatchExperiment, - ProjectIdQuery, SummarizeExperimentResponse, SummarizeScores, - Version, ) diff --git a/py/src/braintrust/api/_generated/models/__init__.py b/py/src/braintrust/api/_generated/models/__init__.py index 4c917afb..20fffd21 100644 --- a/py/src/braintrust/api/_generated/models/__init__.py +++ b/py/src/braintrust/api/_generated/models/__init__.py @@ -4,14 +4,55 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: ad43048e1178347f0d1c31f64d9886cb6e0d91c2e6f901e8d0a3a7f08e8348af +# Content SHA-256: 1ac0980e6a11327b27716b8975b8953969d1c95dd4ed5ac4b9d8118258a9f4bc """Generated private model types with stable package-level imports.""" -from .common import EndingBefore, FunctionTypeEnum, Ids, OrgName, ProjectName, StartingAfter +from .common import ( + AppLimitParam, + Classification, + EndingBefore, + FeedbackResponseSchema, + FetchEventsRequest, + FetchLimit, + FetchLimitParam, + FetchPaginationCursor, + FieldArrayDeleteItem, + FunctionTypeEnum, + Ids, + InsertEventsResponse, + MaxRootSpanId, + MaxXactId, + Metadata, + ObjectReferenceNullish, + OrgName, + ProjectIdQuery, + ProjectName, + SavedFunctionId, + SavedFunctionId1, + SavedFunctionId2, + StartingAfter, + Version, +) +from .datasets import ( + CreateDataset, + DataSummary, + Dataset, + DatasetEvent, + DatasetIdParam, + DatasetName, + FeedbackDatasetEventRequest, + FeedbackDatasetItem, + FetchDatasetEventsResponse, + GetDatasetResponse, + InsertDatasetEvent, + InsertDatasetEventRequest, + PatchDataset, + SummarizeData, + SummarizeDatasetResponse, +) from .experiments import ( AppLimitWithDefaultParam, - Classification, ComparisonExperimentId, Context, CreateExperiment, @@ -21,39 +62,22 @@ ExperimentName, FeedbackExperimentEventRequest, FeedbackExperimentItem, - FeedbackResponseSchema, - FetchEventsRequest, FetchExperimentEventsResponse, - FetchLimit, - FetchLimitParam, - FetchPaginationCursor, - FieldArrayDeleteItem, GetExperimentResponse, - InsertEventsResponse, InsertExperimentEvent, InsertExperimentEventRequest, InternalMetadata, - MaxRootSpanId, - MaxXactId, - Metadata, MetricSummary, Metrics, - ObjectReferenceNullish, PatchExperiment, - ProjectIdQuery, RepoInfo, - SavedFunctionId, - SavedFunctionId1, - SavedFunctionId2, ScoreSummary, SpanAttributes, SpanType, SummarizeExperimentResponse, SummarizeScores, - Version, ) from .projects import ( - AppLimitParam, CreateProject, GetProjectResponse, NullableSavedFunctionId, @@ -74,16 +98,25 @@ "Classification", "ComparisonExperimentId", "Context", + "CreateDataset", "CreateExperiment", "CreateProject", + "DataSummary", + "Dataset", + "DatasetEvent", + "DatasetIdParam", + "DatasetName", "EndingBefore", "Experiment", "ExperimentEvent", "ExperimentIdParam", "ExperimentName", + "FeedbackDatasetEventRequest", + "FeedbackDatasetItem", "FeedbackExperimentEventRequest", "FeedbackExperimentItem", "FeedbackResponseSchema", + "FetchDatasetEventsResponse", "FetchEventsRequest", "FetchExperimentEventsResponse", "FetchLimit", @@ -91,9 +124,12 @@ "FetchPaginationCursor", "FieldArrayDeleteItem", "FunctionTypeEnum", + "GetDatasetResponse", "GetExperimentResponse", "GetProjectResponse", "Ids", + "InsertDatasetEvent", + "InsertDatasetEventRequest", "InsertEventsResponse", "InsertExperimentEvent", "InsertExperimentEventRequest", @@ -108,6 +144,7 @@ "NullableSavedFunctionId2", "ObjectReferenceNullish", "OrgName", + "PatchDataset", "PatchExperiment", "PatchProject", "Project", @@ -125,6 +162,8 @@ "SpanFieldOrderItem", "SpanType", "StartingAfter", + "SummarizeData", + "SummarizeDatasetResponse", "SummarizeExperimentResponse", "SummarizeScores", "Version", diff --git a/py/src/braintrust/api/_generated/models/common.py b/py/src/braintrust/api/_generated/models/common.py index 2704e160..0175de4b 100644 --- a/py/src/braintrust/api/_generated/models/common.py +++ b/py/src/braintrust/api/_generated/models/common.py @@ -4,11 +4,24 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 4f44bc62a969364392528ed2e8919cb5ba98b5a57aa742fe184571dcc61443da +# Content SHA-256: 4f4034d79da55f9379228307ac3f68923c3c0ef40d89c07614d9f141d6cf9862 from typing import Any, Literal, TypeAlias, TypedDict -from collections.abc import Mapping, Sequence from typing_extensions import NotRequired +from collections.abc import Mapping, Sequence + +AppLimitParam: TypeAlias = int | None +""" +Limit the number of objects to return +""" + + +class Metadata(TypedDict): + model: NotRequired[str | None] + """ + The model used for this example + """ + EndingBefore: TypeAlias = str """ @@ -17,6 +30,36 @@ For example, if the initial item in the last page you fetched had an id of `foo`, pass `ending_before=foo` to fetch the previous page. Note: you may only pass one of `starting_after` and `ending_before` """ + +class FeedbackResponseSchema(TypedDict): + status: Literal["success"] + + +FetchLimit: TypeAlias = int | None +""" +limit the number of traces fetched + +Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. + +The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. +""" + +FetchLimitParam: TypeAlias = int | None +""" +limit the number of traces fetched + +Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. + +The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. +""" + +FetchPaginationCursor: TypeAlias = str | None +""" +An opaque string to be used as a cursor for the next page of results, in order from latest to earliest. + +The string can be obtained directly from the `cursor` property of the previous fetch query +""" + FunctionTypeEnum: TypeAlias = ( Literal[ "llm", @@ -42,19 +85,135 @@ Filter search results to a particular set of object IDs. To specify a list of IDs, include the query param multiple times """ + +class FieldArrayDeleteItem(TypedDict): + delete: Sequence[Any] + path: Sequence[str] + + +class InsertEventsResponse(TypedDict): + row_ids: Sequence[str] + """ + The ids of all rows that were inserted, aligning one-to-one with the rows provided as input + """ + + +MaxRootSpanId: TypeAlias = str +""" +DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. + +Together, `max_xact_id` and `max_root_span_id` form a pagination cursor + +Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. +""" + +MaxXactId: TypeAlias = str +""" +DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. + +Together, `max_xact_id` and `max_root_span_id` form a pagination cursor + +Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. +""" + + +class ObjectReferenceNullish(TypedDict): + _xact_id: NotRequired[str | None] + """ + Transaction ID of the original event. + """ + created: NotRequired[str | None] + """ + Created timestamp of the original event. Used to help sort in the UI + """ + id: str + """ + ID of the original event. + """ + object_id: str + """ + ID of the object the event is originating from. + """ + object_type: Literal["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"] + """ + Type of the object the event is originating from. + """ + + OrgName: TypeAlias = str """ Filter search results to within a particular organization """ +ProjectIdQuery: TypeAlias = str +""" +Project id +""" + ProjectName: TypeAlias = str """ Name of the project to search for """ + +class SavedFunctionId1(TypedDict): + id: str + type: Literal["function"] + version: NotRequired[str] + """ + The version of the function + """ + + +class SavedFunctionId2(TypedDict): + function_type: NotRequired[FunctionTypeEnum] + name: str + type: Literal["global"] + + +SavedFunctionId: TypeAlias = SavedFunctionId1 | SavedFunctionId2 | None +""" +Optional function identifier that produced the classification +""" + StartingAfter: TypeAlias = str """ Pagination cursor id. For example, if the final item in the last page you fetched had an id of `foo`, pass `starting_after=foo` to fetch the next page. Note: you may only pass one of `starting_after` and `ending_before` """ + +Version: TypeAlias = str +""" +Retrieve a snapshot of events from a past time + +The version id is essentially a filter on the latest event transaction id. You can use the `max_xact_id` returned by a past fetch as the version to reproduce that exact fetch. +""" + + +class Classification(TypedDict): + confidence: NotRequired[float | None] + """ + Optional confidence score for the classification + """ + id: str + """ + Stable classification identifier + """ + label: NotRequired[str] + """ + Original label of the classification item, which is useful for search and indexing purposes + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + Optional metadata associated with the classification + """ + source: NotRequired[SavedFunctionId] + + +class FetchEventsRequest(TypedDict): + cursor: NotRequired[FetchPaginationCursor | None] + limit: NotRequired[FetchLimit | None] + max_root_span_id: NotRequired[MaxRootSpanId | None] + max_xact_id: NotRequired[MaxXactId | None] + version: NotRequired[Version | None] diff --git a/py/src/braintrust/api/_generated/models/datasets.py b/py/src/braintrust/api/_generated/models/datasets.py new file mode 100644 index 00000000..a6a17fab --- /dev/null +++ b/py/src/braintrust/api/_generated/models/datasets.py @@ -0,0 +1,359 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: fe1bb0191579f947158765086824d9cae75fd00b00e19be91e64fb89914ea3ae + +from typing import Any, Literal, TypeAlias, TypedDict +from typing_extensions import NotRequired +from collections.abc import Mapping, Sequence + +from .common import Classification, FieldArrayDeleteItem, Metadata, ObjectReferenceNullish + + +class CreateDataset(TypedDict): + description: NotRequired[str | None] + """ + Textual description of the dataset + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the dataset + """ + name: str + """ + Name of the dataset. Within a project, dataset names are unique + """ + project_id: str + """ + Unique identifier for the project that the dataset belongs under + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the dataset + """ + + +class DataSummary(TypedDict): + total_records: int + """ + Total number of records in the dataset + """ + + +class Dataset(TypedDict): + created: NotRequired[str | None] + """ + Date of dataset creation + """ + deleted_at: NotRequired[str | None] + """ + Date of dataset deletion, or null if the dataset is still active + """ + description: NotRequired[str | None] + """ + Textual description of the dataset + """ + id: str + """ + Unique identifier for the dataset + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the dataset + """ + name: str + """ + Name of the dataset. Within a project, dataset names are unique + """ + project_id: str + """ + Unique identifier for the project that the dataset belongs under + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the dataset + """ + url_slug: str + """ + URL slug for the dataset. used to construct dataset URLs + """ + user_id: NotRequired[str | None] + """ + Identifies the user who created the dataset + """ + + +DatasetIdParam: TypeAlias = str +""" +Dataset id +""" + +DatasetName: TypeAlias = str +""" +Name of the dataset to search for +""" + + +class FeedbackDatasetItem(TypedDict): + comment: NotRequired[str | None] + """ + An optional comment string to log about the dataset event + """ + id: str + """ + The id of the dataset event to log feedback for. This is the row `id` returned by `POST /v1/dataset/{dataset_id}/insert` + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + A dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI. Note, this metadata does not correspond to the main event itself, but rather the audit log attached to the event. + """ + source: NotRequired[Literal["app", "api", "external"] | None] + """ + The source of the feedback. Must be one of "external" (default), "app", or "api" + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class GetDatasetResponse(TypedDict): + objects: Sequence[Dataset] + """ + A list of dataset objects + """ + + +class PatchDataset(TypedDict): + description: NotRequired[str | None] + """ + Textual description of the dataset + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the dataset + """ + name: NotRequired[str | None] + """ + Name of the dataset. Within a project, dataset names are unique + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the dataset + """ + + +SummarizeData: TypeAlias = bool | None +""" +Whether to summarize the data. If false (or omitted), only the metadata will be returned. +""" + + +class SummarizeDatasetResponse(TypedDict): + data_summary: NotRequired[DataSummary | None] + dataset_name: str + """ + Name of the dataset + """ + dataset_url: str + """ + URL to the dataset's page in the Braintrust app + """ + project_name: str + """ + Name of the project that the dataset belongs to + """ + project_url: str + """ + URL to the project's page in the Braintrust app + """ + + +class DatasetEvent(TypedDict): + _pagination_key: NotRequired[str | None] + """ + A stable, time-ordered key that can be used to paginate over dataset events. This field is auto-generated by Braintrust and only exists in Brainstore. + """ + _xact_id: str + """ + The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the dataset (see the `version` parameter) + """ + audit_data: NotRequired[Sequence[Any] | None] + """ + Optional list of audit entries attached to this event + """ + classifications: NotRequired[Mapping[str, Sequence[Classification]] | None] + """ + Classifications for this event (dictionary from classification name to items) + """ + comments: NotRequired[Sequence[Any] | None] + """ + Optional list of comments attached to this event + """ + created: str + """ + The timestamp the dataset event was created + """ + dataset_id: str + """ + Unique identifier for the dataset + """ + expected: NotRequired[Any | None] + """ + The output of your application, including post-processing (an arbitrary, JSON serializable object) + """ + facets: NotRequired[Mapping[str, str | None] | None] + """ + Facets for categorization (dictionary from facet id to value) + """ + id: str + """ + A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you + """ + input: NotRequired[Any | None] + """ + The argument that uniquely define an input case (an arbitrary, JSON serializable object) + """ + is_root: NotRequired[bool | None] + """ + Whether this span is a root span + """ + metadata: NotRequired[Metadata | None] + """ + A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings + """ + origin: NotRequired[ObjectReferenceNullish | None] + project_id: str + """ + Unique identifier for the project that the dataset belongs under + """ + root_span_id: str + """ + A unique identifier for the trace this dataset event belongs to + """ + span_id: str + """ + A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class FeedbackDatasetEventRequest(TypedDict): + feedback: Sequence[FeedbackDatasetItem] + """ + A list of dataset feedback items + """ + + +class FetchDatasetEventsResponse(TypedDict): + cursor: NotRequired[str | None] + """ + Pagination cursor + + Pass this string directly as the `cursor` param to your next fetch request to get the next page of results. Not provided if the returned result set is empty. + """ + events: Sequence[DatasetEvent] + """ + A list of fetched events + """ + + +class InsertDatasetEvent(TypedDict): + _array_delete: NotRequired[Sequence[FieldArrayDeleteItem] | None] + """ + The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. + + For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. + """ + _is_merge: NotRequired[bool | None] + """ + The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. + + For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` + """ + _merge_paths: NotRequired[Sequence[Sequence[str]] | None] + """ + The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. + + For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. + """ + _object_delete: NotRequired[bool | None] + """ + Pass `_object_delete=true` to mark the dataset event deleted. Deleted events will not show up in subsequent fetches for this dataset + """ + _parent_id: NotRequired[str | None] + """ + DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. + + Use the `_parent_id` field to create this row as a subspan of an existing row. Tracking hierarchical relationships are important for tracing (see the [guide](https://www.braintrust.dev/docs/instrument) for full details). + + For example, say we have logged a row `{"id": "abc", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"_parent_id": "abc", "id": "llm_call", "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + created: NotRequired[str | None] + """ + The timestamp the dataset event was created + """ + expected: NotRequired[Any | None] + """ + The output of your application, including post-processing (an arbitrary, JSON serializable object) + """ + facets: NotRequired[Mapping[str, str | None] | None] + """ + Facets for categorization (dictionary from facet id to value) + """ + id: NotRequired[str | None] + """ + A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you + """ + input: NotRequired[Any | None] + """ + The argument that uniquely define an input case (an arbitrary, JSON serializable object) + """ + metadata: NotRequired[Metadata | None] + """ + A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings + """ + origin: NotRequired[ObjectReferenceNullish | None] + root_span_id: NotRequired[str | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + span_id: NotRequired[str | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + span_parents: NotRequired[Sequence[str] | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class InsertDatasetEventRequest(TypedDict): + events: Sequence[InsertDatasetEvent] + """ + A list of dataset events to insert + """ diff --git a/py/src/braintrust/api/_generated/models/experiments.py b/py/src/braintrust/api/_generated/models/experiments.py index 963d9fa3..39eb30c6 100644 --- a/py/src/braintrust/api/_generated/models/experiments.py +++ b/py/src/braintrust/api/_generated/models/experiments.py @@ -4,13 +4,13 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 8217275b6af7a615da1333759bd9958a66bfbd003548dc35a70cc3430c27b119 +# Content SHA-256: 48aad16184321bf01b7d94dbc3515e18099eeccac91c32dae3c2c5a09af5e38d from typing import Any, Literal, TypeAlias, TypedDict -from collections.abc import Mapping, Sequence from typing_extensions import NotRequired +from collections.abc import Mapping, Sequence -from .common import FunctionTypeEnum +from .common import Classification, FieldArrayDeleteItem, Metadata, ObjectReferenceNullish AppLimitWithDefaultParam: TypeAlias = int | None """ @@ -45,13 +45,6 @@ class Context(TypedDict): """ -class Metadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - class Metrics(TypedDict): caller_filename: NotRequired[Any | None] """ @@ -129,67 +122,6 @@ class FeedbackExperimentItem(TypedDict): """ -class FeedbackResponseSchema(TypedDict): - status: Literal["success"] - - -FetchLimit: TypeAlias = int | None -""" -limit the number of traces fetched - -Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. - -The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. -""" - -FetchLimitParam: TypeAlias = int | None -""" -limit the number of traces fetched - -Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. - -The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. -""" - -FetchPaginationCursor: TypeAlias = str | None -""" -An opaque string to be used as a cursor for the next page of results, in order from latest to earliest. - -The string can be obtained directly from the `cursor` property of the previous fetch query -""" - - -class InsertEventsResponse(TypedDict): - row_ids: Sequence[str] - """ - The ids of all rows that were inserted, aligning one-to-one with the rows provided as input - """ - - -class FieldArrayDeleteItem(TypedDict): - delete: Sequence[Any] - path: Sequence[str] - - -MaxRootSpanId: TypeAlias = str -""" -DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. - -Together, `max_xact_id` and `max_root_span_id` form a pagination cursor - -Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. -""" - -MaxXactId: TypeAlias = str -""" -DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. - -Together, `max_xact_id` and `max_root_span_id` form a pagination cursor - -Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. -""" - - class MetricSummary(TypedDict): diff: NotRequired[float] """ @@ -217,35 +149,6 @@ class MetricSummary(TypedDict): """ -class ObjectReferenceNullish(TypedDict): - field_xact_id: NotRequired[str | None] - """ - Transaction ID of the original event. - """ - created: NotRequired[str | None] - """ - Created timestamp of the original event. Used to help sort in the UI - """ - id: str - """ - ID of the original event. - """ - object_id: str - """ - ID of the object the event is originating from. - """ - object_type: Literal["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"] - """ - Type of the object the event is originating from. - """ - - -ProjectIdQuery: TypeAlias = str -""" -Project id -""" - - class RepoInfo(TypedDict): author_email: NotRequired[str | None] """ @@ -285,27 +188,6 @@ class RepoInfo(TypedDict): """ -class SavedFunctionId1(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class SavedFunctionId2(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -SavedFunctionId: TypeAlias = SavedFunctionId1 | SavedFunctionId2 | None -""" -Optional function identifier that produced the classification -""" - - class ScoreSummary(TypedDict): diff: NotRequired[float] """ @@ -386,13 +268,6 @@ class SummarizeExperimentResponse(TypedDict): Whether to summarize the scores and metrics. If false (or omitted), only the metadata will be returned. """ -Version: TypeAlias = str -""" -Retrieve a snapshot of events from a past time - -The version id is essentially a filter on the latest event transaction id. You can use the `max_xact_id` returned by a past fetch as the version to reproduce that exact fetch. -""" - class CreateExperiment(TypedDict): base_exp_id: NotRequired[str | None] @@ -522,26 +397,6 @@ class Experiment(TypedDict): """ -class Classification(TypedDict): - confidence: NotRequired[float | None] - """ - Optional confidence score for the classification - """ - id: str - """ - Stable classification identifier - """ - label: NotRequired[str] - """ - Original label of the classification item, which is useful for search and indexing purposes - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Optional metadata associated with the classification - """ - source: NotRequired[SavedFunctionId] - - class FeedbackExperimentEventRequest(TypedDict): feedback: Sequence[FeedbackExperimentItem] """ @@ -549,14 +404,6 @@ class FeedbackExperimentEventRequest(TypedDict): """ -class FetchEventsRequest(TypedDict): - cursor: NotRequired[FetchPaginationCursor | None] - limit: NotRequired[FetchLimit | None] - max_root_span_id: NotRequired[MaxRootSpanId | None] - max_xact_id: NotRequired[MaxXactId | None] - version: NotRequired[Version | None] - - class GetExperimentResponse(TypedDict): objects: Sequence[Experiment] """ @@ -625,11 +472,11 @@ class SpanAttributes(TypedDict): class ExperimentEvent(TypedDict): - field_pagination_key: NotRequired[str | None] + _pagination_key: NotRequired[str | None] """ A stable, time-ordered key that can be used to paginate over experiment events. This field is auto-generated by Braintrust and only exists in Brainstore. """ - field_xact_id: str + _xact_id: str """ The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the experiment (see the `version` parameter) """ @@ -735,29 +582,29 @@ class FetchExperimentEventsResponse(TypedDict): class InsertExperimentEvent(TypedDict): - field_array_delete: NotRequired[Sequence[FieldArrayDeleteItem] | None] + _array_delete: NotRequired[Sequence[FieldArrayDeleteItem] | None] """ The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. """ - field_is_merge: NotRequired[bool | None] + _is_merge: NotRequired[bool | None] """ The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` """ - field_merge_paths: NotRequired[Sequence[Sequence[str]] | None] + _merge_paths: NotRequired[Sequence[Sequence[str]] | None] """ The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. """ - field_object_delete: NotRequired[bool | None] + _object_delete: NotRequired[bool | None] """ Pass `_object_delete=true` to mark the experiment event deleted. Deleted events will not show up in subsequent fetches for this experiment """ - field_parent_id: NotRequired[str | None] + _parent_id: NotRequired[str | None] """ DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. diff --git a/py/src/braintrust/api/_generated/models/projects.py b/py/src/braintrust/api/_generated/models/projects.py index 16355ba6..0f744ee4 100644 --- a/py/src/braintrust/api/_generated/models/projects.py +++ b/py/src/braintrust/api/_generated/models/projects.py @@ -4,19 +4,14 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 430c31ef9773d7cd389d86e7ef06d075e8db6c74cd7e6b7ee095ec87ef435a9e +# Content SHA-256: 7f255d8bb302ac26b6281a82889c8108c5a8f65cc4b9f748eb47919dcf89d947 from typing import Any, Literal, TypeAlias, TypedDict -from collections.abc import Mapping, Sequence from typing_extensions import NotRequired +from collections.abc import Mapping, Sequence from .common import FunctionTypeEnum -AppLimitParam: TypeAlias = int | None -""" -Limit the number of objects to return -""" - class CreateProject(TypedDict): description: NotRequired[str | None] diff --git a/py/src/braintrust/api/_generated/projects.py b/py/src/braintrust/api/_generated/projects.py index c40ab5c4..e29d3899 100644 --- a/py/src/braintrust/api/_generated/projects.py +++ b/py/src/braintrust/api/_generated/projects.py @@ -4,7 +4,7 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 4b64219218b76122324149ec0bf037bf1254affc40f8acb2f27e0481937da4e1 +# Content SHA-256: 13bc46e7fff061668546a7e3b807764115580449165934413a88d381a2091675 """Generated Projects REST operations and resource.""" @@ -12,8 +12,8 @@ from .._service import Operation, Parameter, ResourceAPI from ..policies import RetryMode -from .models.common import EndingBefore, Ids, OrgName, ProjectName, StartingAfter -from .models.projects import AppLimitParam, CreateProject, GetProjectResponse, PatchProject, Project, ProjectIdParam +from .models.common import AppLimitParam, EndingBefore, Ids, OrgName, ProjectName, StartingAfter +from .models.projects import CreateProject, GetProjectResponse, PatchProject, Project, ProjectIdParam POST_PROJECT = Operation( diff --git a/py/src/braintrust/api/cassettes/test_datasets_end_to_end_with_real_backend.yaml b/py/src/braintrust/api/cassettes/test_datasets_end_to_end_with_real_backend.yaml new file mode 100644 index 00000000..03348c9f --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_datasets_end_to_end_with_real_backend.yaml @@ -0,0 +1,1043 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '374' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NzZmZGRlMDYtZDJjMi00MzBlLWE3YzItNGIxNjBjODdmNTY0'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:50 GMT + Etag: + - '"13kwty4qcjgae"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NzZmZGRlMDYtZDJjMi00MzBlLWE3YzItNGIxNjBjODdmNTY0 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::84svx-1787080610011-df51cc1ed532 + status: + code: 200 + message: OK +- request: + body: '{"name": "python-sdk-generated-datasets-vcr", "org_name": "abhi-test-org"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '74' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/project + response: + body: + string: '{"id":"6becfeb2-3373-4c68-abe2-c214f032ee92","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-generated-datasets-vcr","description":null,"created":"2026-08-18T19:16:50.545Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:50 GMT + Via: + - 1.1 15b6b7b469d24b32948c94136199cbf6.cloudfront.net (CloudFront), 1.1 21c66eb5f493a6e3ddbaa803cebfe014.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - _-kFR6w0hgP-Q0ILhWiVX8USkO5L44V88IqC6-jg7xw1Fg79qcVLlw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa2-1d315d9b397f25094eb89b78;Parent=00b126f6af29a32d;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '275' + etag: + - W/"113-3hKC9royA/xKcWxXJmZwtU0isks" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhcGrtoAMEFrw= + x-amzn-RequestId: + - 73d7e86f-22fb-4c73-8593-cc9bde4affd9 + x-bt-internal-trace-id: + - 6a84afa2000000005892b014c4bafaf6 + status: + code: 200 + message: OK +- request: + body: '{"project_id": "6becfeb2-3373-4c68-abe2-c214f032ee92", "name": "generated-datasets-api", + "description": "created by the Python SDK VCR test"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '141' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset + response: + body: + string: '{"id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","name":"generated-datasets-api","description":"created + by the Python SDK VCR test","created":"2026-08-18T19:16:50.914Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"generated-datasets-api"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:50 GMT + Via: + - 1.1 e7dbdec0a9983cf82c16b332b6b30812.cloudfront.net (CloudFront), 1.1 d03af248468c898a111754f0666c2316.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - O-mA1HeLqQPcTMiRIUnAH2Z6IE1Kr4a2PXb1WKnNxTaqWDHqwigHwA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa2-23d75949277cda20509c4634;Parent=49daea59af84160f;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '348' + etag: + - W/"15c-q4UVzcnUvQwHMI7NwtPhRuqMwjE" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhgG8gIAMEpbw= + x-amzn-RequestId: + - 38fbf005-3946-425b-bb5f-aae0868902a9 + x-bt-internal-trace-id: + - 6a84afa200000000197bf6756a431c32 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/dataset?dataset_name=generated-datasets-api&project_id=6becfeb2-3373-4c68-abe2-c214f032ee92 + response: + body: + string: '{"objects":[{"id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","name":"generated-datasets-api","description":"created + by the Python SDK VCR test","created":"2026-08-18T19:16:50.914Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"generated-datasets-api"}]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:51 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 8e6145785e47042f882be946f6c05880.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - YOU_t9NoHJgfuh3zPiBpLtYMuoOG9SrBIpyRVdjEAzhOxIOithJlNQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa3-088698aa712291000769c25e;Parent=6691961c85e964e1;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '362' + etag: + - W/"16a-+aZCvtO7b694f3OFpEoTZISOies" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhjHXLoAMEdCA= + x-amzn-RequestId: + - 14309159-53fb-4a9b-b2c4-a49dd2fc1f7a + x-bt-internal-trace-id: + - 6a84afa300000000379cfd38f9e43c5f + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d + response: + body: + string: '{"id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","name":"generated-datasets-api","description":"created + by the Python SDK VCR test","created":"2026-08-18T19:16:50.914Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"generated-datasets-api"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:51 GMT + Via: + - 1.1 15b6b7b469d24b32948c94136199cbf6.cloudfront.net (CloudFront), 1.1 3340b5a392e45fce453c4d978abfd6be.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 4stTB4UBE2LTKTGNKaPHqGls4aDcduh92khAG-4mFJA1gE5UiXSpow== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa3-2cc3f6f914c2d5bc60de6739;Parent=4787d7ab149c7266;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '348' + etag: + - W/"15c-q4UVzcnUvQwHMI7NwtPhRuqMwjE" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhmEWmoAMEdAQ= + x-amzn-RequestId: + - 0d846bfe-e51e-45d6-9bb2-ae15784c3cb8 + x-bt-internal-trace-id: + - 6a84afa3000000003cf4079af97f50c8 + status: + code: 200 + message: OK +- request: + body: '{"description": "updated by the Python SDK VCR test"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '53' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: PATCH + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d + response: + body: + string: '{"id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","name":"generated-datasets-api","description":"updated + by the Python SDK VCR test","created":"2026-08-18T19:16:50.914Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"generated-datasets-api"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:51 GMT + Via: + - 1.1 15b6b7b469d24b32948c94136199cbf6.cloudfront.net (CloudFront), 1.1 53d47b61433f6e1682b806fc166731be.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - Bxn6d0NlM2ObG3WbXDB3xvZmKsjUEMUtiR38QfKJDrhXnkMr8TRI2g== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa3-6985ad335cd951e15fa86151;Parent=13d77f36b78c323d;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '348' + etag: + - W/"15c-GFOJ50tdcaFa3JXHxN8LTSE2L3Q" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhoG3pIAMEl0g= + x-amzn-RequestId: + - 693bfc35-3db1-4569-84a4-1d2fe379b5bb + x-bt-internal-trace-id: + - 6a84afa3000000007306be8937ba4e79 + status: + code: 200 + message: OK +- request: + body: '{"events": [{"id": "generated-datasets-row", "input": {"question": "What + is the answer?", "context": {"preserved": true}, "replace_me": {"old": true}}, + "expected": "42", "tags": ["keep", "remove"]}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '199' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/insert + response: + body: + string: '{"row_ids":["generated-datasets-row"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:52 GMT + Via: + - 1.1 e7dbdec0a9983cf82c16b332b6b30812.cloudfront.net (CloudFront), 1.1 cfcfb1d8fbf5ce2b107182799687a614.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - rc7qXgnarz2VdH6_CeOo51zAswrOfW0sMkPS8gbudM2BPc58zp5Hgg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa3-35e9342f0cd8b168547713d2;Parent=6f584cdd95c15ff0;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '38' + etag: + - W/"26-dbcFrOkyOLEBqu8mqJRcuZNKqKk" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhrEaSIAMEdAQ= + x-amzn-RequestId: + - dc979b3a-d67d-4605-84bf-7799585b2419 + x-bt-internal-trace-id: + - 6a84afa30000000052530fb0ede83e00 + status: + code: 200 + message: OK +- request: + body: '{"events": [{"id": "generated-datasets-row", "_is_merge": true, "_merge_paths": + [["input", "replace_me"]], "_array_delete": [{"path": ["tags"], "delete": ["remove"]}], + "input": {"question": "What is the updated answer?", "replace_me": {"new": true}}}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '252' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/insert + response: + body: + string: '{"row_ids":["generated-datasets-row"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:52 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 cdd327922be1fd75b18f2ae0982269cc.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - B9i_1Vba2Z01-ue2j0sNSZmmsWwWLgaTaYlWnUxMk7UCuaocCBiGtQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa4-4f9bf717350599a03e5cd97e;Parent=228fcbb27ecd26f0;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '38' + etag: + - W/"26-dbcFrOkyOLEBqu8mqJRcuZNKqKk" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBhyE7zoAMEgdA= + x-amzn-RequestId: + - 93bbc886-ac58-4f29-bbf9-572af8e74687 + x-bt-internal-trace-id: + - 6a84afa4000000003bcb26f2149e1977 + status: + code: 200 + message: OK +- request: + body: '{"limit": 10}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '13' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/fetch + response: + body: + string: '{"events":[{"_pagination_key":"p07675452787713376256","_xact_id":"1000197710368010024","audit_data":[{"_xact_id":"1000197710368007664","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197710368010024","audit_data":{"action":"merge","deleted_tags":["remove"],"from":null,"path":["input"],"to":{"question":"What + is the updated answer?","replace_me":{"new":true}}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-18T19:16:52.005Z","dataset_id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","expected":"42","facets":null,"id":"generated-datasets-row","input":{"context":{"preserved":true},"question":"What + is the updated answer?","replace_me":{"new":true}},"is_root":true,"metadata":null,"origin":null,"project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","root_span_id":"76c1479f-745e-495d-9e5f-746b1cc37f6b","span_id":"76c1479f-745e-495d-9e5f-746b1cc37f6b","tags":["keep"]}],"cursor":"aoSvpOXwAAA"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 18 Aug 2026 19:16:53 GMT + Via: + - 1.1 10b7771eba7b9b9a487cb76285130afe.cloudfront.net (CloudFront), 1.1 74797197cacba7d22a7c3a7685b38272.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 8ACc17kJ-zZuoqs-bumRbUOtb2k0VfO0b7XF-RNItaUn_2tE1ySuzQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa4-47ca1c9848d775df203a2a02;Parent=3d7279cec954c92a;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '959' + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CUBh2H7goAMEg8w= + x-amzn-RequestId: + - 50434e45-687a-4377-a232-ffac81a5ac22 + x-bt-api-duration-ms: + - '151' + x-bt-brainstore-duration-ms: + - '91' + x-bt-cursor: + - aoSvpOXwAAA + x-bt-internal-trace-id: + - 6a84afa40000000075ba2e57df524d3c + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/fetch?limit=10 + response: + body: + string: '{"events":[{"_pagination_key":"p07675452787713376256","_xact_id":"1000197710368010024","audit_data":[{"_xact_id":"1000197710368007664","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197710368010024","audit_data":{"action":"merge","deleted_tags":["remove"],"from":null,"path":["input"],"to":{"question":"What + is the updated answer?","replace_me":{"new":true}}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-18T19:16:52.005Z","dataset_id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","expected":"42","facets":null,"id":"generated-datasets-row","input":{"context":{"preserved":true},"question":"What + is the updated answer?","replace_me":{"new":true}},"is_root":true,"metadata":null,"origin":null,"project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","root_span_id":"76c1479f-745e-495d-9e5f-746b1cc37f6b","span_id":"76c1479f-745e-495d-9e5f-746b1cc37f6b","tags":["keep"]}],"cursor":"aoSvpOXwAAA"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 18 Aug 2026 19:16:53 GMT + Via: + - 1.1 5785adb181f17ab60069f54a29dd7b7a.cloudfront.net (CloudFront), 1.1 5a2f8eb373b5a17b769c0fee9b0725a6.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - -IzYtcvlm5QGuiBmo97ji1jvwu6N49b0utHI8ElOPJwbIN_Fh-WnUQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa5-049f90db7f1431164562c029;Parent=6f218dedbb202b8d;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '959' + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CUBh4HeooAMEiig= + x-amzn-RequestId: + - 764fbd98-7451-4119-bf33-b3f3d7dc5cb5 + x-bt-api-duration-ms: + - '128' + x-bt-brainstore-duration-ms: + - '68' + x-bt-cursor: + - aoSvpOXwAAA + x-bt-internal-trace-id: + - 6a84afa5000000007241fa6a4189a601 + status: + code: 200 + message: OK +- request: + body: '{"feedback": [{"id": "generated-datasets-row", "comment": "useful example"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '77' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/feedback + response: + body: + string: '{"status":"success"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:53 GMT + Via: + - 1.1 5785adb181f17ab60069f54a29dd7b7a.cloudfront.net (CloudFront), 1.1 53d47b61433f6e1682b806fc166731be.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - _9dg7H_F1iUjELlq23RGwGsQPouet7Y7akx1A7cB0a0SME4HnO1lhg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa5-2ac302d7447740e22ca83cc7;Parent=6ffcf3857dfa4378;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '20' + etag: + - W/"14-Y53wuE/mmbSikKcT/WualL1N65U" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBh7EC7oAMEQpA= + x-amzn-RequestId: + - a0bdeaa2-b3b6-4a60-ab3b-a374ccfc3969 + x-bt-internal-trace-id: + - 6a84afa50000000010f97d31db76d69d + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/summarize?summarize_data=true + response: + body: + string: '{"project_name":"python-sdk-generated-datasets-vcr","project_url":"https://www.braintrust.dev/app/abhi-test-org/p/python-sdk-generated-datasets-vcr","dataset_name":"generated-datasets-api","dataset_url":"https://www.braintrust.dev/app/abhi-test-org/p/python-sdk-generated-datasets-vcr/datasets/generated-datasets-api","data_summary":{"total_records":1}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:53 GMT + Via: + - 1.1 301a0472e68cdb30a8215c443fcf0ff0.cloudfront.net (CloudFront), 1.1 36c050103b969d83a8b90ba7cba12542.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - U30EM2fWBACS5-FXI1DmbCx9ea-ld53nAP7PD5jDiURl5Ncbe3q6oA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa5-6d2183c666179e022b6545e0;Parent=1188a8552552ed2b;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '353' + etag: + - W/"161-P7RvUEYLkYD3c1Rw/5Met/O3De8" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBh-G85oAMEjDA= + x-amzn-RequestId: + - c1474347-1bcc-492d-89d2-b40d6f59029f + x-bt-internal-trace-id: + - 6a84afa5000000001962b58fe7ea0a98 + status: + code: 200 + message: OK +- request: + body: '{"events": [{"id": "generated-datasets-row", "_object_delete": true}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '70' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/insert + response: + body: + string: '{"row_ids":["generated-datasets-row"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:54 GMT + Via: + - 1.1 301a0472e68cdb30a8215c443fcf0ff0.cloudfront.net (CloudFront), 1.1 0e761f7a5b2481acd893422a702c9fa8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - ckhqPwUONB-pfnpYz4wdxnR0V6g5BsstKG_rqzru2risY968eH9J6Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa6-7b15ca2a16e48ef3091927bf;Parent=5963bfaf87a39abf;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '38' + etag: + - W/"26-dbcFrOkyOLEBqu8mqJRcuZNKqKk" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBiBF78IAMEIsg= + x-amzn-RequestId: + - f96c2722-70f8-4742-92f5-1a20cdc4476c + x-bt-internal-trace-id: + - 6a84afa600000000489014ea0c7b404c + status: + code: 200 + message: OK +- request: + body: '{"limit": 10}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '13' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d/fetch + response: + body: + string: '{"events":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 18 Aug 2026 19:16:54 GMT + Via: + - 1.1 15b6b7b469d24b32948c94136199cbf6.cloudfront.net (CloudFront), 1.1 41c02c3f5acef4f58284b65a8f7a983a.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - YWYIZMIzhSFwJQko60WTHrqI8N2HDDSr_eaqR95FWaImO_4jDf_TAg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa6-5093eba0386d5c90783d2994;Parent=0d8dba63fbab494f;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '13' + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CUBiFFMToAMEndw= + x-amzn-RequestId: + - 9d0e6e45-e8da-4279-a9bf-2a30f042e832 + x-bt-api-duration-ms: + - '192' + x-bt-brainstore-duration-ms: + - '150' + x-bt-internal-trace-id: + - 6a84afa6000000000d397b0d60bac2d9 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/dataset/89cb25ac-eb76-4412-b07a-63f515b8d27d + response: + body: + string: '{"id":"89cb25ac-eb76-4412-b07a-63f515b8d27d","project_id":"6becfeb2-3373-4c68-abe2-c214f032ee92","name":"generated-datasets-api","description":"updated + by the Python SDK VCR test","created":"2026-08-18T19:16:50.914Z","deleted_at":"2026-08-18T19:16:54.899Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"generated-datasets-api"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:54 GMT + Via: + - 1.1 5785adb181f17ab60069f54a29dd7b7a.cloudfront.net (CloudFront), 1.1 2ffb622580a0a24837f798fa62268b12.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - xQRPpHr6RN8JNj0AWkhp2Fq1ZuheLP13atHBwkrvUDSsnPrhN6D9Mg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa6-6990e07019f50a827b2485ac;Parent=1028c0fb99f16bfc;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '370' + etag: + - W/"172-WZvW5435xVm2J6NXI2KO+XFXFgQ" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBiIG_ZIAMESbQ= + x-amzn-RequestId: + - 370859bd-163b-42dc-a1a4-798ff2469c2b + x-bt-internal-trace-id: + - 6a84afa6000000002a53853173f22d09 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/project/6becfeb2-3373-4c68-abe2-c214f032ee92 + response: + body: + string: '{"id":"6becfeb2-3373-4c68-abe2-c214f032ee92","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-generated-datasets-vcr","description":null,"created":"2026-08-18T19:16:50.545Z","deleted_at":"2026-08-18T19:16:55.150Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 19:16:55 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 4ec5f8da969dc981ba2067c9dad5dad8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - BWEgty4Cmdx_NofhP9BI01N0oK-m4lZi3cUYwLYdTkvvcEojmcqC9A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a84afa7-01ef4f8320b418602db5e93f;Parent=7b7e383369ab8b0e;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '297' + etag: + - W/"129-uJdSvT7FK2jzVCxT+dSJlCP73hg" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CUBiLHVAIAMEnIw= + x-amzn-RequestId: + - f04c0695-444a-4b1a-96b5-0defc97aaa32 + x-bt-internal-trace-id: + - 6a84afa7000000006d33629688bf93aa + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/cassettes/test_high_level_dataset_uses_generated_resources.yaml b/py/src/braintrust/api/cassettes/test_high_level_dataset_uses_generated_resources.yaml new file mode 100644 index 00000000..55e69f15 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_high_level_dataset_uses_generated_resources.yaml @@ -0,0 +1,594 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '374' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NjIzMjM5YWItMGZmOS00ZWY1LWE4NDUtOTM5YzgzYWNjYjZj'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:49 GMT + Etag: + - '"13kwty4qcjgae"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NjIzMjM5YWItMGZmOS00ZWY1LWE4NDUtOTM5YzgzYWNjYjZj + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::kvjw6-1787075029412-0b59d51b3653 + status: + code: 200 + message: OK +- request: + body: '{"name": "python-sdk-high-level-datasets-vcr", "org_name": "abhi-test-org"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '75' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/project + response: + body: + string: '{"id":"8bc899ea-301c-43a4-afce-6b7e381f72df","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-high-level-datasets-vcr","description":null,"created":"2026-08-18T17:43:49.805Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:49 GMT + Via: + - 1.1 7b609f6f2da1597a3efb21b332d7ce54.cloudfront.net (CloudFront), 1.1 777f4a7ed43b40353f84311869e119c8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - xgjOSfg2torH-h2YzzYP9-lD_cWvZesy7bpprxrpZfDCsdV7CDA06A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d5-58fb2ffc78c45aed19654beb;Parent=1598a14803640071;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '276' + etag: + - W/"114-KUQ+D50relGBKxWrJbiPQ1lUa7c" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz5dE1VoAMEl0g= + x-amzn-RequestId: + - b6d3b069-d09c-44dd-b391-743d629f1a3c + x-bt-internal-trace-id: + - 6a8499d500000000745a36dec81ff912 + status: + code: 200 + message: OK +- request: + body: '{"project_id": "8bc899ea-301c-43a4-afce-6b7e381f72df", "name": "logs", + "description": "created through braintrust.init_dataset"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '128' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset + response: + body: + string: '{"id":"e0753a78-8202-4f97-9b70-8555cca01f24","project_id":"8bc899ea-301c-43a4-afce-6b7e381f72df","name":"logs","description":"created + through braintrust.init_dataset","created":"2026-08-18T17:43:50.112Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"logs"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:50 GMT + Via: + - 1.1 7f51f788f893911f0cd6b09d9ab21c4a.cloudfront.net (CloudFront), 1.1 04fa8a9e73b27e301fb4b6d36f313186.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - EsPn_ln1NovOs5-Dmk7-sGLHMB4YmUiyHr_Px7uBVM7obAie2tSFLg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d5-235c78ea3fb567ef4d6a0d73;Parent=36a52cda16bbac71;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '317' + etag: + - W/"13d-bhQPuhN/bOu6S5+5+uANd0j5CnM" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz5gGedIAMEohw= + x-amzn-RequestId: + - 5bb15c30-a620-4a52-abb1-cd06abaaf1cd + x-bt-internal-trace-id: + - 6a8499d50000000019e6272960be9737 + status: + code: 200 + message: OK +- request: + body: '{"events": [{"id": "high-level-generated-row", "input": {"question": "What + is the answer?"}, "expected": "42"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '112' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/e0753a78-8202-4f97-9b70-8555cca01f24/insert + response: + body: + string: '{"row_ids":["high-level-generated-row"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:50 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 dcd16c430149132ea12a5783d54ff114.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - LQlLspEyAr_LHJxALCbJrmfK7_5WwRZ8LzJgbiB9JHYmnEKzv7hUDg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d6-078871af3601f8740117c7f6;Parent=2fc78e399729f92f;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '40' + etag: + - W/"28-/WJmCRpRI2OkOxN+4DhKff4swbE" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz5kGw1IAMEWTA= + x-amzn-RequestId: + - 64738b4b-1f41-4561-859e-da4161e0a293 + x-bt-internal-trace-id: + - 6a8499d600000000029cae16c59cb207 + status: + code: 200 + message: OK +- request: + body: '{"limit": 10}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '13' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/e0753a78-8202-4f97-9b70-8555cca01f24/fetch + response: + body: + string: '{"events":[{"_pagination_key":"p07675428811820564480","_xact_id":"1000197710002164573","audit_data":[{"_xact_id":"1000197710002164573","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-18T17:43:50.515Z","dataset_id":"e0753a78-8202-4f97-9b70-8555cca01f24","expected":"42","facets":null,"id":"high-level-generated-row","input":{"question":"What + is the answer?"},"is_root":true,"metadata":null,"origin":null,"project_id":"8bc899ea-301c-43a4-afce-6b7e381f72df","root_span_id":"876464a7-1f60-4802-9935-17437b873c37","span_id":"876464a7-1f60-4802-9935-17437b873c37","tags":null}],"cursor":"aoSZ1pNdAAA"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 18 Aug 2026 17:43:51 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 777f4a7ed43b40353f84311869e119c8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - d6WbhB4EDH4kf_dgcAAaaAa8S5h66C0-d8IZHrZB8Q0ZHJF459Eu4w== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d6-492948d8584a085765606eb6;Parent=13c6b3c43f57d24e;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '669' + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CTz5qFHgoAMEGfA= + x-amzn-RequestId: + - 3f2337b5-cbd1-4243-baae-7914d067e781 + x-bt-api-duration-ms: + - '136' + x-bt-brainstore-duration-ms: + - '78' + x-bt-cursor: + - aoSZ1pNdAAA + x-bt-internal-trace-id: + - 6a8499d6000000006adb50d7a8599b2e + status: + code: 200 + message: OK +- request: + body: '{"limit": 10, "cursor": "aoSZ1pNdAAA"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '38' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/dataset/e0753a78-8202-4f97-9b70-8555cca01f24/fetch + response: + body: + string: '{"events":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 18 Aug 2026 17:43:51 GMT + Via: + - 1.1 5785adb181f17ab60069f54a29dd7b7a.cloudfront.net (CloudFront), 1.1 36c050103b969d83a8b90ba7cba12542.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 4nWsumLw7zOVWlTAvjH6l1ASt5NzJUglPnIf5bs1rY6I4HyloPd1Gw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d7-68ed2747425765401f6022b3;Parent=2817e8289b5fbb74;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '13' + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CTz5tG5EIAMEa6g= + x-amzn-RequestId: + - 92d19bb6-1b7b-4136-87dc-d3327a904bdc + x-bt-api-duration-ms: + - '132' + x-bt-brainstore-duration-ms: + - '82' + x-bt-internal-trace-id: + - 6a8499d7000000005acfdf1725126e8c + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/dataset/e0753a78-8202-4f97-9b70-8555cca01f24/summarize?summarize_data=true + response: + body: + string: '{"project_name":"python-sdk-high-level-datasets-vcr","project_url":"https://www.braintrust.dev/app/abhi-test-org/p/python-sdk-high-level-datasets-vcr","dataset_name":"logs","dataset_url":"https://www.braintrust.dev/app/abhi-test-org/p/python-sdk-high-level-datasets-vcr/datasets/logs","data_summary":{"total_records":1}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:51 GMT + Via: + - 1.1 7b609f6f2da1597a3efb21b332d7ce54.cloudfront.net (CloudFront), 1.1 50d743941b822ae5fa30db69233863a6.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - awm3fg8BEF3tfCENgkz01OK90HX7uTHbBHsLM_EJDFgjp81SV8GM7g== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d7-31a5f78a0b76620d2b202493;Parent=0bf38a04fbd2a1ba;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '320' + etag: + - W/"140-ZIATApjTynNDI+KDjd4FQxLCvzE" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz5wGbDoAMEHoQ= + x-amzn-RequestId: + - 9e349ac2-fa36-4b12-81dd-8da468ce7817 + x-bt-internal-trace-id: + - 6a8499d70000000022021e5ffb86ef95 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/dataset/e0753a78-8202-4f97-9b70-8555cca01f24 + response: + body: + string: '{"id":"e0753a78-8202-4f97-9b70-8555cca01f24","project_id":"8bc899ea-301c-43a4-afce-6b7e381f72df","name":"logs","description":"created + through braintrust.init_dataset","created":"2026-08-18T17:43:50.112Z","deleted_at":"2026-08-18T17:43:51.863Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"logs"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:51 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 12aa3fefbdb5e80269e58f34f94a99e8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - d6ETiDAOUN1-g9j0swXy3sjUKwgGvXYyipoctt8ba-hsN2fbn_tjnw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d7-3a24d2ce27a7706424dbd408;Parent=5e73c5475c4834f4;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '339' + etag: + - W/"153-UO4DgSQH0U8+UDcUWyIinD0oz3Q" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz5yGn7oAMEvYg= + x-amzn-RequestId: + - 81b4aacf-3b14-4f7f-9fd5-14bc3df71448 + x-bt-internal-trace-id: + - 6a8499d700000000701a78a693635ec0 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/project/8bc899ea-301c-43a4-afce-6b7e381f72df + response: + body: + string: '{"id":"8bc899ea-301c-43a4-afce-6b7e381f72df","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-high-level-datasets-vcr","description":null,"created":"2026-08-18T17:43:49.805Z","deleted_at":"2026-08-18T17:43:52.132Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 17:43:52 GMT + Via: + - 1.1 5785adb181f17ab60069f54a29dd7b7a.cloudfront.net (CloudFront), 1.1 cdd327922be1fd75b18f2ae0982269cc.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - U51YfeDrE-5_Jkz2ZpnVq727hQfDx52cj4108zsOyeHDBJ-w8IVk8g== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a8499d8-59546c7c295f23b460a06a06;Parent=0bc589c0f6c8a18e;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '298' + etag: + - W/"12a-tOzoIlwgSySoy4LcJgaHwSCW/OE" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CTz51HFRoAMEmdg= + x-amzn-RequestId: + - c1f642da-a5b3-46b8-8758-bf3db4876f90 + x-bt-internal-trace-id: + - 6a8499d80000000050d08549862804c8 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/client.py b/py/src/braintrust/api/client.py index b77c8b87..6a6e00e2 100644 --- a/py/src/braintrust/api/client.py +++ b/py/src/braintrust/api/client.py @@ -154,10 +154,12 @@ def from_transport( return client def _initialize_services(self, api_key: str) -> None: + from ._generated.datasets import DatasetsAPI from ._generated.experiments import ExperimentsAPI from ._generated.projects import ProjectsAPI self.api_key = api_key + self.datasets = DatasetsAPI(self.transport, self.router, api_key) self.experiments = ExperimentsAPI(self.transport, self.router, api_key) self.projects = ProjectsAPI(self.transport, self.router, api_key) diff --git a/py/src/braintrust/api/conftest.py b/py/src/braintrust/api/conftest.py new file mode 100644 index 00000000..62baa26c --- /dev/null +++ b/py/src/braintrust/api/conftest.py @@ -0,0 +1,8 @@ +import os + +import pytest + + +@pytest.fixture +def api_key() -> str: + return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") diff --git a/py/src/braintrust/api/test_datasets.py b/py/src/braintrust/api/test_datasets.py new file mode 100644 index 00000000..4dca95bc --- /dev/null +++ b/py/src/braintrust/api/test_datasets.py @@ -0,0 +1,206 @@ +import contextlib +import json + +import braintrust +import pytest +from braintrust.api import BraintrustClient, BraintrustOpenApiClient +from braintrust.api._generated.datasets import OPERATIONS +from braintrust.api._test_server import scripted_server +from braintrust.api.policies import RetryMode +from braintrust.api.types import InsertDatasetEventRequest + + +def test_all_dataset_operations_have_complete_retry_classification(): + assert {name: operation.retry_mode for name, operation in OPERATIONS.items()} == { + "postDataset": RetryMode.IDEMPOTENT_WRITE, + "getDataset": RetryMode.SAFE_READ, + "getDatasetId": RetryMode.SAFE_READ, + "patchDatasetId": RetryMode.NONE, + "deleteDatasetId": RetryMode.NONE, + "postDatasetIdInsert": RetryMode.NONE, + "postDatasetIdFetch": RetryMode.SAFE_READ, + "getDatasetIdFetch": RetryMode.SAFE_READ, + "postDatasetIdFeedback": RetryMode.NONE, + "getDatasetIdSummarize": RetryMode.SAFE_READ, + } + + +def test_dataset_insert_preserves_underscore_prefixed_wire_keys(): + body: InsertDatasetEventRequest = { + "events": [ + { + "id": "row-id", + "_is_merge": True, + "_merge_paths": [["input"]], + "_array_delete": [{"path": ["tags"], "delete": ["old"]}], + "_object_delete": True, + "_parent_id": "parent-id", + } + ] + } + with scripted_server([(200, {"Content-Type": "application/json"}, b'{"row_ids":["row-id"]}')]) as ( + api_url, + handler, + ): + with BraintrustOpenApiClient(api_key="test-key", api_url=api_url) as client: + client.datasets.post_dataset_id_insert("dataset-id", body=body) + + assert handler.requests[0][:2] == ("POST", "/v1/dataset/dataset-id/insert") + assert json.loads(handler.requests[0][2]) == body + + +@pytest.mark.vcr +def test_datasets_end_to_end_with_real_backend(api_key): + project_name = "python-sdk-generated-datasets-vcr" + dataset_name = "generated-datasets-api" + event_id = "generated-datasets-row" + + with BraintrustClient(api_key=api_key) as client: + cleanup_project_id = None + cleanup_dataset_id = None + try: + discovery = client.auth.login() + project = client.openapi.projects.post_project( + body={"name": project_name, "org_name": discovery.organization.name} + ) + cleanup_project_id = project["id"] + created = client.openapi.datasets.post_dataset( + body={ + "project_id": project["id"], + "name": dataset_name, + "description": "created by the Python SDK VCR test", + } + ) + cleanup_dataset_id = created["id"] + listed = client.openapi.datasets.get_dataset( + dataset_name=dataset_name, + project_id=project["id"], + ) + fetched = client.openapi.datasets.get_dataset_id(created["id"]) + updated = client.openapi.datasets.patch_dataset_id( + created["id"], body={"description": "updated by the Python SDK VCR test"} + ) + inserted = client.openapi.datasets.post_dataset_id_insert( + created["id"], + body={ + "events": [ + { + "id": event_id, + "input": { + "question": "What is the answer?", + "context": {"preserved": True}, + "replace_me": {"old": True}, + }, + "expected": "42", + "tags": ["keep", "remove"], + } + ] + }, + ) + merged = client.openapi.datasets.post_dataset_id_insert( + created["id"], + body={ + "events": [ + { + "id": event_id, + "_is_merge": True, + "_merge_paths": [["input", "replace_me"]], + "_array_delete": [{"path": ["tags"], "delete": ["remove"]}], + "input": { + "question": "What is the updated answer?", + "replace_me": {"new": True}, + }, + } + ] + }, + ) + fetched_post = client.openapi.datasets.post_dataset_id_fetch(created["id"], body={"limit": 10}) + fetched_get = client.openapi.datasets.get_dataset_id_fetch(created["id"], limit=10) + feedback = client.openapi.datasets.post_dataset_id_feedback( + created["id"], body={"feedback": [{"id": event_id, "comment": "useful example"}]} + ) + summary = client.openapi.datasets.get_dataset_id_summarize(created["id"], summarize_data=True) + removed = client.openapi.datasets.post_dataset_id_insert( + created["id"], body={"events": [{"id": event_id, "_object_delete": True}]} + ) + fetched_after_delete = client.openapi.datasets.post_dataset_id_fetch(created["id"], body={"limit": 10}) + deleted = client.openapi.datasets.delete_dataset_id(created["id"]) + cleanup_dataset_id = None + client.openapi.projects.delete_project_id(project["id"]) + cleanup_project_id = None + finally: + if cleanup_dataset_id is not None: + with contextlib.suppress(Exception): + client.openapi.datasets.delete_dataset_id(cleanup_dataset_id) + if cleanup_project_id is not None: + with contextlib.suppress(Exception): + client.openapi.projects.delete_project_id(cleanup_project_id) + + assert created["name"] == dataset_name + assert [dataset["id"] for dataset in listed["objects"]] == [created["id"]] + assert fetched["id"] == created["id"] + assert updated["description"] == "updated by the Python SDK VCR test" + assert inserted["row_ids"] == [event_id] + assert merged["row_ids"] == [event_id] + assert [event["id"] for event in fetched_post["events"]] == [event_id] + assert fetched_post["events"][0]["input"] == { + "question": "What is the updated answer?", + "context": {"preserved": True}, + "replace_me": {"new": True}, + } + assert fetched_post["events"][0]["tags"] == ["keep"] + assert [event["id"] for event in fetched_get["events"]] == [event_id] + assert feedback == {"status": "success"} + assert summary["project_name"] == project_name + assert summary["dataset_name"] == dataset_name + assert removed["row_ids"] == [event_id] + assert fetched_after_delete["events"] == [] + assert deleted["id"] == created["id"] + + +@pytest.mark.vcr +def test_high_level_dataset_uses_generated_resources(api_key): + project_name = "python-sdk-high-level-datasets-vcr" + event_id = "high-level-generated-row" + cleanup_project_id = None + cleanup_dataset_id = None + + dataset = braintrust.init_dataset( + project=project_name, + description="created through braintrust.init_dataset", + api_key=api_key, + use_output=False, + ) + try: + cleanup_dataset_id = dataset.id + cleanup_project_id = dataset.project.id + api_client = dataset.state.api_client() + api_client.datasets.post_dataset_id_insert( + dataset.id, + body={ + "events": [ + { + "id": event_id, + "input": {"question": "What is the answer?"}, + "expected": "42", + } + ] + }, + ) + + events = list(dataset.fetch(batch_size=10)) + summary = dataset.summarize() + + assert [event["id"] for event in events] == [event_id] + assert summary.project_name == project_name + assert dataset.name == "logs" + assert summary.dataset_name == "logs" + assert summary.data_summary is not None + assert summary.data_summary.total_records == 1 + finally: + if cleanup_dataset_id is not None: + with contextlib.suppress(Exception): + dataset.state.api_client().datasets.delete_dataset_id(cleanup_dataset_id) + if cleanup_project_id is not None: + with contextlib.suppress(Exception): + dataset.state.api_client().projects.delete_project_id(cleanup_project_id) diff --git a/py/src/braintrust/api/test_experiments.py b/py/src/braintrust/api/test_experiments.py index 16f31caa..a882c1ce 100644 --- a/py/src/braintrust/api/test_experiments.py +++ b/py/src/braintrust/api/test_experiments.py @@ -1,5 +1,3 @@ -import os - import braintrust import pytest from braintrust.api._generated.experiments import OPERATIONS @@ -35,18 +33,14 @@ def test_all_experiment_operations_have_complete_retry_classification(): } -def _api_key(): - return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") - - @pytest.mark.vcr @pytest.mark.parametrize("explicit_comparison", [False, True]) -def test_experiment_summarize_with_real_backend(explicit_comparison): +def test_experiment_summarize_with_real_backend(explicit_comparison, api_key): project_name = "python-sdk-generated-experiments-vcr" base = braintrust.init( project=project_name, experiment="generated-experiments-base", - api_key=_api_key(), + api_key=api_key, update=True, set_current=False, repo_info=RepoInfo(), @@ -54,7 +48,7 @@ def test_experiment_summarize_with_real_backend(explicit_comparison): candidate = braintrust.init( project=project_name, experiment="generated-experiments-candidate", - api_key=_api_key(), + api_key=api_key, base_experiment_id=base.id, update=True, set_current=False, diff --git a/py/src/braintrust/api/test_generated_models.py b/py/src/braintrust/api/test_generated_models.py index 329b318f..4b3562ac 100644 --- a/py/src/braintrust/api/test_generated_models.py +++ b/py/src/braintrust/api/test_generated_models.py @@ -19,14 +19,18 @@ def test_import_braintrust_is_lazy_about_generated_api_modules(): def test_generated_models_import_on_supported_python(): + from braintrust.api._generated import datasets as dataset_bindings from braintrust.api._generated import experiments as experiment_bindings from braintrust.api._generated import models from braintrust.api._generated import projects as project_bindings + assert is_typeddict(models.Dataset) assert is_typeddict(models.Experiment) assert is_typeddict(models.Project) + assert models.DatasetIdParam is str assert models.ExperimentIdParam is str assert models.ProjectIdParam is str + assert get_type_hints(dataset_bindings.DatasetsAPI.get_dataset)["return"] is models.GetDatasetResponse assert get_type_hints(experiment_bindings.ExperimentsAPI.get_experiment)["return"] is models.GetExperimentResponse assert get_type_hints(project_bindings.ProjectsAPI.get_project)["return"] is models.GetProjectResponse @@ -37,8 +41,10 @@ def test_generated_package_content_is_installed(): assert generated.joinpath("__init__.py").is_file() assert generated.joinpath("models", "__init__.py").is_file() assert generated.joinpath("models", "common.py").is_file() + assert generated.joinpath("models", "datasets.py").is_file() assert generated.joinpath("models", "experiments.py").is_file() assert generated.joinpath("models", "projects.py").is_file() + assert generated.joinpath("datasets.py").is_file() assert generated.joinpath("experiments.py").is_file() assert generated.joinpath("projects.py").is_file() @@ -49,4 +55,4 @@ def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): overlap = set(generated_types.__all__) & set(types.__all__) - assert overlap == {"Experiment", "Project"} + assert overlap == {"Dataset", "Experiment", "Project"} diff --git a/py/src/braintrust/api/test_projects.py b/py/src/braintrust/api/test_projects.py index a813755d..aaa53c97 100644 --- a/py/src/braintrust/api/test_projects.py +++ b/py/src/braintrust/api/test_projects.py @@ -1,22 +1,16 @@ -import os - import pytest from braintrust.api import BraintrustClient -def _api_key(): - return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") - - @pytest.mark.vcr -def test_projects_end_to_end_with_real_backend(): +def test_projects_end_to_end_with_real_backend(api_key): project_name = "python-sdk-generated-projects-vcr" create_project = { "name": project_name, "description": "created by the Python SDK VCR test", } - with BraintrustClient(api_key=_api_key()) as client: + with BraintrustClient(api_key=api_key) as client: discovery = client.auth.login() create_project["org_name"] = discovery.organization.name created = client.openapi.projects.post_project(body=create_project) diff --git a/py/src/braintrust/api/types/__init__.py b/py/src/braintrust/api/types/__init__.py index ac3d4b75..fdacfe8b 100644 --- a/py/src/braintrust/api/types/__init__.py +++ b/py/src/braintrust/api/types/__init__.py @@ -1,38 +1,54 @@ """Public types for the synchronous Braintrust REST API.""" from .._generated.models import ( + CreateDataset, CreateExperiment, CreateProject, + Dataset, Experiment, + FeedbackDatasetEventRequest, FeedbackExperimentEventRequest, FeedbackResponseSchema, + FetchDatasetEventsResponse, FetchEventsRequest, FetchExperimentEventsResponse, + GetDatasetResponse, GetExperimentResponse, GetProjectResponse, + InsertDatasetEventRequest, InsertEventsResponse, InsertExperimentEventRequest, + PatchDataset, PatchExperiment, PatchProject, Project, + SummarizeDatasetResponse, SummarizeExperimentResponse, ) __all__ = [ + "CreateDataset", "CreateExperiment", "CreateProject", + "Dataset", "Experiment", + "FeedbackDatasetEventRequest", "FeedbackExperimentEventRequest", "FeedbackResponseSchema", + "FetchDatasetEventsResponse", "FetchEventsRequest", "FetchExperimentEventsResponse", + "GetDatasetResponse", "GetExperimentResponse", "GetProjectResponse", + "InsertDatasetEventRequest", "InsertEventsResponse", "InsertExperimentEventRequest", + "PatchDataset", "PatchExperiment", "PatchProject", "Project", + "SummarizeDatasetResponse", "SummarizeExperimentResponse", ] diff --git a/py/src/braintrust/cassettes/test_dataset_internal_btql_limit_caps_total_results.yaml b/py/src/braintrust/cassettes/test_dataset_internal_btql_limit_caps_total_results.yaml index 7144cf4a..1b58d4d4 100644 --- a/py/src/braintrust/cassettes/test_dataset_internal_btql_limit_caps_total_results.yaml +++ b/py/src/braintrust/cassettes/test_dataset_internal_btql_limit_caps_total_results.yaml @@ -33,7 +33,7 @@ interactions: - '374' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-YmMyZTM0NWYtYzJmZS00MzQyLThlOTItOTViMTgwMmUyNzg5'' *.js.stripe.com + ''nonce-YmE2NDk4N2ItOGFiMi00ZTk2LWI3M2EtZTZlOGE1Y2Q0NDI5'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com @@ -45,7 +45,7 @@ interactions: Content-Type: - application/json; charset=utf-8 Date: - - Wed, 05 Aug 2026 14:55:30 GMT + - Tue, 18 Aug 2026 17:32:55 GMT Etag: - '"13kwty4qcjgae"' Reporting-Endpoints: @@ -54,6 +54,8 @@ interactions: - Vercel Strict-Transport-Security: - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' X-Clerk-Auth-Message: - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, token-carrier=header) @@ -68,17 +70,16 @@ interactions: X-Matched-Path: - /api/apikey/login X-Nonce: - - YmMyZTM0NWYtYzJmZS00MzQyLThlOTItOTViMTgwMmUyNzg5 + - YmE2NDk4N2ItOGFiMi00ZTk2LWI3M2EtZTZlOGE1Y2Q0NDI5 X-Vercel-Cache: - MISS X-Vercel-Id: - - yul1::iad1::p2wth-1785941730640-7c7991ba6747 + - yul1::iad1::m5dkt-1787074375027-e16770f80a4d status: code: 200 message: OK - request: - body: '{"project_name": "python-sdk-vcr-tests", "project_id": null, "org_id": - "f5883013-b5c1-438d-9044-5182b4682337", "dataset_name": "test-dataset-internal-btql-total-limit"}' + body: '{"name": "python-sdk-vcr-tests", "org_name": "abhi-test-org"}' headers: Accept: - '*/*' @@ -87,99 +88,33 @@ interactions: Connection: - keep-alive Content-Length: - - '168' + - '61' Content-Type: - application/json User-Agent: - python-requests/2.34.2 method: POST - uri: https://www.braintrust.dev/api/dataset/register - response: - body: - string: '{"project":{"id":"5740905a-baf8-4edf-b634-8aa62ea149d3","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-vcr-tests","description":null,"created":"2026-08-05T14:55:31.022Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"dataset":{"id":"93e21c91-a4a1-441f-9ba5-91331b414da4","project_id":"5740905a-baf8-4edf-b634-8aa62ea149d3","name":"test-dataset-internal-btql-total-limit","description":null,"created":"2026-08-05T14:55:31.022Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"test-dataset-internal-btql-total-limit"},"found_existing":false}' - headers: - Cache-Control: - - public, max-age=0, must-revalidate - Content-Length: - - '656' - Content-Security-Policy: - - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZTZjODczYWQtN2Y3OS00OWQ2LThiNzItMmM3M2E0MmEwMGNk'' *.js.stripe.com - js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com - d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com - cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net - fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' - https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; - report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; - report-to csp-endpoint-0' - Content-Type: - - application/json; charset=utf-8 - Date: - - Wed, 05 Aug 2026 14:55:31 GMT - Etag: - - '"12sv3og5gzpi8"' - Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" - Server: - - Vercel - Strict-Transport-Security: - - max-age=63072000 - X-Clerk-Auth-Message: - - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, - token-carrier=header) - X-Clerk-Auth-Reason: - - token-invalid - X-Clerk-Auth-Status: - - signed-out - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-Matched-Path: - - /api/dataset/register - X-Nonce: - - ZTZjODczYWQtN2Y3OS00OWQ2LThiNzItMmM3M2E0MmEwMGNk - X-Vercel-Cache: - - MISS - X-Vercel-Id: - - yul1::iad1::xn7p5-1785941730906-ade6974e689c - status: - code: 200 - message: OK -- request: - body: null - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - User-Agent: - - python-requests/2.34.2 - method: GET - uri: https://api.braintrust.dev/version + uri: https://api.braintrust.dev/v1/project response: body: - string: '{"version":"2.9.0","date_version":"20260730","ff_version":42,"commit":"752127a4257d43c647e4f26fca64445645b9a982","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + string: '{"id":"5740905a-baf8-4edf-b634-8aa62ea149d3","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"python-sdk-vcr-tests","description":null,"created":"2026-08-05T14:55:31.022Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' headers: Connection: - keep-alive Content-Type: - application/json; charset=utf-8 Date: - - Wed, 05 Aug 2026 14:55:31 GMT + - Tue, 18 Aug 2026 17:32:55 GMT Via: - - 1.1 8757e4d26d0f26e2f05769a88e8a5ace.cloudfront.net (CloudFront), 1.1 8e6145785e47042f882be946f6c05880.cloudfront.net + - 1.1 cf658ee9f945547220eb3f4e1f7ef2fe.cloudfront.net (CloudFront), 1.1 3340b5a392e45fce453c4d978abfd6be.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - nP5Yi5R8JpBCfrXan9JJG9r1N2L14jQ1-zOVH-AaCkUpUCoOjxUCSg== + - UGZIuvdtW-NSNAfcLrNjjA7oBycJRK2Haq2xN1v6NjHFdihNp4b7Iw== X-Amz-Cf-Pop: - YTO53-P2 - YTO50-P2 X-Amzn-Trace-Id: - - Root=1-6a734ee3-4ae6d8fd1361006d0a7a678c;Parent=71d1c6d6ad287dc5;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6a849747-642adf1a76ee5d0026dc2c9c;Parent=3dc2fc5da29f632b;Sampled=0;Lineage=1:24be3d11:0 X-Cache: - Miss from cloudfront access-control-allow-credentials: @@ -189,33 +124,28 @@ interactions: cache-control: - no-store, no-cache, must-revalidate, proxy-revalidate content-length: - - '557' + - '262' etag: - - W/"22d-ctakgszqYQZk3xpUQtT3XtS3QV4" + - W/"106-sw4f3jJYmmtrfBUZ4IeAvOYj8tQ" expires: - '0' surrogate-control: - no-store vary: - - Origin + - Origin, Accept-Encoding x-amz-apigw-id: - - BolDlH1JoAMEJbg= - x-amzn-Remapped-content-length: - - '557' + - CTyTOECToAMETzg= x-amzn-RequestId: - - d1977231-4402-423f-aaa3-26879c5f8257 + - 5050b7c6-49f8-4c94-8cf0-a1846204b31d + x-bt-found-existing: + - 'true' x-bt-internal-trace-id: - - 6a734ee3000000003adf69420fe9bf21 + - 6a849747000000001547ff41cb55072d status: code: 200 message: OK - request: - body: '{"rows": [{"created": "2026-08-05T14:55:30.472778+00:00", "dataset_id": - "93e21c91-a4a1-441f-9ba5-91331b414da4", "expected": "first", "id": "internal-btql-limit-record-1", - "input": "first", "tags": null},{"created": "2026-08-05T14:55:30.473043+00:00", - "dataset_id": "93e21c91-a4a1-441f-9ba5-91331b414da4", "expected": "second", - "id": "internal-btql-limit-record-2", "input": "second", "tags": null}], "api_version": - 2}' + body: '{"project_id": "5740905a-baf8-4edf-b634-8aa62ea149d3", "name": "test-dataset-internal-btql-total-limit"}' headers: Accept: - '*/*' @@ -224,31 +154,33 @@ interactions: Connection: - keep-alive Content-Length: - - '417' + - '104' + Content-Type: + - application/json User-Agent: - python-requests/2.34.2 method: POST - uri: https://api.braintrust.dev/logs3 + uri: https://api.braintrust.dev/v1/dataset response: body: - string: '{"ids":["internal-btql-limit-record-1","internal-btql-limit-record-2"],"xact_id":"1000197635730255173"}' + string: '{"id":"93e21c91-a4a1-441f-9ba5-91331b414da4","project_id":"5740905a-baf8-4edf-b634-8aa62ea149d3","name":"test-dataset-internal-btql-total-limit","description":null,"created":"2026-08-05T14:55:31.022Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","tags":null,"metadata":null,"url_slug":"test-dataset-internal-btql-total-limit"}' headers: Connection: - keep-alive Content-Type: - application/json; charset=utf-8 Date: - - Wed, 05 Aug 2026 14:55:32 GMT + - Tue, 18 Aug 2026 17:32:55 GMT Via: - - 1.1 8757e4d26d0f26e2f05769a88e8a5ace.cloudfront.net (CloudFront), 1.1 71eaa9eb77c2eecb57c03cdcdad1cf76.cloudfront.net + - 1.1 cf658ee9f945547220eb3f4e1f7ef2fe.cloudfront.net (CloudFront), 1.1 3340b5a392e45fce453c4d978abfd6be.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - 3Q1Ri4YPhZs10OPexGrLj3oZpnEyruxPHgHfhYdt-w4op8YZCOPZnA== + - SeDNoJvWvd2bDa7O4pQyvjMvdb_J_fCdaZsxL3mrWXPzN1p04dFABg== X-Amz-Cf-Pop: - YTO53-P2 - YTO50-P2 X-Amzn-Trace-Id: - - Root=1-6a734ee3-6c9750aa4626ef89322574f2;Parent=64344bf0f42f2bd2;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6a849747-1a97f393087ea4c57c05588b;Parent=2d8b0875a5d4d8b9;Sampled=0;Lineage=1:24be3d11:0 X-Cache: - Miss from cloudfront access-control-allow-credentials: @@ -258,9 +190,9 @@ interactions: cache-control: - no-store, no-cache, must-revalidate, proxy-revalidate content-length: - - '103' + - '348' etag: - - W/"67-slrO8h6R7bzoArJaQMDL+SPIzYk" + - W/"15c-TfW7o/HENJkX+ZuBPa/y33dkY8k" expires: - '0' surrogate-control: @@ -268,88 +200,49 @@ interactions: vary: - Origin, Accept-Encoding x-amz-apigw-id: - - BolDnHiqIAMEDlg= + - CTyTQEZToAMEZKQ= x-amzn-RequestId: - - bb6ca664-bf55-422e-9f3b-d776b45d5a7d + - b2388e01-779b-4d01-afc3-b3a90038d3cc + x-bt-found-existing: + - 'true' x-bt-internal-trace-id: - - 6a734ee3000000001f5cc1b071725c11 + - 6a849747000000002061da26ff721be7 status: code: 200 message: OK - request: - body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": - {"op": "ident", "name": ["dataset"]}, "args": [{"op": "literal", "value": "93e21c91-a4a1-441f-9ba5-91331b414da4"}]}, - "cursor": null, "limit": 1}, "use_columnstore": false, "brainstore_realtime": - true, "query_source": "py_sdk_object_fetcher_dataset"}' + body: null headers: Accept: - '*/*' Accept-Encoding: - - gzip + - gzip, deflate, zstd Connection: - keep-alive - Content-Length: - - '323' - Content-Type: - - application/json User-Agent: - python-requests/2.34.2 - method: POST - uri: https://api.braintrust.dev/btql + method: GET + uri: https://api.braintrust.dev/version response: body: - string: '{"data":[{"_pagination_key":"p07670561327966126081","_xact_id":"1000197635730255173","audit_data":[{"_xact_id":"1000197635730255173","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-05T14:55:30.473Z","dataset_id":"93e21c91-a4a1-441f-9ba5-91331b414da4","expected":"second","facets":null,"id":"internal-btql-limit-record-2","input":"second","is_root":true,"metadata":null,"origin":null,"project_id":"5740905a-baf8-4edf-b634-8aa62ea149d3","root_span_id":"6a9a2a70-7597-48c7-affc-375126066bc2","span_id":"6a9a2a70-7597-48c7-affc-375126066bc2","tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A - stable, time-ordered key that can be used to paginate over dataset events. - This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The - transaction id of an event is unique to the network operation that processed - the event insertion. Transaction ids are monotonically increasing over time - and can be used to retrieve a versioned snapshot of the dataset (see the `version` - parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional - confidence score for the classification","type":["number","null"]},"id":{"description":"Stable - classification identifier","type":"string"},"label":{"description":"Original - label of the classification item, which is useful for search and indexing - purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional - metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The - version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The - type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional - function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"created":{"description":"The - timestamp the dataset event was created","format":"date-time","type":"string"},"dataset_id":{"description":"Unique - identifier for the dataset","format":"uuid","type":"string"},"expected":{"description":"The - output of your application, including post-processing (an arbitrary, JSON - serializable object)"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A - unique identifier for the dataset event. If you don''t provide one, Braintrust - will generate one for you","type":"string"},"input":{"description":"The argument - that uniquely define an input case (an arbitrary, JSON serializable object)"},"is_root":{"description":"Whether - this span is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The - model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference - to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction - ID of the original event.","type":["string","null"]},"created":{"description":"Created - timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID - of the original event.","type":"string"},"object_id":{"description":"ID of - the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type - of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"project_id":{"description":"Unique - identifier for the project that the dataset belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A - unique identifier for the trace this dataset event belongs to","type":"string"},"span_id":{"description":"A - unique identifier used to link different dataset events together as part of - a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) - for full details on tracing","type":"string"},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"anNO4y1FAAE","realtime_state":{"type":"on","minimum_xact_id":null,"read_bytes":1356,"actual_xact_id":"1000197635730255173"},"freshness_state":{"last_processed_xact_id":null,"last_considered_xact_id":"1000197635730255173"},"warnings":[]}' + string: '{"version":"2.11.0","date_version":"20260817","ff_version":46,"commit":"bb8ea042b06e943817298c9fc56a802148e024a4","deployment_mode":"lambda","deployment_type":"custom","loop_runtime_enabled":true,"brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' headers: Connection: - keep-alive Content-Type: - - application/json + - application/json; charset=utf-8 Date: - - Wed, 05 Aug 2026 14:55:32 GMT + - Tue, 18 Aug 2026 17:32:55 GMT Via: - - 1.1 8250156022879efefd7a589c8ba8c706.cloudfront.net (CloudFront), 1.1 0e761f7a5b2481acd893422a702c9fa8.cloudfront.net + - 1.1 dee47bbe1fe762707eeae94e76255170.cloudfront.net (CloudFront), 1.1 5a2f8eb373b5a17b769c0fee9b0725a6.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - a9P0CyqWFCX1nbieSCP2r9FWONXVEP4r2RuFfMu_60I6KH3ewQC2-A== + - NmhHLud0kQaG17s5jK2W7FFXyL6OZuZ9ND2oOnSvC9x8u8isVWAxeQ== X-Amz-Cf-Pop: - YTO53-P2 - YTO50-P2 X-Amzn-Trace-Id: - - Root=1-6a734ee4-19a429a8006a9c7703caf496;Parent=4a2aaa6030920b78;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6a849747-75fb864528deb0e6494bedd6;Parent=6f163d63df18fe1b;Sampled=0;Lineage=1:24be3d11:0 X-Cache: - Miss from cloudfront access-control-allow-credentials: @@ -357,100 +250,68 @@ interactions: access-control-expose-headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url cache-control: - - private, no-cache + - no-store, no-cache, must-revalidate, proxy-revalidate content-length: - - '5300' + - '586' + etag: + - W/"24a-yjnAH/Jq2lSD3sfOGxkgOHgzOK0" + expires: + - '0' + surrogate-control: + - no-store vary: - Origin x-amz-apigw-id: - - BolDvFo2IAMEvtQ= + - CTyTSHuRoAMEDdA= + x-amzn-Remapped-content-length: + - '586' x-amzn-RequestId: - - 642cf8b4-67d2-4084-9499-5be8b5441123 - x-bt-api-duration-ms: - - '121' - x-bt-brainstore-duration-ms: - - '79' - x-bt-cursor: - - anNO4y1FAAE + - 34224fc0-7829-47f2-9d84-b394abc21088 x-bt-internal-trace-id: - - 6a734ee40000000005540f09408671f8 + - 6a84974700000000048c3830b68fe4c5 status: code: 200 message: OK - request: - body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": - {"op": "ident", "name": ["dataset"]}, "args": [{"op": "literal", "value": "93e21c91-a4a1-441f-9ba5-91331b414da4"}]}, - "cursor": "anNO4y1FAAE", "limit": 1}, "use_columnstore": false, "brainstore_realtime": - true, "query_source": "py_sdk_object_fetcher_dataset"}' + body: '{"rows": [{"created": "2026-08-18T17:32:54.876745+00:00", "dataset_id": + "93e21c91-a4a1-441f-9ba5-91331b414da4", "expected": "first", "id": "internal-btql-limit-record-1", + "input": "first", "tags": null},{"created": "2026-08-18T17:32:54.877013+00:00", + "dataset_id": "93e21c91-a4a1-441f-9ba5-91331b414da4", "expected": "second", + "id": "internal-btql-limit-record-2", "input": "second", "tags": null}], "api_version": + 2}' headers: Accept: - '*/*' Accept-Encoding: - - gzip + - gzip, deflate, zstd Connection: - keep-alive Content-Length: - - '332' - Content-Type: - - application/json + - '417' User-Agent: - python-requests/2.34.2 method: POST - uri: https://api.braintrust.dev/btql + uri: https://api.braintrust.dev/logs3 response: body: - string: '{"data":[{"_pagination_key":"p07670561327966126080","_xact_id":"1000197635730255173","audit_data":[{"_xact_id":"1000197635730255173","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-05T14:55:30.472Z","dataset_id":"93e21c91-a4a1-441f-9ba5-91331b414da4","expected":"first","facets":null,"id":"internal-btql-limit-record-1","input":"first","is_root":true,"metadata":null,"origin":null,"project_id":"5740905a-baf8-4edf-b634-8aa62ea149d3","root_span_id":"a540f0e9-c6e3-4e32-bff3-9b382db1f1be","span_id":"a540f0e9-c6e3-4e32-bff3-9b382db1f1be","tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A - stable, time-ordered key that can be used to paginate over dataset events. - This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The - transaction id of an event is unique to the network operation that processed - the event insertion. Transaction ids are monotonically increasing over time - and can be used to retrieve a versioned snapshot of the dataset (see the `version` - parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional - confidence score for the classification","type":["number","null"]},"id":{"description":"Stable - classification identifier","type":"string"},"label":{"description":"Original - label of the classification item, which is useful for search and indexing - purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional - metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The - version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The - type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional - function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"created":{"description":"The - timestamp the dataset event was created","format":"date-time","type":"string"},"dataset_id":{"description":"Unique - identifier for the dataset","format":"uuid","type":"string"},"expected":{"description":"The - output of your application, including post-processing (an arbitrary, JSON - serializable object)"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A - unique identifier for the dataset event. If you don''t provide one, Braintrust - will generate one for you","type":"string"},"input":{"description":"The argument - that uniquely define an input case (an arbitrary, JSON serializable object)"},"is_root":{"description":"Whether - this span is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The - model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference - to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction - ID of the original event.","type":["string","null"]},"created":{"description":"Created - timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID - of the original event.","type":"string"},"object_id":{"description":"ID of - the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type - of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"project_id":{"description":"Unique - identifier for the project that the dataset belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A - unique identifier for the trace this dataset event belongs to","type":"string"},"span_id":{"description":"A - unique identifier used to link different dataset events together as part of - a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) - for full details on tracing","type":"string"},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"anNO4y1FAAA","realtime_state":{"type":"on","minimum_xact_id":null,"read_bytes":1356,"actual_xact_id":"1000197635730255173"},"freshness_state":{"last_processed_xact_id":null,"last_considered_xact_id":"1000197635730255173"},"warnings":[]}' + string: '{"ids":["internal-btql-limit-record-1","internal-btql-limit-record-2"],"xact_id":"1000197709959278726"}' headers: Connection: - keep-alive Content-Type: - - application/json + - application/json; charset=utf-8 Date: - - Wed, 05 Aug 2026 14:55:32 GMT + - Tue, 18 Aug 2026 17:32:56 GMT Via: - - 1.1 8757e4d26d0f26e2f05769a88e8a5ace.cloudfront.net (CloudFront), 1.1 cdd327922be1fd75b18f2ae0982269cc.cloudfront.net + - 1.1 e7dbdec0a9983cf82c16b332b6b30812.cloudfront.net (CloudFront), 1.1 74797197cacba7d22a7c3a7685b38272.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - p8i4xVqYm9RbA0FjIvFUo7PEYMQLp_haerqQT7wrHTIm_i7bPCy1kA== + - UJOGlJK3GAoGjz9aqsr4RnoFpVKmf8rmaToblsBh6atrxgnHMxQL3Q== X-Amz-Cf-Pop: - YTO53-P2 - YTO50-P2 X-Amzn-Trace-Id: - - Root=1-6a734ee4-53a6e2be76ca6ad941fffd56;Parent=5d0127a9aa8cff18;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6a849747-205c91a95a4abf2b7a92c77d;Parent=78b86585b93f11cd;Sampled=0;Lineage=1:24be3d11:0 X-Cache: - Miss from cloudfront access-control-allow-credentials: @@ -458,30 +319,30 @@ interactions: access-control-expose-headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url cache-control: - - private, no-cache + - no-store, no-cache, must-revalidate, proxy-revalidate content-length: - - '5298' + - '103' + etag: + - W/"67-JJYZnmsGxepipGA+8Rqd9t2xGM8" + expires: + - '0' + surrogate-control: + - no-store vary: - - Origin + - Origin, Accept-Encoding x-amz-apigw-id: - - BolDyF6moAMER_Q= + - CTyTUFxaIAMEVtQ= x-amzn-RequestId: - - 2c872d8c-963c-470c-954b-da5fb2ead97a - x-bt-api-duration-ms: - - '231' - x-bt-brainstore-duration-ms: - - '184' - x-bt-cursor: - - anNO4y1FAAA + - 00860d45-27c2-4ec9-9570-cb031c745732 x-bt-internal-trace-id: - - 6a734ee40000000044a5247423f04fb1 + - 6a849747000000006836f82150a2548a status: code: 200 message: OK - request: body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": {"op": "ident", "name": ["dataset"]}, "args": [{"op": "literal", "value": "93e21c91-a4a1-441f-9ba5-91331b414da4"}]}, - "cursor": "anNO4y1FAAA", "limit": 1}, "use_columnstore": false, "brainstore_realtime": + "cursor": null, "limit": 1}, "use_columnstore": false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_dataset"}' headers: Accept: @@ -491,7 +352,7 @@ interactions: Connection: - keep-alive Content-Length: - - '332' + - '323' Content-Type: - application/json User-Agent: @@ -500,7 +361,7 @@ interactions: uri: https://api.braintrust.dev/btql response: body: - string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + string: '{"data":[{"_pagination_key":"p07675426001253695489","_xact_id":"1000197709959278726","audit_data":[{"_xact_id":"1000197709959278726","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-08-18T17:32:54.877Z","dataset_id":"93e21c91-a4a1-441f-9ba5-91331b414da4","expected":"second","facets":null,"id":"internal-btql-limit-record-2","input":"second","is_root":true,"metadata":null,"origin":null,"project_id":"5740905a-baf8-4edf-b634-8aa62ea149d3","root_span_id":"25a9a8fc-0229-42d2-a5cb-704369e76b9f","span_id":"25a9a8fc-0229-42d2-a5cb-704369e76b9f","tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A stable, time-ordered key that can be used to paginate over dataset events. This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The transaction id of an event is unique to the network operation that processed @@ -534,24 +395,24 @@ interactions: unique identifier for the trace this dataset event belongs to","type":"string"},"span_id":{"description":"A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) - for full details on tracing","type":"string"},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":null,"read_bytes":1356,"actual_xact_id":"1000197635730255173"},"freshness_state":{"last_processed_xact_id":null,"last_considered_xact_id":"1000197635730255173"},"warnings":[]}' + for full details on tracing","type":"string"},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"aoSXSDCGAAE","realtime_state":{"type":"on","minimum_xact_id":"1000197709952402389","read_bytes":1356,"actual_xact_id":"1000197709959278726"},"freshness_state":{"last_processed_xact_id":"1000197709952402389","last_considered_xact_id":"1000197709959278726"},"warnings":[]}' headers: Connection: - keep-alive Content-Type: - application/json Date: - - Wed, 05 Aug 2026 14:55:33 GMT + - Tue, 18 Aug 2026 17:32:56 GMT Via: - - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 5fef2688877996791689cf17ab2832d0.cloudfront.net + - 1.1 cf658ee9f945547220eb3f4e1f7ef2fe.cloudfront.net (CloudFront), 1.1 4f3eaee3896fb5ad2377261bd0d773c8.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - OUOs4hNWapakRzFf7h224KR47hCmaRRMs3qEEVcnZER4dP8VbidQgg== + - YnfhFiFv6zThETOkK4-y-oC1pEPwEyv2-1SX69ez6Q8eyO7N_itQPw== X-Amz-Cf-Pop: - YTO53-P2 - YTO50-P2 X-Amzn-Trace-Id: - - Root=1-6a734ee4-51f9e72509d68cc95c0b2cd5;Parent=7f72df8695b1d063;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6a849748-1ab42bad65cc8211562c5d44;Parent=47ddb79078a43246;Sampled=0;Lineage=1:24be3d11:0 X-Cache: - Miss from cloudfront access-control-allow-credentials: @@ -561,19 +422,21 @@ interactions: cache-control: - private, no-cache content-length: - - '4662' + - '5334' vary: - Origin x-amz-apigw-id: - - BolD2Fb1oAMEW-Q= + - CTyTaH3SIAMEVoA= x-amzn-RequestId: - - b295f072-8be4-4466-ada4-d3e5f30dcd50 + - fd16dc66-5727-46d2-b4f0-c1938998ff91 x-bt-api-duration-ms: - - '127' + - '159' x-bt-brainstore-duration-ms: - - '75' + - '99' + x-bt-cursor: + - aoSXSDCGAAE x-bt-internal-trace-id: - - 6a734ee5000000001aab8ad265e6a1d3 + - 6a84974800000000144ad6faf33a0765 status: code: 200 message: OK diff --git a/py/src/braintrust/devserver/dataset.py b/py/src/braintrust/devserver/dataset.py index 8ace8a20..ed8dffd9 100644 --- a/py/src/braintrust/devserver/dataset.py +++ b/py/src/braintrust/devserver/dataset.py @@ -7,18 +7,11 @@ async def get_dataset_by_id(state: BraintrustState, dataset_id: str) -> dict[str, str]: """Fetch dataset information by ID.""" - # Make API call to get dataset info - conn = state.api_conn() - # Note: The Python SDK doesn't have async API calls yet, so we use sync - response = conn.get_json(f"v1/dataset/{dataset_id}") - - if response is None: - raise ValueError(f"Dataset with id {dataset_id} not found") - - # Extract project_id and dataset name from response + # The public client is synchronous; the devserver currently runs this call inline. + response = state.api_client().datasets.get_dataset_id(dataset_id) return { - "project_id": response.get("project_id"), - "dataset": response.get("name"), + "project_id": response["project_id"], + "dataset": response["name"], } diff --git a/py/src/braintrust/devserver/test_dataset.py b/py/src/braintrust/devserver/test_dataset.py index 5da4d0f4..9e69aff1 100644 --- a/py/src/braintrust/devserver/test_dataset.py +++ b/py/src/braintrust/devserver/test_dataset.py @@ -6,6 +6,7 @@ from urllib.parse import parse_qs, urlsplit import pytest +from braintrust.api import BraintrustClient from braintrust.devserver.dataset import get_dataset from braintrust.logger import BraintrustState @@ -29,7 +30,9 @@ def do_GET(self) -> None: self.requests.append(("GET", parsed_url.path, parse_qs(parsed_url.query), None)) if parsed_url.path == "/v1/dataset/dataset-reference": - self._send_json({"project_id": "project-id", "name": "dataset-name"}) + self._send_json({"id": "dataset-reference", "project_id": "project-id", "name": "dataset-name"}) + elif parsed_url.path == "/v1/project/project-id": + self._send_json({"id": "project-id", "name": "project-name"}) elif parsed_url.path == "/environment-object/dataset/dataset-id/prod%2Fstable": self._send_json({"object_version": "2"}) else: @@ -41,13 +44,12 @@ def do_POST(self) -> None: body = json.loads(self.rfile.read(content_length)) self.requests.append(("POST", parsed_url.path, parse_qs(parsed_url.query), body)) - if parsed_url.path == "/api/dataset/register": - self._send_json( - { - "project": {"id": "project-id", "name": "project-name"}, - "dataset": {"id": "dataset-id", "name": "dataset-name"}, - } - ) + if parsed_url.path == "/v1/project": + self._send_json({"id": "project-id", "name": "project-name"}) + elif parsed_url.path == "/v1/dataset": + self._send_json({"id": "dataset-id", "project_id": "project-id", "name": "dataset-name"}) + elif parsed_url.path == "/v1/dataset/dataset-id/fetch": + self._send_json({"events": []}) elif parsed_url.path == "/btql": self._send_json({"data": []}) else: @@ -77,12 +79,15 @@ def _logged_in_state(base_url: str, *, org_name: str | None = "test org") -> Bra state.api_url = base_url state.org_id = "org-id" state.org_name = org_name + state._client = BraintrustClient(api_key="test-key", app_url=base_url, api_url=base_url) return state -def _btql_request_body() -> dict[str, Any]: +def _request_body(path: str) -> dict[str, Any]: return next( - body for method, path, _query, body in _DatasetAPIHandler.requests if method == "POST" and path == "/btql" + body + for method, request_path, _query, body in _DatasetAPIHandler.requests + if method == "POST" and request_path == path ) @@ -123,8 +128,8 @@ async def test_get_dataset_resolves_environment_to_pinned_version( expected_query, None, ) in _DatasetAPIHandler.requests - assert _btql_request_body()["version"] == "2" - assert _btql_request_body()["query"]["limit"] == 10 + assert _request_body("/btql")["version"] == "2" + assert _request_body("/btql")["query"]["limit"] == 10 @pytest.mark.asyncio @@ -151,7 +156,7 @@ async def test_get_dataset_prefers_explicit_version_over_environment( assert not any( path.startswith("/environment-object/") for _method, path, _query, _body in _DatasetAPIHandler.requests ) - assert _btql_request_body()["version"] == "1" + assert _request_body("/v1/dataset/dataset-id/fetch")["version"] == "1" @pytest.mark.asyncio diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 5759902d..3dc88b5f 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -1683,7 +1683,7 @@ def init_dataset( Create a new dataset in a specified project. If the project does not exist, it will be created. :param project_name: The name of the project to create the dataset in. Must specify at least one of `project_name` or `project_id`. - :param name: The name of the dataset to create. If not specified, a name will be generated automatically. + :param name: The name of the dataset to create. Defaults to `logs` if not specified. :param description: An optional description of the dataset. :param version: An optional version of the dataset (to read). If not specified, the latest version will be used. :param environment: The environment to load the dataset from. If both `version` and `environment` are provided, `version` takes precedence. @@ -1711,18 +1711,20 @@ def init_dataset( def compute_metadata(): state.login(org_name=org_name, api_key=api_key, app_url=app_url) - args = _populate_args( - {"project_name": project, "project_id": project_id, "org_id": state.org_id}, - dataset_name=name, + api_client = state.api_client() + if project_id is not None: + resp_project = api_client.projects.get_project_id(project_id) + else: + resp_project = api_client.projects.post_project(body={"name": project, "org_name": state.org_name}) + body = _populate_args( + {"project_id": resp_project["id"], "name": name or "logs"}, description=description, metadata=metadata, ) - response = state.app_conn().post_json("api/dataset/register", args) - resp_project = response["project"] - resp_dataset = response["dataset"] + resp_dataset = api_client.datasets.post_dataset(body=body) return ProjectDatasetMetadata( - project=ObjectMetadata(id=resp_project["id"], name=resp_project["name"], full_info=resp_project), - dataset=ObjectMetadata(id=resp_dataset["id"], name=resp_dataset["name"], full_info=resp_dataset), + project=ObjectMetadata(id=resp_project["id"], name=resp_project["name"], full_info=dict(resp_project)), + dataset=ObjectMetadata(id=resp_dataset["id"], name=resp_dataset["name"], full_info=dict(resp_dataset)), ) return Dataset( @@ -4963,6 +4965,41 @@ def _get_state(self) -> BraintrustState: self._pinned_version = response["object_version"] return self.state + def _refetch(self, batch_size: int | None = None) -> list[DatasetEvent]: + if self._internal_btql: + return super()._refetch(batch_size=batch_size) + if self._fetched_data is not None: + return self._fetched_data + + state = self._get_state() + limit = batch_size if batch_size is not None else DEFAULT_FETCH_BATCH_SIZE + cursor = None + data: list[DatasetEvent] = [] + iterations = 0 + while True: + body: dict[str, Any] = {"limit": limit} + if cursor is not None: + body["cursor"] = cursor + if self._pinned_version is not None: + body["version"] = self._pinned_version + response = state.api_client().datasets.post_dataset_id_fetch(self.id, body=body) + events = response.get("events") + if not isinstance(events, list): + raise ValueError(f"Expected a list in the response, got {type(events)}") + data.extend(cast(list[DatasetEvent], events)) + cursor = response.get("cursor") + if not cursor: + break + iterations += 1 + if iterations > MAX_BTQL_ITERATIONS: + raise RuntimeError("Too many dataset fetch iterations") + + if self._mutate_record is not None: + self._fetched_data = [self._mutate_record(record) for record in data] + else: + self._fetched_data = data + return self._fetched_data + def _validate_event( self, expected: Any | None = None, @@ -5125,19 +5162,16 @@ def summarize(self, summarize_data: bool = True) -> "DatasetSummary": # includes the new experiment. self.flush() state = self._get_state() + response = state.api_client().datasets.get_dataset_id_summarize(self.id, summarize_data=summarize_data) + raw_data_summary = response.get("data_summary") + data_summary = ( + DataSummary(new_records=self.new_records, **raw_data_summary) + if isinstance(raw_data_summary, Mapping) + else None + ) project_url = f"{state.app_public_url}/app/{encode_uri_component(state.org_name)}/p/{encode_uri_component(self.project.name)}" dataset_url = f"{project_url}/datasets/{encode_uri_component(self.name)}" - data_summary = None - if summarize_data: - data_summary_d = state.api_conn().get_json( - "dataset-summary", - args={ - "dataset_id": self.id, - }, - ) - data_summary = DataSummary(new_records=self.new_records, **data_summary_d) - return DatasetSummary( project_name=self.project.name, dataset_name=self.name, diff --git a/py/src/braintrust/test_logger.py b/py/src/braintrust/test_logger.py index 31445034..279742e1 100644 --- a/py/src/braintrust/test_logger.py +++ b/py/src/braintrust/test_logger.py @@ -10,7 +10,7 @@ import time from collections.abc import AsyncGenerator from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import braintrust import exceptiongroup @@ -3531,6 +3531,161 @@ def test_extract_attachments_with_json_attachment(self): self.assertEqual(event["input"]["data"], json_attachment.reference) +class TestDatasetGeneratedAPI(TestCase): + def test_init_dataset_uses_generated_project_and_dataset_resources(self): + mock_state = MagicMock() + mock_state.org_name = "test-org" + api_client = mock_state.api_client.return_value + api_client.projects.post_project.return_value = { + "id": "test-project-id", + "name": "test-project", + } + api_client.datasets.post_dataset.return_value = { + "id": "test-dataset-id", + "project_id": "test-project-id", + "name": "test-dataset", + } + + dataset = braintrust.init_dataset( + project="test-project", + name="test-dataset", + description="description", + metadata={"purpose": "test"}, + use_output=False, + state=mock_state, + ) + + self.assertEqual(dataset.id, "test-dataset-id") + api_client.projects.post_project.assert_called_once_with(body={"name": "test-project", "org_name": "test-org"}) + api_client.datasets.post_dataset.assert_called_once_with( + body={ + "project_id": "test-project-id", + "name": "test-dataset", + "description": "description", + "metadata": {"purpose": "test"}, + } + ) + mock_state.app_conn.assert_not_called() + + def test_init_dataset_without_name_uses_logs_name_with_generated_resources(self): + mock_state = MagicMock() + mock_state.org_name = "test-org" + api_client = mock_state.api_client.return_value + api_client.projects.post_project.return_value = { + "id": "test-project-id", + "name": "test-project", + } + api_client.datasets.post_dataset.return_value = { + "id": "test-dataset-id", + "project_id": "test-project-id", + "name": "logs", + } + + dataset = braintrust.init_dataset( + project="test-project", + description="description", + use_output=False, + state=mock_state, + ) + + self.assertEqual(dataset.name, "logs") + api_client.projects.post_project.assert_called_once_with(body={"name": "test-project", "org_name": "test-org"}) + api_client.datasets.post_dataset.assert_called_once_with( + body={"project_id": "test-project-id", "name": "logs", "description": "description"} + ) + mock_state.app_conn.assert_not_called() + + def test_init_dataset_looks_up_explicit_project_id(self): + mock_state = MagicMock() + api_client = mock_state.api_client.return_value + api_client.projects.get_project_id.return_value = { + "id": "test-project-id", + "name": "test-project", + } + api_client.datasets.post_dataset.return_value = { + "id": "test-dataset-id", + "project_id": "test-project-id", + "name": "test-dataset", + } + + dataset = braintrust.init_dataset( + project="ignored-project-name", + project_id="test-project-id", + name="test-dataset", + use_output=False, + state=mock_state, + ) + + self.assertEqual(dataset.project.name, "test-project") + api_client.projects.get_project_id.assert_called_once_with("test-project-id") + api_client.projects.post_project.assert_not_called() + + def test_dataset_fetch_uses_generated_resource(self): + mock_state = MagicMock() + api_client = mock_state.api_client.return_value + api_client.datasets.post_dataset_id_fetch.side_effect = [ + {"events": [{"id": "row-1", "expected": "first"}], "cursor": "next"}, + {"events": [{"id": "row-2", "expected": "second"}]}, + ] + metadata = logger.ProjectDatasetMetadata( + project=logger.ObjectMetadata(id="test-project-id", name="test-project", full_info={}), + dataset=logger.ObjectMetadata(id="test-dataset-id", name="test-dataset", full_info={}), + ) + dataset = logger.Dataset( + lazy_metadata=LazyValue(lambda: metadata, use_mutex=False), + version=123, + legacy=False, + state=mock_state, + ) + + self.assertEqual([row["id"] for row in dataset.fetch(batch_size=2)], ["row-1", "row-2"]) + self.assertEqual( + api_client.datasets.post_dataset_id_fetch.call_args_list, + [ + call("test-dataset-id", body={"limit": 2, "version": "123"}), + call("test-dataset-id", body={"limit": 2, "cursor": "next", "version": "123"}), + ], + ) + mock_state.api_conn.assert_not_called() + + def test_dataset_summary_uses_generated_resource_and_configured_public_url(self): + mock_state = MagicMock() + mock_state.app_public_url = "https://public.example.com" + mock_state.org_name = "test org" + mock_state.api_client.return_value.datasets.get_dataset_id_summarize.return_value = { + "project_name": "backend-project", + "dataset_name": "backend-dataset", + "project_url": "https://backend.example.com/project", + "dataset_url": "https://backend.example.com/dataset", + "data_summary": {"total_records": 3}, + } + metadata = logger.ProjectDatasetMetadata( + project=logger.ObjectMetadata(id="test-project-id", name="test project", full_info={}), + dataset=logger.ObjectMetadata(id="test-dataset-id", name="test dataset", full_info={}), + ) + dataset = logger.Dataset( + lazy_metadata=LazyValue(lambda: metadata, use_mutex=False), + legacy=False, + state=mock_state, + ) + dataset.new_records = 1 + + summary = dataset.summarize() + + self.assertEqual(summary.project_name, "test project") + self.assertEqual(summary.dataset_name, "test dataset") + self.assertEqual(summary.project_url, "https://public.example.com/app/test%20org/p/test%20project") + self.assertEqual( + summary.dataset_url, + "https://public.example.com/app/test%20org/p/test%20project/datasets/test%20dataset", + ) + self.assertEqual(summary.data_summary, logger.DataSummary(new_records=1, total_records=3)) + mock_state.api_client.return_value.datasets.get_dataset_id_summarize.assert_called_once_with( + "test-dataset-id", summarize_data=True + ) + mock_state.api_conn.assert_not_called() + + class TestDatasetInternalBtql(TestCase): """Test that _internal_btql parameters (especially limit) are properly passed through to BTQL queries.""" @@ -3718,9 +3873,8 @@ def test_dataset_internal_btql_zero_limit_skips_fetch(self): compute_metadata.assert_not_called() mock_state.api_conn.assert_not_called() - @patch("braintrust.logger.BraintrustState") - def test_dataset_default_limit_when_not_specified(self, mock_state_class): - """Test that DEFAULT_FETCH_BATCH_SIZE is used when no custom limit is specified.""" + def test_dataset_default_limit_when_not_specified(self): + """Default dataset fetches use the generated API's standard batch size.""" from braintrust.logger import ( DEFAULT_FETCH_BATCH_SIZE, Dataset, @@ -3729,21 +3883,8 @@ def test_dataset_default_limit_when_not_specified(self, mock_state_class): ProjectDatasetMetadata, ) - # Set up mock state mock_state = MagicMock() - mock_state_class.return_value = mock_state - - # Mock the API connection and response - mock_api_conn = MagicMock() - mock_state.api_conn.return_value = mock_api_conn - - # Mock response object - mock_response = MagicMock() - mock_response.json.return_value = { - "data": [], - "cursor": None, - } - mock_api_conn.post.return_value = mock_response + mock_state.api_client.return_value.datasets.post_dataset_id_fetch.return_value = {"events": []} # Create dataset without custom limit project_metadata = ObjectMetadata(id="test-project", name="test-project", full_info={}) @@ -3759,39 +3900,20 @@ def test_dataset_default_limit_when_not_specified(self, mock_state_class): state=mock_state, ) - # Trigger a fetch which will make the BTQL query list(dataset.fetch()) - # Verify the API was called - mock_api_conn.post.assert_called_once() - - # Get the actual call arguments - call_args = mock_api_conn.post.call_args - query_json = call_args[1]["json"]["query"] - - # Verify that the default limit is used - self.assertEqual(query_json["limit"], DEFAULT_FETCH_BATCH_SIZE) + mock_state.api_client.return_value.datasets.post_dataset_id_fetch.assert_called_once_with( + "test-dataset", body={"limit": DEFAULT_FETCH_BATCH_SIZE} + ) - @patch("braintrust.logger.BraintrustState") - def test_dataset_custom_batch_size_in_fetch(self, mock_state_class): - """Test that custom batch_size in fetch() is properly passed to BTQL query.""" + def test_dataset_custom_batch_size_in_fetch(self): + """Custom batch sizes are forwarded to the generated fetch operation.""" from braintrust.logger import Dataset, LazyValue, ObjectMetadata, ProjectDatasetMetadata - # Set up mock state mock_state = MagicMock() - mock_state_class.return_value = mock_state - - # Mock the API connection and response - mock_api_conn = MagicMock() - mock_state.api_conn.return_value = mock_api_conn - - # Mock response object - mock_response = MagicMock() - mock_response.json.return_value = { - "data": [{"id": "1", "input": "test1", "expected": "output1"}], - "cursor": None, + mock_state.api_client.return_value.datasets.post_dataset_id_fetch.return_value = { + "events": [{"id": "1", "input": "test1", "expected": "output1"}] } - mock_api_conn.post.return_value = mock_response # Create dataset project_metadata = ObjectMetadata(id="test-project", name="test-project", full_info={}) @@ -3810,15 +3932,9 @@ def test_dataset_custom_batch_size_in_fetch(self, mock_state_class): custom_batch_size = 250 list(dataset.fetch(batch_size=custom_batch_size)) - # Verify the API was called - mock_api_conn.post.assert_called_once() - - # Get the actual call arguments - call_args = mock_api_conn.post.call_args - query_json = call_args[1]["json"]["query"] - - # Verify that the custom batch_size is used - self.assertEqual(query_json["limit"], custom_batch_size) + mock_state.api_client.return_value.datasets.post_dataset_id_fetch.assert_called_once_with( + "test-dataset", body={"limit": custom_batch_size} + ) @pytest.mark.vcr @@ -3826,7 +3942,7 @@ def test_dataset_internal_btql_limit_caps_total_results(): dataset = braintrust.init_dataset( project="python-sdk-vcr-tests", name="test-dataset-internal-btql-total-limit", - api_key="sk-dummy-for-vcr-replay", + api_key=os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay"), use_output=False, _internal_btql={"limit": 1}, ) diff --git a/py/src/braintrust/type_tests/test_api_client.py b/py/src/braintrust/type_tests/test_api_client.py index 91e1639d..474091d7 100644 --- a/py/src/braintrust/type_tests/test_api_client.py +++ b/py/src/braintrust/type_tests/test_api_client.py @@ -4,16 +4,23 @@ from braintrust.api import BraintrustClient, BraintrustOpenApiClient, EndpointRouter, RequestTarget from braintrust.api.types import ( + CreateDataset, CreateExperiment, CreateProject, + Dataset, Experiment, + FetchDatasetEventsResponse, FetchEventsRequest, FetchExperimentEventsResponse, + GetDatasetResponse, GetExperimentResponse, GetProjectResponse, + InsertDatasetEventRequest, + PatchDataset, PatchExperiment, PatchProject, Project, + SummarizeDatasetResponse, SummarizeExperimentResponse, ) @@ -35,6 +42,32 @@ updated_project: Project = openapi_client.projects.patch_project_id(project["id"], body=patch_project) deleted_project: Project = openapi_client.projects.delete_project_id(project["id"]) + create_dataset: CreateDataset = {"project_id": project["id"], "name": "typed-dataset"} + dataset: Dataset = openapi_client.datasets.post_dataset(body=create_dataset) + datasets: GetDatasetResponse = openapi_client.datasets.get_dataset(ids=[dataset["id"]], project_id=project["id"]) + fetched_dataset: Dataset = openapi_client.datasets.get_dataset_id(dataset["id"]) + patch_dataset: PatchDataset = {"description": "updated"} + updated_dataset: Dataset = openapi_client.datasets.patch_dataset_id(dataset["id"], body=patch_dataset) + insert_dataset_events: InsertDatasetEventRequest = { + "events": [ + { + "id": "row-id", + "_is_merge": True, + "_merge_paths": [["input"]], + "_array_delete": [{"path": ["tags"], "delete": ["old"]}], + "_object_delete": True, + "_parent_id": "parent-id", + } + ] + } + openapi_client.datasets.post_dataset_id_insert(dataset["id"], body=insert_dataset_events) + fetched_dataset_events: FetchDatasetEventsResponse = openapi_client.datasets.post_dataset_id_fetch( + dataset["id"], body={"limit": 10} + ) + fetched_dataset_xact_id: str | None = fetched_dataset_events["events"][0].get("_xact_id") + dataset_summary: SummarizeDatasetResponse = openapi_client.datasets.get_dataset_id_summarize(dataset["id"]) + deleted_dataset: Dataset = openapi_client.datasets.delete_dataset_id(dataset["id"]) + create_experiment: CreateExperiment = {"project_id": project["id"], "name": "typed-experiment"} experiment: Experiment = openapi_client.experiments.post_experiment(body=create_experiment) experiments: GetExperimentResponse = openapi_client.experiments.get_experiment( diff --git a/py/tests/api_codegen/test_generation.py b/py/tests/api_codegen/test_generation.py index ca6fbcb0..680bff77 100644 --- a/py/tests/api_codegen/test_generation.py +++ b/py/tests/api_codegen/test_generation.py @@ -1,8 +1,20 @@ +import ast import copy import re import pytest -from openapi_codegen import CodegenError, atomic_replace_tree, compare_generated, generate_tree +from openapi_codegen import ( + CONFIG_PATH, + GENERATED_ROOT, + SPEC_PATH, + CodegenError, + _snake_case, + atomic_replace_tree, + compare_generated, + generate_tree, + load_config, + read_and_verify_spec, +) def _generate(tmp_path, name, config, spec): @@ -30,6 +42,50 @@ def test_generation_selects_generated_tag_regardless_of_tag_order(tmp_path, code assert "def get_widget(" in (generated / "widgets.py").read_text() +def test_pinned_selected_spec_operations_match_generated_registries(): + config = load_config(CONFIG_PATH) + spec = read_and_verify_spec(config, SPEC_PATH) + selected_tags = config["endpoint_generator"]["generated_tags"] + + for tag in selected_tags: + expected = { + operation["operationId"] + for path_item in spec["paths"].values() + for method, operation in path_item.items() + if method != "options" and isinstance(operation, dict) and tag in operation.get("tags", []) + } + tree = ast.parse((GENERATED_ROOT / f"{_snake_case(tag)}.py").read_text()) + registry = next( + node.value + for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "OPERATIONS" for target in node.targets) + ) + assert isinstance(registry, ast.Dict) + registry_operation_ids = {key.value for key in registry.keys if isinstance(key, ast.Constant)} + operation_modes = [ + keyword.value.attr + for node in tree.body + if isinstance(node, ast.Assign) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "Operation" + for keyword in node.value.keywords + if keyword.arg == "retry_mode" and isinstance(keyword.value, ast.Attribute) + ] + + assert registry_operation_ids == expected + assert set(operation_modes) <= {"NONE", "SAFE_READ", "IDEMPOTENT_WRITE"} + assert len(operation_modes) == len(expected) + assert len(registry_operation_ids) == sum( + isinstance(node, ast.Assign) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "Operation" + for node in tree.body + ) + + def test_declarative_post_reads_use_safe_read_retry_mode(tmp_path, codegen_config, minimal_spec): minimal_spec["paths"]["/widgets"] = { "post": { @@ -235,6 +291,29 @@ def test_stale_artifacts_are_reported_and_removed(tmp_path, codegen_config, mini assert not stale_package.exists() +def test_leading_underscore_model_fields_preserve_wire_names(tmp_path, codegen_config, minimal_spec): + wire_names = ( + "_array_delete", + "_is_merge", + "_merge_paths", + "_object_delete", + "_pagination_key", + "_parent_id", + "_xact_id", + ) + widget = minimal_spec["components"]["schemas"]["Widget"] + widget["properties"].update({name: {"type": "string"} for name in wire_names}) + widget["required"].append("_xact_id") + + generated = _generate(tmp_path, "leading-underscore-fields", codegen_config, minimal_spec) + models = _models_text(generated) + + for name in wire_names: + assert f" {name}:" in models + assert " _xact_id: str" in models + assert "field_" not in models + + def test_nullable_and_missing_fields_remain_distinct(tmp_path, codegen_config, minimal_spec): spec = copy.deepcopy(minimal_spec) widget = spec["components"]["schemas"]["Widget"]