Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

10 changes: 10 additions & 0 deletions py/src/braintrust/api/test_projects.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import pytest
from braintrust import init_logger
from braintrust.api import BraintrustClient
from braintrust.logger import BraintrustState


@pytest.mark.vcr
Expand All @@ -22,6 +24,14 @@ def test_projects_end_to_end_with_real_backend(api_key):
updated = client.openapi.projects.patch_project_id(
created["id"], body={"description": "updated by the Python SDK VCR test"}
)

state = BraintrustState()
state.login(api_key=api_key, org_name=discovery.organization.name)
registered_logger = init_logger(project=project_name, state=state, set_current=False)
looked_up_logger = init_logger(project_id=created["id"], state=state, set_current=False)
assert registered_logger.id == created["id"]
assert looked_up_logger.project.name == project_name

deleted = client.openapi.projects.delete_project_id(created["id"])

assert create_project == {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,76 +1,4 @@
interactions:
- request:
body: '{"project_name": "My Project", "org_id": "bf61839e-94bb-4a81-85ed-26b533a24b61"}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '80'
Content-Type:
- application/json
User-Agent:
- python-requests/2.32.5
method: POST
uri: https://www.braintrust.dev/api/project/register
response:
body:
string: '{"project":{"id":"6b770640-a26a-4680-9d69-537262023137","org_id":"bf61839e-94bb-4a81-85ed-26b533a24b61","name":"My
Project","description":null,"created":"2026-03-12T19:49:37.394Z","deleted_at":null,"user_id":"fd67d1e5-c011-43fa-8096-ee5fed2b7df1","settings":null},"found_existing":true}'
headers:
Cache-Control:
- public, max-age=0, must-revalidate
Content-Length:
- '286'
Content-Security-Policy:
- 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic''
''nonce-ZDFkNjJkY2EtMWVmOS00ZTZmLThjZTItNGRmMjAzMWZlZDZh'' *.js.stripe.com
js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev
btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com
d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com
cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net
fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'';
frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16;
report-to csp-endpoint-0'
Content-Type:
- application/json; charset=utf-8
Date:
- Thu, 12 Mar 2026 20:26:58 GMT
Etag:
- '"kzpvhxucks7y"'
Reporting-Endpoints:
- csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16"
Server:
- Vercel
Strict-Transport-Security:
- max-age=63072000
X-Bt-Was-Udf-Cached:
- 'true'
X-Clerk-Auth-Message:
- Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
token-carrier=header)
X-Clerk-Auth-Reason:
- token-invalid
X-Clerk-Auth-Status:
- signed-out
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Matched-Path:
- /api/project/register
X-Nonce:
- ZDFkNjJkY2EtMWVmOS00ZTZmLThjZTItNGRmMjAzMWZlZDZh
X-Vercel-Cache:
- MISS
X-Vercel-Id:
- sfo1::iad1::tp2pw-1773347218334-903d13f4bb17
status:
code: 200
message: OK
- request:
body: '{"sandbox_spec": {"provider": "modal", "snapshot_ref": "im-icRxmsk1Sz9XPP2f8OblVU"},
"entrypoints": ["./local/js/optimization/evals/btql-generation/btql-queries.eval.ts"],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,76 +1,4 @@
interactions:
- request:
body: '{"project_name": "My Project", "org_id": "bf61839e-94bb-4a81-85ed-26b533a24b61"}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '80'
Content-Type:
- application/json
User-Agent:
- python-requests/2.32.5
method: POST
uri: https://www.braintrust.dev/api/project/register
response:
body:
string: '{"project":{"id":"6b770640-a26a-4680-9d69-537262023137","org_id":"bf61839e-94bb-4a81-85ed-26b533a24b61","name":"My
Project","description":null,"created":"2026-03-12T19:49:37.394Z","deleted_at":null,"user_id":"fd67d1e5-c011-43fa-8096-ee5fed2b7df1","settings":null},"found_existing":true}'
headers:
Cache-Control:
- public, max-age=0, must-revalidate
Content-Length:
- '286'
Content-Security-Policy:
- 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic''
''nonce-ZDFkNjJkY2EtMWVmOS00ZTZmLThjZTItNGRmMjAzMWZlZDZh'' *.js.stripe.com
js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev
btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com
d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com
cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net
fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'';
frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16;
report-to csp-endpoint-0'
Content-Type:
- application/json; charset=utf-8
Date:
- Thu, 12 Mar 2026 20:26:58 GMT
Etag:
- '"kzpvhxucks7y"'
Reporting-Endpoints:
- csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16"
Server:
- Vercel
Strict-Transport-Security:
- max-age=63072000
X-Bt-Was-Udf-Cached:
- 'true'
X-Clerk-Auth-Message:
- Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
token-carrier=header)
X-Clerk-Auth-Reason:
- token-invalid
X-Clerk-Auth-Status:
- signed-out
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Matched-Path:
- /api/project/register
X-Nonce:
- ZDFkNjJkY2EtMWVmOS00ZTZmLThjZTItNGRmMjAzMWZlZDZh
X-Vercel-Cache:
- MISS
X-Vercel-Id:
- sfo1::iad1::tp2pw-1773347218334-903d13f4bb17
status:
code: 200
message: OK
- request:
body: '{"sandbox_spec": {"provider": "modal", "snapshot_ref": "im-icRxmsk1Sz9XPP2f8OblVU"},
"entrypoints": ["./local/js/optimization/evals/btql-generation/btql-queries.eval.ts"],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,76 +1,4 @@
interactions:
- request:
body: '{"project_name": "My Project", "org_id": "bf61839e-94bb-4a81-85ed-26b533a24b61"}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '80'
Content-Type:
- application/json
User-Agent:
- python-requests/2.32.5
method: POST
uri: https://www.braintrust.dev/api/project/register
response:
body:
string: '{"project":{"id":"6b770640-a26a-4680-9d69-537262023137","org_id":"bf61839e-94bb-4a81-85ed-26b533a24b61","name":"My
Project","description":null,"created":"2026-03-12T19:49:37.394Z","deleted_at":null,"user_id":"fd67d1e5-c011-43fa-8096-ee5fed2b7df1","settings":null},"found_existing":true}'
headers:
Cache-Control:
- public, max-age=0, must-revalidate
Content-Length:
- '286'
Content-Security-Policy:
- 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic''
''nonce-OTZlNWI3NDUtOWJjYi00NjA3LWI3MzEtNzhhZWQ0OGNmMmFh'' *.js.stripe.com
js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev
btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com
d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com
cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net
fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'';
frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16;
report-to csp-endpoint-0'
Content-Type:
- application/json; charset=utf-8
Date:
- Thu, 12 Mar 2026 20:26:48 GMT
Etag:
- '"kzpvhxucks7y"'
Reporting-Endpoints:
- csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16"
Server:
- Vercel
Strict-Transport-Security:
- max-age=63072000
X-Bt-Was-Udf-Cached:
- 'true'
X-Clerk-Auth-Message:
- Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
token-carrier=header)
X-Clerk-Auth-Reason:
- token-invalid
X-Clerk-Auth-Status:
- signed-out
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Matched-Path:
- /api/project/register
X-Nonce:
- OTZlNWI3NDUtOWJjYi00NjA3LWI3MzEtNzhhZWQ0OGNmMmFh
X-Vercel-Cache:
- MISS
X-Vercel-Id:
- sfo1::iad1::8hdkr-1773347208825-87a220df8e21
status:
code: 200
message: OK
- request:
body: '{"sandbox_spec": {"provider": "modal", "snapshot_ref": "im-icRxmsk1Sz9XPP2f8OblVU"},
"entrypoints": ["./local/js/optimization/evals/btql-generation/btql-queries.eval.ts"],
Expand Down
7 changes: 4 additions & 3 deletions py/src/braintrust/framework2.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import Any, overload

import slugify
from braintrust.logger import api_conn, app_conn, login
from braintrust.logger import _internal_get_global_state, api_conn, login

from .framework import _is_lazy_load, bcolors # type: ignore
from .generated_types import (
Expand All @@ -28,8 +28,9 @@ def __init__(self):

def get_by_name(self, project_name: str) -> str:
if project_name not in self._name_cache:
resp = app_conn().post_json("api/project/register", {"project_name": project_name})
self._name_cache[project_name] = resp["project"]["id"]
state = _internal_get_global_state()
project = state.api_client().projects.post_project(body={"name": project_name, "org_name": state.org_name})
self._name_cache[project_name] = project["id"]
return self._name_cache[project_name]

def get(self, project: "Project") -> str:
Expand Down
43 changes: 20 additions & 23 deletions py/src/braintrust/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -1737,26 +1737,27 @@ def compute_metadata():
)


def _compute_logger_metadata(project_name: str | None = None, project_id: str | None = None):
login()
org_id = _state.org_id
def _compute_logger_metadata(
project_name: str | None = None,
project_id: str | None = None,
state: BraintrustState | None = None,
):
state = state or _state
state.login()
org_id = state.org_id
if project_id is None:
response = _state.app_conn().post_json(
"api/project/register",
{
"project_name": project_name or GLOBAL_PROJECT,
"org_id": org_id,
},
response = state.api_client().projects.post_project(
body={"name": project_name or GLOBAL_PROJECT, "org_name": state.org_name}
)
resp_project = response["project"]
return OrgProjectMetadata(
org_id=org_id,
project=ObjectMetadata(id=resp_project["id"], name=resp_project["name"], full_info=resp_project),
project=ObjectMetadata(id=response["id"], name=response["name"], full_info=dict(response)),
)
elif project_name is None:
response = _state.app_conn().get_json("api/project", {"id": project_id})
response = state.api_client().projects.get_project_id(project_id)
return OrgProjectMetadata(
org_id=org_id, project=ObjectMetadata(id=project_id, name=response["name"], full_info=response)
org_id=org_id,
project=ObjectMetadata(id=response["id"], name=response["name"], full_info=dict(response)),
)
else:
return OrgProjectMetadata(
Expand Down Expand Up @@ -1804,7 +1805,7 @@ def init_logger(

def compute_metadata():
state.login(org_name=org_name, api_key=api_key, app_url=app_url, force_login=force_login)
return _compute_logger_metadata(**compute_metadata_args)
return _compute_logger_metadata(**compute_metadata_args, state=state)

# For loggers, enable queue size limit enforcement (bounded queue)
state.enforce_queue_size_limit(True)
Expand Down Expand Up @@ -5515,17 +5516,13 @@ def lazy_init(self):
if self._id is None or self._name is None:
with self.init_lock:
if self._id is None:
response = _state.app_conn().post_json(
"api/project/register",
{
"project_name": self._name or GLOBAL_PROJECT,
"org_id": _state.org_id,
},
response = _state.api_client().projects.post_project(
body={"name": self._name or GLOBAL_PROJECT, "org_name": _state.org_name}
)
self._id = response["project"]["id"]
self._name = response["project"]["name"]
self._id = response["id"]
self._name = response["name"]
elif self._name is None:
response = _state.app_conn().get_json("api/project", {"id": self._id})
response = _state.api_client().projects.get_project_id(self._id)
self._name = response["name"]

return self
Expand Down
6 changes: 2 additions & 4 deletions py/src/braintrust/sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,8 @@ def register_sandbox(
state = state or _state
state.login(api_key=api_key, app_url=app_url, org_name=org_name)

project_response = state.app_conn().post_json(
"api/project/register", {"project_name": project, "org_id": state.org_id}
)
project_id = project_response["project"]["id"]
project_response = state.api_client().projects.post_project(body={"name": project, "org_name": state.org_name})
project_id = project_response["id"]

runtime_context = {
"runtime": "python",
Expand Down
19 changes: 18 additions & 1 deletion py/src/braintrust/test_framework2.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Tests for framework2 module, specifically metadata and tags support."""

import importlib.util
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch

import pytest

Expand All @@ -13,6 +13,23 @@
HAS_PYDANTIC = importlib.util.find_spec("pydantic") is not None


def test_project_id_cache_uses_generated_project_registration():
mock_state = MagicMock()
mock_state.org_name = "test-org"
mock_state.api_client.return_value.projects.post_project.return_value = {
"id": "generated-project-id",
"name": "test-project",
}
with patch("braintrust.logger._state", mock_state):
project_id = ProjectIdCache().get_by_name("test-project")

assert project_id == "generated-project-id"
mock_state.api_client.return_value.projects.post_project.assert_called_once_with(
body={"name": "test-project", "org_name": "test-org"}
)
mock_state.app_conn.assert_not_called()


class TestCodeFunctionMetadata:
"""Tests for CodeFunction metadata support."""

Expand Down
Loading