Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/art/api_costs.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,15 @@ def _extract_direct_response_cost(response: Any) -> float | None:
def _extract_openai_token_counts(response: Any) -> _OpenAITokenUsage | None:
usage = _response_usage(response)
prompt_tokens = _read_usage_field(usage, "prompt_tokens")
if prompt_tokens is None:
prompt_tokens = _read_usage_field(usage, "input_tokens")
completion_tokens = _read_usage_field(usage, "completion_tokens")
if completion_tokens is None:
completion_tokens = _read_usage_field(usage, "output_tokens")
cached_prompt_tokens = (
_read_usage_nested_field(usage, "prompt_tokens_details", "cached_tokens") or 0.0
_read_usage_nested_field(usage, "prompt_tokens_details", "cached_tokens")
or _read_usage_nested_field(usage, "input_tokens_details", "cached_tokens")
or 0.0
)
if (
prompt_tokens is None
Expand Down
66 changes: 66 additions & 0 deletions tests/unit/test_track_api_cost.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,40 @@ def __init__(
self.model = model


class _OpenAIResponsesUsage:
def __init__(
self,
input_tokens: int,
output_tokens: int,
*,
cached_tokens: int = 0,
) -> None:
self.input_tokens = input_tokens
self.output_tokens = output_tokens
self.input_tokens_details = type(
"InputTokensDetails",
(),
{"cached_tokens": cached_tokens},
)()


class _OpenAIResponsesResponse:
def __init__(
self,
input_tokens: int,
output_tokens: int,
*,
cached_tokens: int = 0,
model: str | None = None,
) -> None:
self.usage = _OpenAIResponsesUsage(
input_tokens,
output_tokens,
cached_tokens=cached_tokens,
)
self.model = model


class _AnthropicUsage:
def __init__(
self,
Expand Down Expand Up @@ -162,6 +196,38 @@ async def _judge() -> _OpenAIResponse:
metrics = await builder.flush()
assert metrics["costs/train/llm_judge/cached_openai"] == pytest.approx(0.00255)

@pytest.mark.asyncio
async def test_openai_cost_extraction_accounts_for_responses_cached_tokens(
self,
) -> None:
builder = MetricsBuilder(cost_context="train")

@track_api_cost(
source="llm_judge/cached_openai_responses",
provider="openai",
model_name="openai/gpt-4.1",
prompt_price_per_million=2.0,
completion_price_per_million=8.0,
cached_prompt_price_per_million=0.5,
)
async def _judge() -> _OpenAIResponsesResponse:
return _OpenAIResponsesResponse(
input_tokens=2_000,
output_tokens=100,
cached_tokens=1_500,
)

token = builder.activate()
try:
await _judge()
finally:
token.var.reset(token)

metrics = await builder.flush()
assert metrics["costs/train/llm_judge/cached_openai_responses"] == pytest.approx(
0.00255
)

@pytest.mark.asyncio
async def test_anthropic_cost_extraction_uses_registered_model_pricing(
self,
Expand Down