From 326188aa456a96d19d7c1a6bb44c48e214e11d80 Mon Sep 17 00:00:00 2001 From: aaron Date: Sun, 18 Oct 2026 15:32:06 -0700 Subject: [PATCH] Report a failed turn's tokens as None, not 0, and close the release blockers TurnRecord.input_tokens and output_tokens are now int | None, and None for a call that raised before answering. 0 read as "measured, and free", against the rule that None means "not reported". It changes a public model, so it ships in 0.6.0 with the other observer changes rather than in a later breaking release. BACKLOG: items 5 and 6 are done (Dependabot PR 10 and the main merge landed), so nothing blocks 0.6.0. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 34 ++++------------------------------ CHANGELOG.md | 3 +++ docs/api/summary.md | 3 ++- docs/guides/summaries.md | 2 +- src/agentloop/loop.py | 4 ++-- src/agentloop/summary.py | 5 +++-- tests/test_summary.py | 2 ++ 7 files changed, 17 insertions(+), 36 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 6eccfe7..c308b2a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -13,42 +13,16 @@ on day one. ## Must land before the next release (0.6.0) -Items 1–4 (Anthropic transport classification, the Anthropic `http_client` type and -ownership, `phase` / `request_sent` on body errors, and the API reference pages) landed on -`failure-telemetry`. The numbers below are kept so existing references stay valid. +Nothing open. Items 1–6 landed: 1–4 (Anthropic transport classification, the Anthropic +`http_client` type and ownership, `phase` / `request_sent` on body errors, the API reference +pages) in PR 11; 5 (a failed turn's tokens are `None`, not `0`) and 6 (Dependabot PR 10 and +the `main` merge) after it. The numbers below are kept so existing references stay valid. Left over from item 2, optional: `pyproject.toml` still allows `anthropic>=1.0`, which includes SDKs built on httpx rather than httpx2. `AnthropicClient` reads the expected client class from the installed SDK, so either works, but the type hints and docs name `httpx2.Client`. Raise the floor to the first httpx2-based SDK once that version is known. -### 5. Decide: failed turns report `0` tokens instead of "not reported" [decide before next release] - -**Status:** not implemented. `TurnRecord.input_tokens` / `output_tokens` are plain -`int = 0` (`summary.py`). `_Run._add_turn` (`loop.py`) fills `0` for a call that raised -before answering. The project rule is that `None` means "not reported", never zero. - -**Concrete trigger:** set `SummaryOptions(timeline=True)` and let one model call fail. The -failed `TurnRecord` reports 0 input and 0 output tokens. That reads as "measured, and -free". A consumer summing the timeline can't tell a failed call from a cheap one without -also checking `error_type`. - -**What implementing it takes:** -- Make both fields `int | None`, set them to `None` when `response is None`, and update - anything that sums them. -- Add a CHANGELOG "Changed" entry. - -`TurnRecord` is in `__all__`, so this breaks code doing arithmetic on the fields. **Why it -is a release decision:** 0.6.0 is already a minor bump that changes `LoopObserver`. Making -this change in the same release costs nothing extra. Making it later costs another -breaking release. Either do it now, or record that it waits. - -### 6. Release housekeeping for PR 11 [blocks next release] - -- Merge Dependabot PR 10 (`actions/upload-artifact` 7.0.2 and `download-artifact` 8.0.2 in - `release.yml`; CI green) before tagging. Otherwise the release runs on the older - actions. - --- ## High priority diff --git a/CHANGELOG.md b/CHANGELOG.md index cadbd3e..1f92e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ while `0.x`, a minor bump may break the public API (`agentloop.__all__`). - A non-JSON body now reports its real status in `ResponseInvalid.status_code` and the message. - `OpenAICompatClient` sends its URL and headers per request instead of configuring them on the `httpx.Client`; code that replaced the private `_http` attribute still works. +- `TurnRecord.input_tokens` and `output_tokens` are `int | None`, and `None` (not `0`) for a + call that raised before answering: there was no usage to read, and `0` read as "measured, + and free". Code that sums the timeline must skip `None`. ### Fixed - `TurnRecord.attempts` reported `1` for a call that failed after retries. diff --git a/docs/api/summary.md b/docs/api/summary.md index ed2aae7..9ec10fb 100644 --- a/docs/api/summary.md +++ b/docs/api/summary.md @@ -56,7 +56,8 @@ These options report conditions but do not stop or otherwise change the conversa Metadata for one model call: duration, stop reason, token use, cost, attempts, tools, tool errors, and whether a truncated call was discarded. When a call fails before returning, -`error_type` contains the exception class name. `attempt_log` has one `AttemptRecord` per +`error_type` contains the exception class name, and `input_tokens` / `output_tokens` are +`None` (not reported), never `0`. `attempt_log` has one `AttemptRecord` per attempt (empty when the first attempt succeeded), and `attempts` is the real count even for a call that failed. diff --git a/docs/guides/summaries.md b/docs/guides/summaries.md index 6d7b58f..a0a0914 100644 --- a/docs/guides/summaries.md +++ b/docs/guides/summaries.md @@ -129,7 +129,7 @@ These features are off by default. Configure them together with `SummaryOptions` - **`timeline`**: `summary.timeline` becomes a tuple of `TurnRecord`s, one per model call: duration, stop reason, tokens, cost, attempts, the tools it led to and which of them errored, and whether a truncated turn was discarded. A call that raised appears with its - `error_type`. The timeline also excludes conversation content. + `error_type`, and with `None` tokens: it was never answered, so nothing was measured. The timeline also excludes conversation content. ## How reporting stays separate from execution diff --git a/src/agentloop/loop.py b/src/agentloop/loop.py index 88d0074..8ce9b2a 100644 --- a/src/agentloop/loop.py +++ b/src/agentloop/loop.py @@ -334,8 +334,8 @@ def _add_turn( model_seconds=model_seconds, stop_reason=str(response.stop_reason) if response is not None else None, error_type=type(error).__name__ if error is not None else None, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, + input_tokens=usage.input_tokens if usage else None, + output_tokens=usage.output_tokens if usage else None, cost_usd=usage.cost_usd if usage else None, attempts=( response.attempts diff --git a/src/agentloop/summary.py b/src/agentloop/summary.py index 743e945..4507e4d 100644 --- a/src/agentloop/summary.py +++ b/src/agentloop/summary.py @@ -120,8 +120,9 @@ class TurnRecord(BaseModel): # None when the call raised before answering; `error_type` then says why. stop_reason: str | None = None error_type: str | None = None - input_tokens: int = 0 - output_tokens: int = 0 + # None, not 0, for a call that raised before answering: there was no usage to read. + input_tokens: int | None = None + output_tokens: int | None = None cost_usd: float | None = None attempts: int = 1 # Per-attempt detail; empty when the first attempt succeeded. diff --git a/tests/test_summary.py b/tests/test_summary.py index c8b9ba9..d7af715 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -802,6 +802,8 @@ def test_a_failed_call_reports_its_real_attempts_and_whether_it_was_sent() -> No assert summary.timeline is not None assert summary.timeline[0].attempts == 1 assert summary.timeline[0].attempt_log[0].phase is FailurePhase.TLS + # Not answered, so not measured: None, never a free-looking 0. + assert (summary.timeline[0].input_tokens, summary.timeline[0].output_tokens) == (None, None) def test_a_retried_failure_is_counted_and_the_turn_shows_every_attempt(