From 293a4a073235894c88e01fe0ec8dbb954b56ad69 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:49:02 +0100 Subject: [PATCH 1/4] feat(agent): per-day numbers on /debug, a live line for today, a service pause refusal Work in progress taken over from a killed session: the per-day table, the agents service's live budget view and the `service_paused` refusal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- agents/anyplot/plugins/budget.py | 15 +- agents/anyplot/plugins/ledger.py | 121 ++++- agents/anyplot/plugins/scope_guard.py | 9 +- agents/anyplot/prompts/refusals.yaml | 16 + agents/anyplot/telemetry.py | 4 + agents/main.py | 62 ++- agents/stream.py | 2 +- api/agent_usage.py | 226 +++++++++ api/routers/agent.py | 36 +- api/routers/debug.py | 13 +- app/src/hooks/useAgentSession.ts | 14 +- app/src/lib/agent.ts | 4 + app/src/pages/AgentChatPage.test.tsx | 50 +- app/src/pages/AgentChatPage.tsx | 37 +- app/src/pages/DebugPage.test.tsx | 158 ++++++- .../sections/agent-chat/ChatThread.test.tsx | 22 + app/src/sections/agent-chat/ChatThread.tsx | 8 +- app/src/sections/agent-chat/messages.test.ts | 38 +- app/src/sections/agent-chat/messages.ts | 39 +- app/src/sections/debug/AgentDailyUsage.tsx | 433 ++++++++++++++++++ app/src/sections/debug/AgentUsageSection.tsx | 7 + tests/unit/agents/runtime/test_day_budget.py | 269 +++++++++++ tests/unit/agents/runtime/test_policy.py | 13 + tests/unit/api/test_agent_router.py | 48 +- tests/unit/api/test_agent_usage.py | 136 +++++- 25 files changed, 1745 insertions(+), 35 deletions(-) create mode 100644 app/src/sections/debug/AgentDailyUsage.tsx create mode 100644 tests/unit/agents/runtime/test_day_budget.py diff --git a/agents/anyplot/plugins/budget.py b/agents/anyplot/plugins/budget.py index 8e202d8bc90..95f28c1c6e5 100644 --- a/agents/anyplot/plugins/budget.py +++ b/agents/anyplot/plugins/budget.py @@ -3,7 +3,8 @@ * `before_model_callback` halts **only the root** with the fixed `budget` reply when the request has spent `AGENT_MAX_LLM_CALLS` calls or `AGENT_REQUEST_TOKEN_BUDGET` cost-weighted tokens (`ledger.BUDGET_WEIGHTS`), or the user or the service has spent - its daily ones. A halt inside the adapter or the reviewer would break their schema + its daily ones: `service_paused` when only the service's day budget is spent, and + `locked_out` past the day's strikes (`ledger.day_refusal`). A halt inside the adapter or the reviewer would break their schema parsing, so the pipeline checks `ledger.budget_allows` before each `run_node` and finishes with `PlotResult(failed, reason=budget)` instead. Before a review it keeps room for the review and the root's closing reply (`pipeline.REVIEW_RESERVE_TOKENS`, @@ -50,8 +51,10 @@ attribution, budget_allows, budget_tokens, + day_refusal, finish_name, ledger_for, + request_ok, usage_breakdown, usage_tokens, ) @@ -121,6 +124,7 @@ async def before_model_callback( ledger = ledger_for(callback_context.invocation_id) if callback_context.agent_name != ROOT_AGENT: return None + code = "budget" try: settings = get_settings() if not ledger.user_id: @@ -130,12 +134,15 @@ async def before_model_callback( strip_tools(llm_request) attribution("budget_exempt", ledger, agent=callback_context.agent_name) return None - if budget_allows(ledger, get_services().usage, settings): + usage = get_services().usage + if budget_allows(ledger, usage, settings): return None + if request_ok(ledger, settings): # the day's budgets, not the request's: say which one + code = day_refusal(usage, ledger.user_id or "anonymous", settings) except Exception as exc: logger.warning("budget check failed: %s", type(exc).__name__) - text = refusal("budget", ledger.lang) - ledger.refuse("budget", text) + text = refusal(code, ledger.lang) + ledger.refuse(code, text) attribution("budget_halt", ledger, agent=callback_context.agent_name) return budget_response(text) diff --git a/agents/anyplot/plugins/ledger.py b/agents/anyplot/plugins/ledger.py index ac5113a8fc9..e104575957c 100644 --- a/agents/anyplot/plugins/ledger.py +++ b/agents/anyplot/plugins/ledger.py @@ -20,6 +20,12 @@ which the `/messages` route checks before the scope judge (`message_limit`). Only a new UTC day, a redeploy or a scale-to-zero resets them. +The global day budget binds the whole service: once it is spent (`UsageBook.paused`), +every user gets the `service_paused` refusal (`day_refusal`) until 00:00 UTC, told apart +from a user's own spent budget (`budget`, a paste's `daily_limit`). `UsageBook.snapshot` +answers the admin's live view of the day (`GET /v1/budget`): the global spend and how many +users are at their own limits, counts only, for the instance's lifetime. + The request, user-day and global-day budgets count cost-weighted tokens: each kind weighted by its price relative to an uncached input token (`BUDGET_WEIGHTS`), so a cache read counts a tenth and an output token five times. `RequestLedger.tokens` and @@ -41,7 +47,7 @@ from collections.abc import Awaitable, Callable from contextvars import ContextVar from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any, Literal from ..schemas import Theme @@ -222,6 +228,23 @@ def today() -> str: return datetime.now(UTC).date().isoformat() +def utc_stamp(moment: datetime) -> str: + """An instant as the API writes one: ISO 8601 in UTC to the second, with a `Z`.""" + return moment.astimezone(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def next_reset(now: datetime | None = None) -> datetime: + """The next 00:00 UTC: when the usage book starts a new day and every daily counter, the global one included, + starts again from zero.""" + current = (now or datetime.now(UTC)).astimezone(UTC) + return datetime.combine(current.date() + timedelta(days=1), datetime.min.time(), UTC) + + +SERVICE_PAUSED = "service_paused" +"""The refusal and the paste route's `429` once the global day budget is spent: the whole service pauses until +00:00 UTC, whatever the user's own counts (owner request of 2026-11-02: tell it apart from the user's own limit).""" + + @dataclass class _Day: date: str @@ -257,6 +280,30 @@ class _Day: settings, like the outcome beacons'.""" +LimitKind = Literal["tokens", "runs", "data_runs", "messages", "pastes", "conversions", "locked_out"] +"""The day caps of one user that `UsageBook.snapshot` counts users at: the token budget, the pipeline runs, the +`run_data_code` runs, the chat turns, the pastes, the file conversions, and the `attack` strikes.""" +LIMIT_KINDS: tuple[LimitKind, ...] = ("tokens", "runs", "data_runs", "messages", "pastes", "conversions", "locked_out") + + +@dataclass(frozen=True) +class DaySnapshot: + """The usage book's day as the admin's live view reads it (`GET /v1/budget`): counts only, never a user id.""" + + date: str + since: datetime + """When this day's counts began: the book's start when the instance started today, else 00:00 UTC.""" + started_at: datetime + """When this instance's usage book started: a redeploy or a scale-to-zero starts a new one at zero.""" + global_tokens: int + users: int + """Users with a chat turn, a paste, a conversion, a run, booked tokens or a strike today.""" + at_limit: int + """Users at one or more of their own day caps (`limits`), each counted once.""" + limits: dict[LimitKind, int] + """Users at each day cap; a user at two caps counts under both.""" + + class UsageBook: """Per-user and global daily counters for this instance's lifetime, and each user's messages and pastes of the last minute.""" @@ -265,6 +312,8 @@ def __init__(self, clock: Callable[[], float] = time.monotonic) -> None: self._day = _Day(today()) self._clock = clock self._recent: dict[tuple[WindowKind, str], deque[float]] = {} + self.started_at = datetime.now(UTC) + """When the book began counting: in production the instance's first request (`services.get_services`).""" def _current(self) -> _Day: date = today() @@ -318,6 +367,45 @@ def locked_out(self, user_id: str, settings: AgentSettings) -> bool: `daily_ok` folds into its answer, told apart from a spent token budget.""" return self.user_strikes(user_id) >= settings.attack_strikes + def paused(self, settings: AgentSettings, *, reserve: int = 0) -> bool: + """Whether the service's day budget (`AGENT_GLOBAL_DAILY_TOKEN_BUDGET`) is spent, `reserve` tokens beyond the + spend: the `service_paused` refusal for everyone until 00:00 UTC, which `daily_ok` folds into its answer.""" + return self.global_tokens() + reserve >= settings.global_daily_token_budget + + def snapshot(self, settings: AgentSettings) -> DaySnapshot: + """Today's global spend and how many users are at their own day caps, as counts (`DaySnapshot`).""" + day = self._current() + reached: dict[LimitKind, set[str]] = { + "tokens": _at_least(day.user_tokens, settings.daily_token_budget), + "runs": _at_least(day.user_runs, settings.daily_pipeline_runs), + "data_runs": _at_least(day.user_data_runs, settings.daily_data_runs), + "messages": _at_least(day.user_messages, settings.user_messages_per_day), + "pastes": _at_least(day.user_pastes, settings.user_pastes_per_day), + "conversions": _at_least(day.user_conversions, settings.user_conversions_per_day), + "locked_out": { + user for user, strikes in day.user_strikes.items() if len(strikes) >= settings.attack_strikes + }, + } + users = set().union( + day.user_tokens, + day.user_runs, + day.user_data_runs, + day.user_messages, + day.user_pastes, + day.user_conversions, + day.user_strikes, + ) + midnight = datetime.fromisoformat(day.date).replace(tzinfo=UTC) + return DaySnapshot( + date=day.date, + since=max(self.started_at, midnight), + started_at=self.started_at, + global_tokens=day.global_tokens, + users=len(users), + at_limit=len(set().union(*reached.values())), + limits={kind: len(reached[kind]) for kind in LIMIT_KINDS}, + ) + def runs_ok(self, user_id: str, settings: AgentSettings) -> bool: return self.user_runs(user_id) < settings.daily_pipeline_runs @@ -444,6 +532,21 @@ def _trimmed(self, kind: WindowKind, user_id: str) -> deque[float] | None: return recent +def _at_least(counts: dict[str, int], cap: int) -> set[str]: + return {user for user, count in counts.items() if count >= cap} + + +def day_refusal(usage: UsageBook, user_id: str, settings: AgentSettings, *, reserve: int = 0) -> str: + """Which refusal a failed `daily_ok` is: `locked_out` past the day's strikes, `budget` when the user's own day + budget is spent, `service_paused` when only the service's is (its text says the user's own limit is untouched).""" + if usage.locked_out(user_id, settings): + return "locked_out" + own_left = usage.user_tokens(user_id) + reserve < settings.daily_token_budget + if own_left and usage.paused(settings, reserve=reserve): + return SERVICE_PAUSED + return "budget" + + def request_ok(ledger: RequestLedger, settings: AgentSettings, *, next_calls: int = 1, reserve: int = 0) -> bool: """Whether the request may make `next_calls` more LLM calls and still has `reserve` cost-weighted tokens left.""" return ( @@ -533,6 +636,22 @@ def usage_breakdown(usage: Any) -> dict[str, int]: 160,000 of `AGENT_REQUEST_TOKEN_BUDGET` stays about 36 % above the costliest of these.""" +UNIT_PRICES: dict[str, float] = {"claude-haiku-5-5": 0.10, "gemini-3.8-flash": 1.50} +"""The uncached input list price, USD per million tokens at the `global` endpoint, of each main model the budgets +are counted in: one weighted token (`BUDGET_WEIGHTS`, `JUDGE_WEIGHTS`) costs this much. The prices of +`agents/evals/pricing.py`, which the image does not ship (a test keeps both equal).""" +LOCATION_SURCHARGES: dict[str, float] = {"global": 0.0, "eu": 0.10, "us": 0.10} +"""`pricing.LOCATION_SURCHARGE`: the multi-regions add 10 % to the global list price (a test keeps both equal).""" + + +def weighted_usd(tokens: int, model: str, location: str) -> float | None: + """Weighted tokens of `model` at `location` in USD at list price; None for a model without a price here.""" + price = UNIT_PRICES.get(model) + if price is None: + return None + return tokens * price * (1.0 + LOCATION_SURCHARGES.get(location, 0.0)) / 1e6 + + def weighted_tokens(*, uncached: int = 0, cached: int = 0, cache_write: int = 0, output: int = 0) -> int: """Token counts in input-token equivalents (`BUDGET_WEIGHTS`), rounded to a whole token.""" weights = BUDGET_WEIGHTS diff --git a/agents/anyplot/plugins/scope_guard.py b/agents/anyplot/plugins/scope_guard.py index 2bb078c6576..6225ffc44aa 100644 --- a/agents/anyplot/plugins/scope_guard.py +++ b/agents/anyplot/plugins/scope_guard.py @@ -5,8 +5,9 @@ * a structured action (`RequestLedger.kind == "action"`, set by the `/v1` route for "Create plot") is not judged; * a message with a non-text part is withheld with the `unsupported_content` refusal; -* the per-user and global daily budget is checked before the judge runs, and the - judge's tokens are booked to the ledger and the usage book; +* the per-user and global daily budget is checked before the judge runs (the + `service_paused` refusal when only the global one is spent, `ledger.day_refusal`), + and the judge's tokens are booked to the ledger and the usage book; * the judge sees all text parts inside ``, the root's last real reply (at most 500 characters; never a fixed refusal or the outage line, `is_service_text`) inside ``, and the user's last three @@ -97,7 +98,7 @@ from ..services import KeptRefusal, get_services from ..session_state import LANGUAGE_SOURCE, SPEC_ID, language_delta from ..settings import get_settings -from .ledger import RequestLedger, attribution, judge_budget_tokens, ledger_for +from .ledger import RequestLedger, attribution, day_refusal, judge_budget_tokens, ledger_for logger = logging.getLogger(__name__) @@ -289,7 +290,7 @@ async def _judge( settings = get_settings() user = ledger.user_id or invocation_context.session.user_id if not services.usage.daily_ok(user, settings): - reason = LOCKED_OUT if services.usage.locked_out(user, settings) else "budget" + reason = day_refusal(services.usage, user, settings) # `locked_out`, `service_paused` or `budget` ledger.refuse(reason, refusal(reason, ledger.lang)) attribution("scope_guard", ledger, verdict=reason) return _withheld() diff --git a/agents/anyplot/prompts/refusals.yaml b/agents/anyplot/prompts/refusals.yaml index 436b631a031..63e7173f487 100644 --- a/agents/anyplot/prompts/refusals.yaml +++ b/agents/anyplot/prompts/refusals.yaml @@ -47,6 +47,22 @@ budget: vi: "Hiện đã đạt giới hạn sử dụng. Bạn vui lòng thử lại sau." ru: "Лимит использования пока исчерпан. Пожалуйста, попробуй ещё раз позже." +# The service's day budget (`AGENT_GLOBAL_DAILY_TOKEN_BUDGET`) is spent, not the user's own: +# every user gets this until the usage book's new day at 00:00 UTC. The page adds the time +# in the user's own time zone next to it (the server does not know the zone). +service_paused: + en: "The chat has reached today's overall limit and is paused until 00:00 UTC. Your own limit is not affected." + de: "Der Chat hat das heutige Gesamtlimit erreicht und ist bis 00:00 UTC pausiert. Dein eigenes Limit ist davon nicht betroffen." + fr: "Le chat a atteint la limite globale du jour et est en pause jusqu'à 00:00 UTC. Ta propre limite n'est pas concernée." + es: "El chat ha alcanzado el límite global de hoy y está en pausa hasta las 00:00 UTC. Tu propio límite no se ve afectado." + it: "La chat ha raggiunto il limite complessivo di oggi ed è in pausa fino alle 00:00 UTC. Il tuo limite personale non è interessato." + pt: "O chat atingiu o limite geral de hoje e está pausado até 00:00 UTC. O seu próprio limite não é afetado." + nl: "De chat heeft de totale limiet van vandaag bereikt en is gepauzeerd tot 00:00 UTC. Je eigen limiet blijft onaangetast." + pl: "Czat osiągnął dzisiejszy limit ogólny i jest wstrzymany do 00:00 UTC. Twój własny limit pozostaje bez zmian." + zh: "聊天已达到今天的总上限，暂停至 UTC 00:00。你自己的额度不受影响。" + vi: "Cuộc trò chuyện đã đạt giới hạn chung của hôm nay và tạm dừng đến 00:00 UTC. Giới hạn của riêng bạn không bị ảnh hưởng." + ru: "Чат достиг общего лимита на сегодня и приостановлен до 00:00 UTC. Твой личный лимит не затронут." + rate_limited: en: "That is too many messages in a short time. Please wait a minute and try again." de: "Das sind zu viele Nachrichten in kurzer Zeit. Bitte warte eine Minute und versuche es dann noch einmal." diff --git a/agents/anyplot/telemetry.py b/agents/anyplot/telemetry.py index 26b40f2d390..8ef09fce101 100644 --- a/agents/anyplot/telemetry.py +++ b/agents/anyplot/telemetry.py @@ -167,6 +167,7 @@ def _library(value: str) -> str: "capacity", "rate_limited", "daily_limit", + "service_paused", "locked_out", "file_refused", "file_failed", @@ -252,10 +253,12 @@ def _library(value: str) -> str: "locked_out", "session_closed", "budget", + "service_paused", "rate_limited", "daily_limit", "other", ] +"""`service_paused` (since 2026-11-02): the service's day budget is spent, not the user's own (`ledger.day_refusal`).""" ErrorCode = Literal["guard_unavailable", "capacity", "run_limit", "deadline", "internal", "cancelled", "other"] """`cancelled`: the client closed the stream before the turn ended (the page's cancel, a reload, a lost connection); `run_limit`: the turn's sandbox run would have been one more than `AGENT_USER_MAX_RUNS` across the user's sessions.""" @@ -1013,6 +1016,7 @@ def next_input(session_id: str) -> int: "capacity": "capacity", "rate_limited": "rate_limited", "daily_limit": "daily_limit", + "service_paused": "service_paused", "locked_out": "locked_out", "convert_unavailable": "convert_unavailable", } diff --git a/agents/main.py b/agents/main.py index 24542c5124d..1a9cdfdb08b 100644 --- a/agents/main.py +++ b/agents/main.py @@ -8,12 +8,13 @@ | Route | Body | Result | Errors | |---|---|---|---| -| `GET /v1/status` | | `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs}` (`user_max_runs`: `AGENT_USER_MAX_RUNS`) | | +| `GET /v1/status` | | `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs, paused, resets_at}` (`user_max_runs`: `AGENT_USER_MAX_RUNS`; `paused`: the service's day budget is spent, so every turn gets `service_paused` until `resets_at`, the next 00:00 UTC) | | +| `GET /v1/budget` | | `{date, since, started_at, resets_at, paused, model, tokens, budget_tokens, usd, budget_usd, users, users_at_limit, limits}`: the admin's live view of the day from the usage book (`UsageBook.snapshot`), counts only, never a user id: the service's weighted tokens against `AGENT_GLOBAL_DAILY_TOKEN_BUDGET`, both in USD at the main model's list price (`ledger.weighted_usd`, null for a model without one), the users seen today, how many are at one or more of their own day caps, and how many at each (`limits`: `tokens`, `runs`, `data_runs`, `messages`, `pastes`, `conversions`, `locked_out`). In memory: `since` is when today's counts began, the instance's start or 00:00 UTC | | | `GET /v1/eligibility?spec=&library=` | | `{eligible, status, reasons}` | | | `POST /v1/sessions` | `{user, spec_id, library, locale, snapshot, source?}` (`source`: `fanout` for a session the admin's "plot in all libraries" opens; its telemetry lines carry that source) | `{session_id, eligibility}` | `422 not_eligible` | | `POST /v1/sessions/{sid}/library` | `{library, snapshot}` | `{session_id, eligibility}` | `422 not_eligible`, `409 run_active` | -| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?, source?, table?}` (`source`: the name of the file the page read the text from, at most 1,024 characters; screened on its own, never a note; `table`: 1 to 10, which of several tables in the text to read, from the previous answer's `tables`) | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version, language, file_name?, tables?, table?}` (`file_name`: the dropped file's name as the session keeps it, `(name withheld)` when the screen refused it; `tables`: one label per table, its header names and its row count, when the text held two or more tables each under its own header after a blank line, and `table` the one read, see `data/parse.py` `_one_table`;`suggestion`: the wide-table offer of `data/shape.py` with `siblings`, the columns a role that takes labels should not take (`bindings.label_siblings`), or null; `context_lines`: the notes pasted above or below the table that passed the pre-filter and the scope judge, which the root gets with the next `summarize_data` turn; `language`: the session language as `{language, source}`, which a paste before any typed message or choice sets from the notes' or the column names' language, source `pasted`), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips that fallback and keeps every row) | `413 too_long`, `422 unparseable` (both with `cause`, the parser's sentence naming what the data fails on, at most 300 characters), `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `403 session_closed` (the scope judge closed the session), `409 run_active` (a turn, a theme toggle or another paste in flight in this session), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_PASTES_PER_MINUTE` / `_PER_DAY`, or the daily token budgets for `daily_limit`), `429 locked_out` (the day's `attack` strikes), `503 guard_unavailable` | -| `POST /v1/sessions/{sid}/dataset/convert` | `{data_base64, sheet?}` (a dropped Excel, OpenDocument, Parquet, Feather, Arrow or NumPy file, at most 5 MiB; `sheet` a sheet or `.npz` array the previous answer named) | `{text, format, kind, rows, columns, sheet, sheets, sheet_count, warnings, dropped}`: the file read in the renderer's sandbox (`harness.py --convert`) into the text the paste box takes (CSV, or JSON records for a table with list columns, `format`), every value with the precision it came with, the sheet read and at most 10 visible ones with cells (`sheet_count` all of them), `warnings` counts (`formulas_without_value`, `hidden_sheets`, `dropped_columns`, `list_columns`, `utc_timestamps`, `renamed_columns`), `dropped` the names of the columns left out; nothing stored, no model call | `413 too_long`, `422 invalid`, `422 file_` (`wire.CONVERT_REASONS`, with `measured` and `limit` for a broken cap) and `file_timeout`, `file_too_complex` (also a reader that died without its `status.json`), `file_too_large`, `file_unreadable`, `403 session_closed`, `409 run_active` (another conversion in flight in this session), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_CONVERSIONS_PER_MINUTE` / `_PER_DAY`, or the daily token budgets), `429 locked_out`, `503 capacity` (no render slot within `AGENT_CONVERT_SLOT_WAIT_S`), `503 convert_unavailable` (no renderer, one without `/convert`, or a sandbox without the harness's required exec filter) | +| `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?, source?, table?}` (`source`: the name of the file the page read the text from, at most 1,024 characters; screened on its own, never a note; `table`: 1 to 10, which of several tables in the text to read, from the previous answer's `tables`) | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version, language, file_name?, tables?, table?}` (`file_name`: the dropped file's name as the session keeps it, `(name withheld)` when the screen refused it; `tables`: one label per table, its header names and its row count, when the text held two or more tables each under its own header after a blank line, and `table` the one read, see `data/parse.py` `_one_table`;`suggestion`: the wide-table offer of `data/shape.py` with `siblings`, the columns a role that takes labels should not take (`bindings.label_siblings`), or null; `context_lines`: the notes pasted above or below the table that passed the pre-filter and the scope judge, which the root gets with the next `summarize_data` turn; `language`: the session language as `{language, source}`, which a paste before any typed message or choice sets from the notes' or the column names' language, source `pasted`), stored as version 1 (the undo history of an earlier paste is dropped); or, for free text, `{transcribed, text, preview, profile, warnings, note}`, not stored (`confirmed: true` skips that fallback and keeps every row) | `413 too_long`, `422 unparseable` (both with `cause`, the parser's sentence naming what the data fails on, at most 300 characters), `422 formula` (text of at most 2,000 characters that asks for a formula, which the chat's `generate_data` computes; never transcribed), `403 data_refused`, `403 session_closed` (the scope judge closed the session), `409 run_active` (a turn, a theme toggle or another paste in flight in this session), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_PASTES_PER_MINUTE` / `_PER_DAY`, or the user's daily token budget for `daily_limit`), `429 service_paused` (the service's day budget), `429 locked_out` (the day's `attack` strikes), `503 guard_unavailable` | +| `POST /v1/sessions/{sid}/dataset/convert` | `{data_base64, sheet?}` (a dropped Excel, OpenDocument, Parquet, Feather, Arrow or NumPy file, at most 5 MiB; `sheet` a sheet or `.npz` array the previous answer named) | `{text, format, kind, rows, columns, sheet, sheets, sheet_count, warnings, dropped}`: the file read in the renderer's sandbox (`harness.py --convert`) into the text the paste box takes (CSV, or JSON records for a table with list columns, `format`), every value with the precision it came with, the sheet read and at most 10 visible ones with cells (`sheet_count` all of them), `warnings` counts (`formulas_without_value`, `hidden_sheets`, `dropped_columns`, `list_columns`, `utc_timestamps`, `renamed_columns`), `dropped` the names of the columns left out; nothing stored, no model call | `413 too_long`, `422 invalid`, `422 file_` (`wire.CONVERT_REASONS`, with `measured` and `limit` for a broken cap) and `file_timeout`, `file_too_complex` (also a reader that died without its `status.json`), `file_too_large`, `file_unreadable`, `403 session_closed`, `409 run_active` (another conversion in flight in this session), `429 rate_limited` / `429 daily_limit` (`AGENT_USER_CONVERSIONS_PER_MINUTE` / `_PER_DAY`, or the user's daily token budget), `429 service_paused`, `429 locked_out`, `503 capacity` (no render slot within `AGENT_CONVERT_SLOT_WAIT_S`), `503 convert_unavailable` (no renderer, one without `/convert`, or a sandbox without the harness's required exec filter) | | `PUT /v1/sessions/{sid}/bindings` | `[{role, column}]` | `{bindings, complete, missing_roles, warnings}`: `warnings` names a role that needs values of 0 or more bound to a column with a negative value, or left open for that reason (`data/signs.py`); never a refusal | `409 run_active`, `422 invalid` (with `errors`, at most 20 lines) | | `PUT /v1/sessions/{sid}/language` | `{language}` (a short name or code, `^[A-Za-z][A-Za-z -]{0,39}$`) | `{language, source, suggestion?}`: the session language in its stored form (`English` is `en`), source `chosen`, which detection never overrides; `suggestion`, when the session has data, the `.to_long()` offer (or null) with its new column names in that language | `409 run_active`, `422` (the pattern) | | `POST /v1/sessions/{sid}/dataset/confirm` | | `{confirmed, rows, columns}`: step 1 of two, the user confirmed the dataset with its bindings; the pipeline runs only then | `409 run_active`, `422 no_dataset` | @@ -59,6 +60,8 @@ Before the scope judge runs, and before any model call, the route answers with a fixed refusal right after `ready`: `locked_out` for a user past the day's `attack` strikes (`AGENT_ATTACK_STRIKES`), `budget` for a user over the daily token budget, +`service_paused` for everyone else once the service's day budget is spent +(`AGENT_GLOBAL_DAILY_TOKEN_BUDGET`; a paste or a conversion `429 service_paused`), `session_closed` in a session the scope judge closed (`AGENT_SESSION_REFUSALS` refusals of type `out_of_scope` or `attack`), and `rate_limited` or `daily_limit` past the user's message limits (`AGENT_USER_MESSAGES_PER_MINUTE`, @@ -194,8 +197,12 @@ RequestLedger, attribution, budget_allows, + day_refusal, judge_budget_tokens, + next_reset, session_hash, + utc_stamp, + weighted_usd, ) from agents.anyplot.plugins.scope_guard import LOCKED_OUT, SESSION_CLOSED, scope_refusals from agents.anyplot.policy import fence, file_name_block, refusal, scope_rubric, transcribe_rubric, user_notes_block @@ -840,7 +847,9 @@ def _stamp(ledger: RequestLedger, view: SessionView | None, session: Session) -> async def status(runtime: RuntimeDep) -> dict[str, Any]: """The service's configuration, plus the run queue: `waiting` entries and runs `in_flight`. - `user_max_runs` is `AGENT_USER_MAX_RUNS`, which the admin chat's fan-out over every library keeps to. + `user_max_runs` is `AGENT_USER_MAX_RUNS`, which the admin chat's fan-out over every library keeps to. `paused` + says the service's day budget is spent, so every turn gets `service_paused` until `resets_at` (the next 00:00 + UTC): the page shows the note before the user types. """ settings = get_settings() queue = runtime.run_queue() @@ -854,9 +863,42 @@ async def status(runtime: RuntimeDep) -> dict[str, Any]: "waiting": queue.waiting_count, "in_flight": queue.in_flight, "user_max_runs": settings.user_max_runs, + "paused": get_services().usage.paused(settings), + "resets_at": utc_stamp(next_reset()), } +@app.get("/v1/budget", dependencies=v1_dependencies) +async def day_budget() -> dict[str, Any]: + """The admin's live view of the day (the BFF's `/debug/agent/budget`, behind its admin gate): the service's spend + against `AGENT_GLOBAL_DAILY_TOKEN_BUDGET` and how many users are at their own day caps. Counts only, never a user + id. The counts live in this instance's memory: `since` is when they began today, and a redeploy or a + scale-to-zero starts them again at zero (`ledger.UsageBook.snapshot`).""" + settings = get_settings() + usage = get_services().usage + day = usage.snapshot(settings) + budget_tokens = settings.global_daily_token_budget + return { + "date": day.date, + "since": utc_stamp(day.since), + "started_at": utc_stamp(day.started_at), + "resets_at": utc_stamp(next_reset()), + "paused": usage.paused(settings), + "model": settings.model, + "tokens": day.global_tokens, + "budget_tokens": budget_tokens, + "usd": _usd(weighted_usd(day.global_tokens, settings.model, settings.location)), + "budget_usd": _usd(weighted_usd(budget_tokens, settings.model, settings.location)), + "users": day.users, + "users_at_limit": day.at_limit, + "limits": day.limits, + } + + +def _usd(value: float | None) -> float | None: + return round(value, 4) if value is not None else None + + @app.get("/v1/eligibility", dependencies=v1_dependencies) async def eligibility( runtime: RuntimeDep, _: User, spec: Annotated[str, Query(pattern=SPEC_PATTERN)], library: Annotated[str, Query()] @@ -1183,10 +1225,12 @@ async def _convert( def _paste_budget(services: Services, user: str, settings: AgentSettings) -> None: - """`429 locked_out` past the day's `attack` strikes, `429 daily_limit` past the daily token budgets.""" + """`429 locked_out` past the day's `attack` strikes, `429 daily_limit` past the user's daily token budget, and + `429 service_paused` when only the service's is spent (`ledger.day_refusal`).""" if services.usage.daily_ok(user, settings): return - raise AgentsError(429, LOCKED_OUT if services.usage.locked_out(user, settings) else "daily_limit") + code = day_refusal(services.usage, user, settings) + raise AgentsError(429, "daily_limit" if code == "budget" else code) async def _ingest_paste( @@ -2058,8 +2102,10 @@ async def post_message( # Past the day's `attack` strikes: its own refusal, never the misleading "usage limit" text (2026-10-14). return _refuse_now(translator, ledger, facts, LOCKED_OUT, "budget_halt", agent="route") if not budget_allows(ledger, services.usage, settings): - # The refusal the root would send, sent now: a user over the daily budget spends nothing. - return _refuse_now(translator, ledger, facts, "budget", "budget_halt", agent="route") + # The refusal the root would send, sent now: a user over the daily budget spends nothing. When only the + # service's day budget is spent, every user gets `service_paused`, which says their own limit is untouched. + code = day_refusal(services.usage, user, settings) + return _refuse_now(translator, ledger, facts, code, "budget_halt", agent="route") limit = services.usage.message_limit(user, settings) if limit is not None: return _refuse_now(translator, ledger, facts, limit, "message_limit") diff --git a/agents/stream.py b/agents/stream.py index 472083b809b..e721c73954b 100644 --- a/agents/stream.py +++ b/agents/stream.py @@ -10,7 +10,7 @@ | `status` | `step`, `attempt`, and on `repairing` `outcome`, `findings`, `defects` | the pipeline's content-free `custom_metadata` progress events; the step that opens a repair round says how the round before it ended (`RoundOutcome`, a count) and the rubric ids it repairs (`VQ-05`, closed codes) | | `message` | `text` | a final, non-partial text response authored by the root (`anyplot`) | | `plot` | `status`, `reason`, `attempts`, `artifacts`, `changes`, `residual_defects`, `version`, `rounds` | the pipeline's `PlotResult` output event; `version`, the stored version's number, only on `ok` and `needs_attention`; `rounds`, how each attempt ended (`{attempt, outcome, findings, step, shipped, defects}`: the step it ended in, whether its render is the one shipped, and the rubric ids it passed on to the repair) | -| `refusal` | `code`, `text` | the request ledger's refusal, in place of the message: `out_of_scope`, `needs_data` (data that would have to be fetched), `other_library` (this plot in another library or language: a fixed pointer to the catalogue, not counted toward closing by default), `unsupported_content`, `locked_out` (past the day's `attack` strikes) and `session_closed` (the session's third counted refusal) from the scope guard, `budget` from the budgets; the refusal one before the closing one ends with the fixed `close_warning` sentence; right after `ready`, before the scope judge: `locked_out`, `budget` for a user over the daily budget, `rate_limited` or `daily_limit` past the user's message limits, `session_closed` in a closed session | +| `refusal` | `code`, `text` | the request ledger's refusal, in place of the message: `out_of_scope`, `needs_data` (data that would have to be fetched), `other_library` (this plot in another library or language: a fixed pointer to the catalogue, not counted toward closing by default), `unsupported_content`, `locked_out` (past the day's `attack` strikes) and `session_closed` (the session's third counted refusal) from the scope guard, `budget` from the budgets (`service_paused` when only the service's day budget is spent, for every user); the refusal one before the closing one ends with the fixed `close_warning` sentence; right after `ready`, before the scope judge: `locked_out`, `budget` for a user over the daily budget, `rate_limited` or `daily_limit` past the user's message limits, `session_closed` in a closed session | | `error` | `code`, `ref` | `guard_unavailable`, `capacity` (also when a sandbox run found the queue full or waited its maximum; the turn ends there, without a `plot`), `run_limit` (the user already has `AGENT_USER_MAX_RUNS` sandbox runs queued or running in other sessions; the turn ends the same way), `deadline` or `internal` | | `done` | `llm_calls`, `tokens`, and after a run `bindings`, `data_confirmed`, `language` (`{language, source}`, the session language and how it was set), `suggestion` (the `.to_long()` offer in the session language, or null, when the session has data) and, when the turn changed the data (`prepare_data`, `generate_data`), `dataset` | the end of every run, always last; the route adds the session's sync fields (`_session_sync` in `agents/main.py`) | diff --git a/api/agent_usage.py b/api/agent_usage.py index 461a847741f..3585868fa4a 100644 --- a/api/agent_usage.py +++ b/api/agent_usage.py @@ -20,6 +20,16 @@ `AGENT_TELEMETRY_DATASET` unset answers `not_configured`; a query that fails (no table yet, no permission, a timeout) answers `unavailable`, logged with the exception class only. Both carry zeros, so the page shows its empty state and never an error. Neither is cached. + +The per-day table (`GET /debug/agent-usage/daily`, `daily_sql`, `read_daily`) answers the last +`DAILY_DAYS` UTC days, newest first, from a second query under the same rules: its window starts +at 00:00 UTC `DAILY_DAYS - 1` days before today, constant for the whole day, so within a day the +query text and its parameter stay byte-identical too. Each day counts the live chats that began +on it, the plots shipped, the cost (the `cost` view's sum of every turn step and data-input judge +call, priced by the views dataset's `model_prices`), the limit refusals by code +(`LIMIT_CODES`) and the 95th percentile of the run queue's wait (`pipeline_run.queue_wait_ms`, +null on a day without it). Fan-out lines cost real money against the same day budget, so their +cost is counted apart (`fanout_cost_usd`); everything else is live only. """ from __future__ import annotations @@ -63,6 +73,15 @@ `downloaded_page` (a JavaScript plot's `plot.html`, since JS-4b) keeps one like the code download.""" REJECT_REACTIONS = ("thumbs_down", "bug") """Feedback reactions that make a session not kept, whatever else the user did.""" +DAILY_DAYS = 14 +"""The per-day table's rows: today and the 13 UTC days before it.""" +LIMIT_CODES = ("service_paused", "budget", "daily_limit", "rate_limited", "run_limit", "capacity") +"""The refusals and errors a limit gives, counted per day wherever a line records one: a `chat_turn`'s `refusal` or +`error`, a `data_input`'s `outcome`. `service_paused` is the service's day budget (since 2026-11-02; before, the same +event read `budget` in a turn and `daily_limit` in a paste), `budget` a user's own token budget, `daily_limit` and +`rate_limited` a user's turns or pastes per day and per minute, `run_limit` a user's sandbox runs at once, `capacity` +a full run queue or an expired wait. `locked_out` is a safety lockout, not a usage limit: the card's refusal list has +it.""" REJECT_LOOKBACK_DAYS = 2 """How far before the window a session's rejections and runs are still read (reviews of #12207): a session that straddles the window's start keeps a thumbs_down sent before it, and a plot shipped before it and kept inside it @@ -97,6 +116,35 @@ class AgentUsage(BaseModel): generated_at: str +class UsageDay(BaseModel): + """One UTC day of the per-day table.""" + + date: str + sessions: int = Field(default=0, ge=0) + """Live chats whose first line falls on this day; a chat that wrote no line (opened, nothing sent) is not one.""" + plots_shipped: int = Field(default=0, ge=0) + cost_usd: float = Field(default=0.0, ge=0) + """The live lines' model cost at list price, as the `cost` view prices it.""" + fanout_cost_usd: float = Field(default=0.0, ge=0) + """The same for the admin fan-out's sessions, counted apart.""" + unpriced_calls: int = Field(default=0, ge=0) + """Steps with tokens whose model has no `model_prices` row: their cost is missing from both sums.""" + limit_hits: dict[str, int] = Field(default_factory=dict) + """The count of each of `LIMIT_CODES`, every code present.""" + queue_wait_p95_ms: int | None = None + """The 95th percentile of the live runs' queue wait; None on a day whose runs predate the field.""" + queue_waits: int = Field(default=0, ge=0) + """The runs behind that percentile.""" + + +class AgentUsageDaily(BaseModel): + status: UsageStatus + days: int = DAILY_DAYS + rows: list[UsageDay] = Field(default_factory=list) + """Every day of the window, newest first, a day without lines with zeros; empty unless `status` is `ok`.""" + generated_at: str + + class UsageQuery(Protocol): """Runs one standard-SQL query with a TIMESTAMP `@since` parameter and returns its rows as dicts.""" @@ -294,6 +342,124 @@ def usage_sql(dataset: str) -> str: """ +def daily_start(now: datetime | None = None) -> datetime: + """The per-day table's start: 00:00 UTC `DAILY_DAYS - 1` days before today, the same for the whole day.""" + current = (now or datetime.now(UTC)).astimezone(UTC) + return datetime.combine(current.date() - timedelta(days=DAILY_DAYS - 1), datetime.min.time(), UTC) + + +SESSION_LOOKBACK_DAYS = REJECT_LOOKBACK_DAYS +"""How far before the per-day window a chat's first line is still looked for: a chat that began before the window and +went on inside it is not a new one on the window's first day. Sessions live in one instance's memory.""" +MODEL_SUFFIX = r"(@.*|-[0-9]{8})$" +"""What the `cost` view strips from a step's model id before it looks up the price: a version suffix.""" + + +def daily_sql(dataset: str) -> str: + """The query behind the per-day table: the window's lines of the linked log dataset, deduplicated, per UTC day. + + The `timestamp` filter comes before the deduplication, as in `usage_sql`; the lines from up to + `SESSION_LOOKBACK_DAYS` before the window only date the chats, every count reads `ts >= @since`. The cost + is the `cost` view's (`agents/telemetry/views/05_prices_cost.sql`) over the window: every `chat_turn` step and + every data input's judge calls, priced from the views dataset's `model_prices`, which reads nothing; a + `pipeline_run`'s steps repeat its turn's and are left out, so no token is priced twice. A model without a + price row counts in `unpriced_calls` instead of the sums. The limit codes are counted wherever a line records + one (`LIMIT_CODES`), and the queue wait from the `pipeline_run` lines that carry `queue_wait_ms`. + """ + hits = ",\n ".join(f"COUNTIF(code = '{code}') AS hits_{code}" for code in LIMIT_CODES) + return f""" +WITH ev AS ( + SELECT + timestamp AS ts, + JSON_VALUE(json_payload.event) AS event, + JSON_VALUE(json_payload.source) AS source, + JSON_VALUE(json_payload.session) AS session, + json_payload AS payload + FROM `{dataset}{LOGS_SUFFIX}._AllLogs` + WHERE timestamp >= TIMESTAMP_SUB(@since, INTERVAL {SESSION_LOOKBACK_DAYS} DAY) + AND JSON_VALUE(json_payload.log_type) = 'agent_telemetry' + AND JSON_VALUE(json_payload.source) IN ('live', 'fanout') + QUALIFY ROW_NUMBER() OVER (PARTITION BY JSON_VALUE(json_payload.idempotency_key) ORDER BY timestamp) = 1 +), +chats AS ( + SELECT DATE(first_ts) AS day, COUNT(*) AS sessions + FROM (SELECT session, MIN(ts) AS first_ts FROM ev WHERE source = 'live' GROUP BY session) + WHERE first_ts >= @since + GROUP BY day +), +lines AS ( + SELECT + DATE(ts) AS day, + COUNTIF(event = 'pipeline_run' AND JSON_VALUE(payload.status) IN ('ok', 'needs_attention')) AS plots_shipped, + {hits}, + APPROX_QUANTILES(queue_wait_ms, 100 IGNORE NULLS)[SAFE_OFFSET(95)] AS queue_wait_p95_ms, + COUNT(queue_wait_ms) AS queue_waits + FROM ( + SELECT + ts, event, payload, + CASE event + WHEN 'chat_turn' THEN IFNULL(JSON_VALUE(payload.refusal), JSON_VALUE(payload.error)) + WHEN 'data_input' THEN JSON_VALUE(payload.outcome) + END AS code, + IF(event = 'pipeline_run', LAX_INT64(payload.queue_wait_ms), NULL) AS queue_wait_ms + FROM ev + WHERE ts >= @since AND source = 'live' + ) + GROUP BY day +), +steps AS ( + SELECT + DATE(ts) AS day, + source, + REGEXP_REPLACE(JSON_VALUE(step.model), r'{MODEL_SUFFIX}', '') AS model, + IFNULL(LAX_INT64(step.input), 0) AS input_tokens, + IFNULL(LAX_INT64(step.output), 0) AS output_tokens, + IFNULL(LAX_INT64(step.cached), 0) AS cached_tokens, + IFNULL(LAX_INT64(step.cache_write), 0) AS cache_write_tokens + FROM ev, UNNEST(JSON_QUERY_ARRAY(payload.steps)) AS step + WHERE ts >= @since AND event = 'chat_turn' + UNION ALL + SELECT + DATE(ts), + source, + REGEXP_REPLACE(JSON_VALUE(payload.judge_model), r'{MODEL_SUFFIX}', ''), + IFNULL(LAX_INT64(payload.judge_input), 0), + IFNULL(LAX_INT64(payload.judge_output), 0), + 0, + 0 + FROM ev + WHERE ts >= @since AND event = 'data_input' +), +costs AS ( + SELECT + day, + SUM(IF(source = 'live', usd, 0)) AS cost_usd, + SUM(IF(source = 'fanout', usd, 0)) AS fanout_cost_usd, + COUNTIF(usd IS NULL) AS unpriced_calls + FROM ( + SELECT + steps.day, + steps.source, + ( + GREATEST(steps.input_tokens - steps.cached_tokens - steps.cache_write_tokens, 0) * prices.input_usd_per_m + + steps.cached_tokens * prices.input_usd_per_m * 0.1 + + steps.cache_write_tokens * prices.input_usd_per_m * 1.25 + + steps.output_tokens * prices.output_usd_per_m + ) / 1e6 AS usd + FROM steps + LEFT JOIN `{dataset}.model_prices` AS prices USING (model) + WHERE steps.input_tokens + steps.output_tokens > 0 + ) + GROUP BY day +) +SELECT * +FROM chats +FULL JOIN lines USING (day) +FULL JOIN costs USING (day) +ORDER BY day DESC +""" + + def _now() -> str: return datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z") @@ -317,7 +483,18 @@ def _rate(value: Any) -> float | None: return round(float(value), 4) if isinstance(value, int | float) else None +def _usd(value: Any) -> float: + return ( + round(float(value), 4) if isinstance(value, int | float) and not isinstance(value, bool) and value > 0 else 0.0 + ) + + +def _ms(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + _CACHE: dict[tuple[str, int], tuple[float, AgentUsage]] = {} +_DAILY_CACHE: dict[tuple[str, str], tuple[float, AgentUsageDaily]] = {} _CACHE_LOCK = threading.Lock() @@ -325,6 +502,7 @@ def clear_usage_cache() -> None: """Forget every kept answer (tests, and a dataset change at runtime).""" with _CACHE_LOCK: _CACHE.clear() + _DAILY_CACHE.clear() def read_usage( @@ -370,6 +548,54 @@ def read_usage( return usage +def _usage_day(date: str, row: dict[str, Any]) -> UsageDay: + return UsageDay( + date=date, + sessions=_count(row.get("sessions")), + plots_shipped=_count(row.get("plots_shipped")), + cost_usd=_usd(row.get("cost_usd")), + fanout_cost_usd=_usd(row.get("fanout_cost_usd")), + unpriced_calls=_count(row.get("unpriced_calls")), + limit_hits={code: _count(row.get(f"hits_{code}")) for code in LIMIT_CODES}, + queue_wait_p95_ms=_ms(row.get("queue_wait_p95_ms")), + queue_waits=_count(row.get("queue_waits")), + ) + + +def read_daily( + query: UsageQuery | None, + dataset: str | None, + *, + now: datetime | None = None, + clock: Callable[[], float] = time.monotonic, +) -> AgentUsageDaily: + """The per-day table for the last `DAILY_DAYS` UTC days, newest first; `not_configured` or `unavailable` without + rows otherwise. Kept like `read_usage`'s answer, per dataset and window start, so a new UTC day queries anew.""" + if not dataset or not DATASET_PATTERN.fullmatch(dataset): + return AgentUsageDaily(status="not_configured", generated_at=_now()) + since = daily_start(now) + key = (dataset, since.date().isoformat()) + with _CACHE_LOCK: + kept = _DAILY_CACHE.get(key) + if kept is not None and clock() < kept[0]: + return kept[1] + if query is None: + return AgentUsageDaily(status="unavailable", generated_at=_now()) + try: + rows = query.rows(daily_sql(dataset), since) + except Exception as exc: # as in `read_usage`: the table shows its empty state + logger.warning("agent daily usage query failed: %s", type(exc).__name__) + return AgentUsageDaily(status="unavailable", generated_at=_now()) + by_day = {row["day"]: row for row in rows if isinstance(row.get("day"), str)} + dates = [(since + timedelta(days=offset)).date().isoformat() for offset in reversed(range(DAILY_DAYS))] + daily = AgentUsageDaily( + status="ok", rows=[_usage_day(date, by_day.get(date, {})) for date in dates], generated_at=_now() + ) + with _CACHE_LOCK: + _DAILY_CACHE[key] = (clock() + CACHE_TTL_S, daily) + return daily + + _CLIENTS: dict[str, UsageQuery] = {} _CLIENTS_LOCK = threading.Lock() diff --git a/api/routers/agent.py b/api/routers/agent.py index 6ff88939c5e..7ecd630d0d2 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -247,6 +247,7 @@ "unconfirmed", "rate_limited", "daily_limit", + "service_paused", "locked_out", "session_closed", "convert_unavailable", @@ -1113,12 +1114,45 @@ async def agent_status(ctx: Ctx, client: Client) -> dict[str, Any]: """The agents service's status plus `enabled`. Upstream fields: `libraries`, `model`, `location`, `version`, `provider`, the - run queue's `waiting` entries and `in_flight` runs, and `user_max_runs`. + run queue's `waiting` entries and `in_flight` runs, `user_max_runs`, and `paused` + with `resets_at` (the service's day budget is spent until then). """ upstream = await _call_upstream(client, ctx, "GET", "/status") return {**(upstream if isinstance(upstream, dict) else {}), "enabled": True} +BUDGET_FIELDS = frozenset( + { + "date", + "since", + "started_at", + "resets_at", + "paused", + "model", + "tokens", + "budget_tokens", + "usd", + "budget_usd", + "users", + "users_at_limit", + "limits", + } +) +"""The fields of the agents service's `GET /v1/budget` that reach the page: counts, times and the model id, never a +user id, so a field the service adds later stays upstream until it is named here.""" + + +@router.get("/budget", dependencies=[Depends(require_admin)]) +async def agent_budget(ctx: Ctx, client: Client) -> dict[str, Any]: + """The `/debug` page's live line for today: the agents service's global spend against its day budget, and how + many users are at their own limits, for the instance's lifetime. Admin-gated on the route itself as well as by + the router, so it stays an admin view once the chat routes open to everyone.""" + upstream = await _call_upstream(client, ctx, "GET", "/budget") + return { + key: value for key, value in (upstream if isinstance(upstream, dict) else {}).items() if key in BUDGET_FIELDS + } + + @router.get("/eligibility") async def agent_eligibility( ctx: Ctx, diff --git a/api/routers/debug.py b/api/routers/debug.py index 9069847a883..b7470a63fa2 100644 --- a/api/routers/debug.py +++ b/api/routers/debug.py @@ -16,7 +16,7 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession -from api.agent_usage import USAGE_WINDOWS, AgentUsage, UsageQuery, read_usage, usage_query +from api.agent_usage import USAGE_WINDOWS, AgentUsage, AgentUsageDaily, UsageQuery, read_daily, read_usage, usage_query from api.cache import clear_cache, get_cache_stats from api.dependencies import require_db from api.exceptions import raise_validation_error @@ -531,6 +531,17 @@ async def agent_usage( return await run_in_threadpool(read_usage, query, settings.agent_telemetry_dataset, days) +@router.get("/agent-usage/daily", response_model=AgentUsageDaily, dependencies=[Depends(require_admin)]) +async def agent_usage_daily(query: UsageQuery | None = Depends(get_usage_query)) -> AgentUsageDaily: + """The agent chat's last 14 UTC days, one row each, newest first, read-only from the BigQuery telemetry. + + Per day: new live chats, plots shipped, the cost in USD (live, and the admin fan-out's apart), the limit + refusals by code and the run queue's 95th-percentile wait. The same statuses as `/agent-usage`, with no rows + (`api/agent_usage.py` `read_daily`, `docs/reference/agent-telemetry.md`). + """ + return await run_in_threadpool(read_daily, query, settings.agent_telemetry_dataset) + + class CacheInvalidateResponse(BaseModel): cleared: int maxsize: int diff --git a/app/src/hooks/useAgentSession.ts b/app/src/hooks/useAgentSession.ts index 097162a2777..efa61c27517 100644 --- a/app/src/hooks/useAgentSession.ts +++ b/app/src/hooks/useAgentSession.ts @@ -196,6 +196,7 @@ import { CONFIRM_LOST_TEXT, LANGUAGE_DETECTED_TEXT, LANGUAGE_PASTED_TEXT, + nextUtcMidnight, } from 'src/sections/agent-chat/messages'; import { reloadOnceForAccess } from 'src/utils/adminAuth'; @@ -272,7 +273,8 @@ export type ChatItem = | { id: number; kind: 'user'; text: string } | { id: number; kind: 'action'; action: TurnAction; library: string } | { id: number; kind: 'assistant'; text: string } - | { id: number; kind: 'refusal'; code: string; text: string } + /** `until`: when a `service_paused` refusal ends (the next 00:00 UTC as it arrived). */ + | { id: number; kind: 'refusal'; code: string; text: string; until?: string } | { id: number; kind: 'error'; code: string; ref: string | null } | { id: number; kind: 'plot'; version: number } | { id: number; kind: 'plot_failed'; result: PlotResult } @@ -1078,6 +1080,8 @@ const GUARDRAIL_REASONS = new Set([ // a user paused for the day after repeated attack verdicts. 'other_library', 'locked_out', + // The whole service paused for the day: its day budget is spent, not the user's own. + 'service_paused', ]); /** The paste route's guard refusals (HTTP errors, not stream refusals) that count as guardrail blocks. */ @@ -1085,6 +1089,7 @@ const PASTE_GUARDRAIL_REASONS = new Set([ 'data_refused', 'rate_limited', 'daily_limit', + 'service_paused', 'locked_out', 'session_closed', ]); @@ -1680,7 +1685,12 @@ export function useAgentSession({ onPlot(event.result); break; case 'refusal': - addItem({ kind: 'refusal', code: event.code, text: event.text }); + addItem({ + kind: 'refusal', + code: event.code, + text: event.text, + ...(event.code === 'service_paused' ? { until: nextUtcMidnight() } : {}), + }); trackEvent('agent_guardrail_block', { reason: GUARDRAIL_REASONS.has(event.code) ? event.code : 'other', }); diff --git a/app/src/lib/agent.ts b/app/src/lib/agent.ts index 58312fb1606..5fbd3468afd 100644 --- a/app/src/lib/agent.ts +++ b/app/src/lib/agent.ts @@ -111,6 +111,10 @@ export interface AgentStatus { in_flight?: number; /** `AGENT_USER_MAX_RUNS`: one user's sandbox runs queued or running at once; absent from an older server. */ user_max_runs?: number; + /** The service's day budget is spent: every turn gets `service_paused` until `resets_at`. */ + paused?: boolean; + /** The next 00:00 UTC, when the day budget starts again (ISO 8601). */ + resets_at?: string; enabled: boolean; } diff --git a/app/src/pages/AgentChatPage.test.tsx b/app/src/pages/AgentChatPage.test.tsx index e93e25d8209..c1a4a7163a7 100644 --- a/app/src/pages/AgentChatPage.test.tsx +++ b/app/src/pages/AgentChatPage.test.tsx @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ThemeProvider } from '@mui/material/styles'; import { AgentChatPage } from 'src/pages/AgentChatPage'; -import { LEAVE_TEXT } from 'src/sections/agent-chat/messages'; +import { LEAVE_TEXT, servicePausedText } from 'src/sections/agent-chat/messages'; import { theme } from 'src/theme'; import { ADMIN_HINT_KEY } from 'src/utils/adminAuth'; @@ -1765,3 +1765,51 @@ describe('AgentChatPage URL', () => { expect(query().get('library')).toBe('matplotlib'); }); }); + +describe('AgentChatPage service pause', () => { + /** The status answer as `stubFetch` gives it, plus `extra`. */ + function statusWith(extra: Record) { + const base = fetchMock.getMockImplementation() as ( + input: string, + init?: RequestInit + ) => Promise; + fetchMock.mockImplementation((input: string, init?: RequestInit) => + input.endsWith('/debug/agent/status') + ? Promise.resolve( + json(200, { libraries: ['matplotlib', 'seaborn'], enabled: true, ...extra }) + ) + : base(input, init) + ); + } + + it('says before the user types that the whole chat paused, until when in their time', async () => { + const resetsAt = new Date(Date.now() + 3_600_000).toISOString(); + statusWith({ paused: true, resets_at: resetsAt }); + localStorage.setItem(ADMIN_HINT_KEY, '1'); + renderAt(PAGE); + + const note = await screen.findByTestId('service-paused'); + expect(note).toHaveAttribute('role', 'note'); + expect(note).toHaveTextContent(`# ${servicePausedText(resetsAt)}`); + expect(note).toHaveTextContent(/your own limit is not affected$/); + // The composer stays: the note may go stale at midnight, the server decides each turn. + expect(screen.getByLabelText('Message')).toBeInTheDocument(); + }); + + it('shows no note while the service runs, nor for a reset already past', async () => { + statusWith({ paused: false, resets_at: new Date(Date.now() + 3_600_000).toISOString() }); + localStorage.setItem(ADMIN_HINT_KEY, '1'); + const { unmount } = renderAt(PAGE); + await screen.findByLabelText('Message'); + await waitFor(() => + expect(agentCalls().some(([input]) => pathOf(input).endsWith('/status'))).toBe(true) + ); + expect(screen.queryByTestId('service-paused')).not.toBeInTheDocument(); + unmount(); + + statusWith({ paused: true, resets_at: new Date(Date.now() - 1_000).toISOString() }); + renderAt(PAGE); + await screen.findByLabelText('Message'); + expect(screen.queryByTestId('service-paused')).not.toBeInTheDocument(); + }); +}); diff --git a/app/src/pages/AgentChatPage.tsx b/app/src/pages/AgentChatPage.tsx index 9fd5885d659..3866fcd26c4 100644 --- a/app/src/pages/AgentChatPage.tsx +++ b/app/src/pages/AgentChatPage.tsx @@ -102,7 +102,7 @@ import { DATA_SUMMARY_ID, DataPanel } from 'src/sections/agent-chat/DataPanel'; import { FanOut } from 'src/sections/agent-chat/FanOut'; import { DEFAULT_USER_MAX_RUNS } from 'src/sections/agent-chat/fanoutRules'; import { LanguageChip } from 'src/sections/agent-chat/LanguageChip'; -import { ERROR_TEXT, LEAVE_TEXT } from 'src/sections/agent-chat/messages'; +import { ERROR_TEXT, LEAVE_TEXT, servicePausedText } from 'src/sections/agent-chat/messages'; import { ProgressTimeline } from 'src/sections/agent-chat/ProgressTimeline'; import { ReloadHint } from 'src/sections/agent-chat/ReloadHint'; import { SaveButton } from 'src/sections/agent-chat/SaveButton'; @@ -309,6 +309,8 @@ function AgentChat({ const [maxRuns, setMaxRuns] = useState(DEFAULT_USER_MAX_RUNS); // Both answers below are in (or failed): before that, the libraries and the limit are fallbacks. const [listsSettled, setListsSettled] = useState(false); + // The service's day budget is spent: until when (ISO), for the note above the composer. + const [pausedUntil, setPausedUntil] = useState(null); const [fanOut, setFanOut] = useState<{ sourceSession: string; upload?: DatasetUpload; @@ -376,12 +378,29 @@ function AgentChat({ if (typeof status.user_max_runs === 'number' && status.user_max_runs > 0) { setMaxRuns(status.user_max_runs); } + if ( + status.paused === true && + typeof status.resets_at === 'string' && + Date.parse(status.resets_at) > Date.now() + ) { + setPausedUntil(status.resets_at); + } }) .catch(() => undefined) .finally(settle); return () => controller.abort(); }, [specId, token]); + // The pause ends with the service's new day: the note goes at that moment, without a reload. + useEffect(() => { + if (!pausedUntil) return; + const timer = window.setTimeout( + () => setPausedUntil(null), + Math.max(0, Date.parse(pausedUntil) - Date.now()) + ); + return () => window.clearTimeout(timer); + }, [pausedUntil]); + useEffect(() => { if (state.phase === 'unauthorized') setAdminHint(false); }, [state.phase]); @@ -875,6 +894,22 @@ function AgentChat({ : '# step 2 — create the plot as it is, or say first what it should do differently'} )} + {/* Told before the user types: every turn would get `service_paused` until then. */} + {pausedUntil && ( + + # {servicePausedText(pausedUntil)} + + )} { generated_at: '2026-10-16T12:00:00Z', }; - /** The page's own calls answered as in the tests above; the usage call by `usage`. */ - function stubFetch(usage: (days: string) => Promise) { + /** The page's own calls answered as in the tests above; the usage call by `usage`, the day views by theirs. */ + function stubFetch( + usage: (days: string) => Promise, + days: () => Promise = () => ok({ status: 'not_configured', rows: [] }), + today: () => Promise = () => Promise.resolve({ ok: false, json: () => ({}) }) + ) { const fetchMock = vi.fn((url: string) => { + if (url.includes('/debug/agent-usage/daily')) return days(); + if (url.includes('/debug/agent/budget')) return today(); if (url.includes('/debug/agent-usage')) { return usage(new URL(url).searchParams.get('days') ?? ''); } @@ -497,7 +503,7 @@ describe('DebugPage agent usage', () => { 'none in this window' ); const urls = fetchMock.mock.calls.map(call => call[0]); - expect(urls.filter(url => url.includes('/debug/agent-usage'))).toEqual([ + expect(urls.filter(url => url.includes('/debug/agent-usage?'))).toEqual([ expect.stringMatching(/\/debug\/agent-usage\?days=7$/), ]); }); @@ -561,11 +567,155 @@ describe('DebugPage agent usage', () => { ); const usageUrls = fetchMock.mock.calls .map(call => call[0]) - .filter(url => url.includes('/debug/agent-usage')); + .filter(url => url.includes('/debug/agent-usage?')); expect(usageUrls).toEqual([ expect.stringMatching(/\?days=7$/), expect.stringMatching(/\?days=30$/), ]); + // The day views do not follow the toggle: one call each. + const calls = fetchMock.mock.calls.map(call => call[0]); + expect(calls.filter(url => url.includes('/debug/agent-usage/daily'))).toHaveLength(1); + expect(calls.filter(url => url.includes('/debug/agent/budget'))).toHaveLength(1); + }); + + const DAY = { + date: '2026-11-02', + sessions: 4, + plots_shipped: 3, + cost_usd: 0.0228, + fanout_cost_usd: 0.1211, + unpriced_calls: 0, + limit_hits: { + service_paused: 2, + budget: 0, + daily_limit: 0, + rate_limited: 0, + run_limit: 0, + capacity: 1, + }, + queue_wait_p95_ms: 12_400, + queue_waits: 3, + }; + const QUIET = { + ...DAY, + date: '2026-11-01', + sessions: 0, + plots_shipped: 0, + cost_usd: 0, + fanout_cost_usd: 0, + unpriced_calls: 2, + limit_hits: { ...DAY.limit_hits, service_paused: 0, capacity: 0 }, + queue_wait_p95_ms: null, + queue_waits: 0, + }; + const BUDGET = { + date: '2026-11-02', + since: '2026-11-02T14:05:12Z', + started_at: '2026-11-02T14:05:12Z', + resets_at: '2026-11-03T00:00:00Z', + paused: false, + model: 'claude-haiku-5-5', + tokens: 12_000_000, + budget_tokens: 100_000_000, + usd: 1.32, + budget_usd: 11, + users: 3, + users_at_limit: 1, + limits: { tokens: 1, runs: 0, data_runs: 0, messages: 0, pastes: 0, conversions: 0 }, + }; + + it('shows one row per day, newest first, with a dash for a queue wait the day has not', async () => { + stubFetch( + () => ok(USAGE), + () => ok({ status: 'ok', days: 14, rows: [DAY, QUIET], generated_at: USAGE.generated_at }) + ); + render(); + + const days = await screen.findByRole('group', { name: 'Per day' }); + const table = await within(days).findByRole('table'); + const [head, today, quiet] = within(table).getAllByRole('row'); + expect(head).toHaveTextContent( + 'day (UTC)new chatsplotscostfan-outlimit hitsp95 queue wait' + ); + expect(today).toHaveTextContent( + '2026-11-02 · so far43$0.02$0.12service_paused 2 · capacity 112.4 s' + ); + // A day before the queue wait field reads `—`; unpriced calls mark the cost as a floor. + expect(quiet).toHaveTextContent('2026-11-0100$0.00+$0.000—'); + expect(within(quiet).getByTitle('2 model calls without a price')).toHaveTextContent('$0.00+'); + }); + + it('says so when the per-day numbers are not configured or the query failed', async () => { + let answer = () => ok({ status: 'not_configured', rows: [] }); + stubFetch( + () => ok(USAGE), + () => answer() + ); + const { unmount } = render(); + expect(await screen.findByTestId('agent-daily-empty')).toHaveTextContent( + 'BigQuery telemetry is not configured yet' + ); + unmount(); + + answer = () => Promise.reject(new TypeError('Failed to fetch')); + render(); + expect(await screen.findByTestId('agent-daily-empty')).toHaveTextContent( + 'per-day numbers unavailable right now' + ); + }); + + it("shows today's spend live, with the counts' lifetime", async () => { + stubFetch( + () => ok(USAGE), + undefined, + () => ok(BUDGET) + ); + render(); + + const line = await screen.findByTestId('agent-today'); + expect(line).toHaveTextContent( + 'today: $1.32 of $11.00 (12.0 M of 100.0 M weighted tokens, 12%) · 3 users, 1 at their own limit (tokens 1)' + ); + const group = screen.getByRole('group', { name: 'Today, live' }); + expect(group).toHaveTextContent('counted since 14:05 UTC, when this instance started'); + expect(group).toHaveTextContent('a redeploy or a scale to zero starts the counts again'); + expect(screen.queryByTestId('agent-today-paused')).not.toBeInTheDocument(); + }); + + it('marks a paused day, and an instance older than today', async () => { + stubFetch( + () => ok(USAGE), + undefined, + () => + ok({ + ...BUDGET, + paused: true, + since: '2026-11-02T00:00:00Z', + started_at: '2026-10-31T09:12:00Z', + usd: null, + budget_usd: null, + }) + ); + render(); + + expect(await screen.findByTestId('agent-today-paused')).toHaveTextContent( + 'paused · the day budget is spent: every turn gets service_paused until' + ); + // No price for the model: the tokens alone. + expect(screen.getByTestId('agent-today')).toHaveTextContent( + 'today: 12.0 M of 100.0 M weighted tokens · 3 users' + ); + expect(screen.getByRole('group', { name: 'Today, live' })).toHaveTextContent( + 'counted since 00:00 UTC; this instance has run since 2026-10-31 09:12:00 UTC' + ); + }); + + it('says so when the agents service does not answer', async () => { + stubFetch(() => ok(USAGE)); + render(); + expect(await screen.findByTestId('agent-today-empty')).toHaveTextContent( + 'live numbers unavailable: the agents service did not answer' + ); }); }); diff --git a/app/src/sections/agent-chat/ChatThread.test.tsx b/app/src/sections/agent-chat/ChatThread.test.tsx index 55547281786..40e25f7cd1a 100644 --- a/app/src/sections/agent-chat/ChatThread.test.tsx +++ b/app/src/sections/agent-chat/ChatThread.test.tsx @@ -7,6 +7,7 @@ import { type RunState, } from 'src/hooks/useAgentSession'; import { ChatThread } from 'src/sections/agent-chat/ChatThread'; +import { localClock } from 'src/sections/agent-chat/messages'; import { pendingReply } from 'src/sections/agent-chat/steps'; import { render, screen } from 'src/test-utils'; @@ -61,6 +62,27 @@ describe('ChatThread', () => { expect(screen.getByText('no plot this time')).toBeInTheDocument(); }); + it("adds the user's own time to the service's pause, which names only UTC", () => { + const until = '2026-11-03T00:00:00.000Z'; + const text = + "The chat has reached today's overall limit and is paused until 00:00 UTC. Your own limit is not affected."; + renderThread({ + ...initialAgentState('matplotlib'), + phase: 'ready', + sessionId: 'S1', + items: [ + { id: 1, kind: 'user', text: 'log scale on y' }, + { id: 2, kind: 'refusal', code: 'service_paused', text, until }, + { id: 3, kind: 'refusal', code: 'budget', text: 'The usage limit for now is reached.' }, + ], + }); + const [paused, own] = screen.getAllByTestId('refusal'); + expect(paused).toHaveTextContent( + `refusal · service_paused · until ${localClock(until)}${text}` + ); + expect(own).toHaveTextContent(/^refusal · budgetThe usage limit/); + }); + it("shows the page's own turns: the summary without a library, create_plot with it", () => { renderThread({ ...initialAgentState('matplotlib'), diff --git a/app/src/sections/agent-chat/ChatThread.tsx b/app/src/sections/agent-chat/ChatThread.tsx index fd9ebe4344b..676685fcde8 100644 --- a/app/src/sections/agent-chat/ChatThread.tsx +++ b/app/src/sections/agent-chat/ChatThread.tsx @@ -20,7 +20,7 @@ import { } from 'src/hooks/useAgentSession'; import type { ArtifactName, Theme } from 'src/lib/agent'; import { Ellipsis } from 'src/sections/agent-chat/Ellipsis'; -import { ERROR_TEXT, PLOT_FAILED_TEXT } from 'src/sections/agent-chat/messages'; +import { ERROR_TEXT, localClock, PLOT_FAILED_TEXT } from 'src/sections/agent-chat/messages'; import { ReloadHint } from 'src/sections/agent-chat/ReloadHint'; import { type CardOutcome, ResultCard } from 'src/sections/agent-chat/ResultCard'; import { type PendingReply, pendingReply } from 'src/sections/agent-chat/steps'; @@ -99,7 +99,11 @@ function Item({ case 'refusal': return ( - refusal · {item.code} + + refusal · {item.code} + {/* The service's own text says 00:00 UTC: the page adds the user's own time. */} + {item.until && localClock(item.until) ? ` · until ${localClock(item.until)}` : ''} + {item.text} ); diff --git a/app/src/sections/agent-chat/messages.test.ts b/app/src/sections/agent-chat/messages.test.ts index fb8daba7214..65d683b59f9 100644 --- a/app/src/sections/agent-chat/messages.test.ts +++ b/app/src/sections/agent-chat/messages.test.ts @@ -1,6 +1,42 @@ import { describe, expect, it } from 'vitest'; -import { describeConvertFailure, describeDataFailure } from './messages'; +import { + describeConvertFailure, + describeDataFailure, + localClock, + nextUtcMidnight, + servicePausedText, +} from './messages'; + +describe('the service pause', () => { + it('ends at the next 00:00 UTC, also a second before it', () => { + expect(nextUtcMidnight(Date.parse('2026-11-02T14:05:00Z'))).toBe('2026-11-03T00:00:00.000Z'); + expect(nextUtcMidnight(Date.parse('2026-11-02T23:59:59Z'))).toBe('2026-11-03T00:00:00.000Z'); + expect(nextUtcMidnight(Date.parse('2026-12-31T00:00:00Z'))).toBe('2027-01-01T00:00:00.000Z'); + }); + + it("names the end in the user's own time, and says their own limit is untouched", () => { + const until = '2026-11-03T00:00:00Z'; + expect(localClock(until)).toMatch(/^\d{2}:\d{2}/); + expect(servicePausedText(until)).toBe( + `the chat has reached today's overall limit and is paused until ${localClock(until)} · your own limit is not affected` + ); + expect(localClock('not a time')).toBeNull(); + expect(servicePausedText('not a time')).toContain('paused until 00:00 UTC'); + }); + + it('is the line of a paused paste or file, never the daily limit of the user', () => { + expect(describeDataFailure('service_paused', 'r-9')).toBe( + `${servicePausedText()} · ref r-9` + ); + expect(describeConvertFailure('a.xlsx', 'service_paused', null, null, null)).toBe( + `"a.xlsx" ${servicePausedText()}` + ); + expect(describeDataFailure('daily_limit', null)).toBe( + "today's limit is reached; try again tomorrow" + ); + }); +}); describe('describeDataFailure', () => { it('leads with what a refused paste fails on, before the ref', () => { diff --git a/app/src/sections/agent-chat/messages.ts b/app/src/sections/agent-chat/messages.ts index 3e088e68ab7..b78488bd154 100644 --- a/app/src/sections/agent-chat/messages.ts +++ b/app/src/sections/agent-chat/messages.ts @@ -47,6 +47,33 @@ export const PLOT_FAILED_TEXT: Record = { data_unconfirmed: 'confirm your data first: check the preview and the bindings, then .confirm()', }; +/** + * An instant on the browser's own clock (`01:00`), or null for one that does not + * parse: the agents service counts its days in UTC, the user reads their own time. + */ +export function localClock(iso: string): string | null { + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return null; + return date.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }); +} + +/** The next 00:00 UTC, when the agents service's day budget starts again from zero. */ +export function nextUtcMidnight(now: number = Date.now()): string { + const date = new Date(now); + return new Date( + Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate() + 1) + ).toISOString(); +} + +/** + * The whole chat paused for the day (`service_paused`): the service's day budget is + * spent, not the user's own. `until` is the reset in the user's time; the server's + * own refusal says 00:00 UTC, since it does not know the user's time zone. + */ +export function servicePausedText(until: string = nextUtcMidnight()): string { + return `the chat has reached today's overall limit and is paused until ${localClock(until) ?? '00:00 UTC'} · your own limit is not affected`; +} + /** Failures of the dataset, bindings, confirmation and undo calls. */ export const DATA_FAILURE: Record = { too_long: 'more than 1 MB; paste fewer rows', @@ -71,6 +98,14 @@ export const DATA_FAILURE: Record = { network: 'the server did not answer; check your connection', }; +/** + * A data call's failure text: `DATA_FAILURE`, except the service's pause + * (`429 service_paused`), which names its end in the user's own time. + */ +function dataFailureText(code: string): string | undefined { + return code === 'service_paused' ? servicePausedText() : DATA_FAILURE[code]; +} + /** * What dropped a confirmation the user had given: the thread's warning notice * and the data card's step line next to `.confirm()` say it in these words. @@ -222,7 +257,7 @@ export function describeConvertFailure( kind: string | null = null, damage: string | null = null ): string { - let text = CONVERT_FAILURE[code] ?? DATA_FAILURE[code] ?? `could not be read (${code})`; + let text = CONVERT_FAILURE[code] ?? dataFailureText(code) ?? `could not be read (${code})`; // A damaged file names the format its first bytes claimed, and the part the server found broken. if (code === 'file_corrupt' && damage && DAMAGE_TEXT[damage]) { const as = kind && KIND_NAME[kind] ? ` as ${KIND_NAME[kind]}` : ''; @@ -316,6 +351,6 @@ export function describeDataFailure( ref: string | null, cause: string | null = null ): string { - const text = cause || (DATA_FAILURE[code] ?? `request failed (${code})`); + const text = cause || (dataFailureText(code) ?? `request failed (${code})`); return ref ? `${text} · ref ${ref}` : text; } diff --git a/app/src/sections/debug/AgentDailyUsage.tsx b/app/src/sections/debug/AgentDailyUsage.tsx new file mode 100644 index 00000000000..1c3e6a60172 --- /dev/null +++ b/app/src/sections/debug/AgentDailyUsage.tsx @@ -0,0 +1,433 @@ +/** + * The agent usage section's two views of the day, below the 7- and 30-day tiles: + * + * - `AgentTodayLine`: today so far, live from the agents service's usage book + * (`GET /debug/agent/budget`): the service's spend against its day budget, in + * USD and weighted tokens, and how many users are at their own limits. The + * counts live in the instance's memory, so the line says since when they + * count and that a restart starts them again. + * - `AgentDailyTable`: the last 14 UTC days from the BigQuery telemetry + * (`GET /debug/agent-usage/daily`), newest first: new chats, plots shipped, + * the cost (live, and the admin fan-out's apart), the limit refusals by code + * and the run queue's 95th-percentile wait, `—` on a day before the field. + * + * Each fetches on its own and fails to one muted line, like the section above. + */ + +import { useEffect, useState } from 'react'; + +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; + +import { DEBUG_API_URL } from 'src/constants'; +import { fetchWithAuth } from 'src/lib/api'; +import { localClock } from 'src/sections/agent-chat/messages'; +import { formatUtc, timeAgo } from 'src/sections/debug/format'; +import { listMutedSx, listTextSx } from 'src/sections/debug/styles'; +import { colors } from 'src/theme'; + +// ============================================================================ +// Answers +// ============================================================================ + +/** The day caps the agents service counts users at (`ledger.LIMIT_KINDS`). */ +const LIMIT_KINDS = [ + 'tokens', + 'runs', + 'data_runs', + 'messages', + 'pastes', + 'conversions', + 'locked_out', +] as const; + +/** The answer of `GET /debug/agent/budget`: counts and times only. */ +export interface AgentBudget { + since: string; + started_at: string; + resets_at: string; + paused: boolean; + tokens: number; + budget_tokens: number; + usd: number | null; + budget_usd: number | null; + users: number; + users_at_limit: number; + limits: Record; +} + +/** The refusal codes the per-day table counts, in its order (`api/agent_usage.py` `LIMIT_CODES`). */ +const LIMIT_CODES = [ + 'service_paused', + 'budget', + 'daily_limit', + 'rate_limited', + 'run_limit', + 'capacity', +] as const; + +export interface UsageDay { + date: string; + sessions: number; + plots_shipped: number; + cost_usd: number; + fanout_cost_usd: number; + unpriced_calls: number; + limit_hits: Record; + queue_wait_p95_ms: number | null; + queue_waits: number; +} + +/** The answer of `GET /debug/agent-usage/daily`. */ +export interface AgentUsageDaily { + status: 'ok' | 'not_configured' | 'unavailable'; + rows: UsageDay[]; + generated_at: string | null; +} + +const count = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0; + +const amount = (value: unknown): number | null => + typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null; + +const instant = (value: unknown): string | null => + typeof value === 'string' && !Number.isNaN(Date.parse(value)) ? value : null; + +const counts = (value: unknown, keys: readonly string[]): Record => { + const record = value && typeof value === 'object' ? (value as Record) : {}; + return Object.fromEntries(keys.map(key => [key, count(record[key])])); +}; + +/** The live answer as the line reads it; `null` for one without its times. */ +function readBudget(body: unknown): AgentBudget | null { + if (!body || typeof body !== 'object') return null; + const record = body as Record; + const since = instant(record.since); + const started = instant(record.started_at); + const resets = instant(record.resets_at); + if (!since || !started || !resets) return null; + return { + since, + started_at: started, + resets_at: resets, + paused: record.paused === true, + tokens: count(record.tokens), + budget_tokens: count(record.budget_tokens), + usd: amount(record.usd), + budget_usd: amount(record.budget_usd), + users: count(record.users), + users_at_limit: count(record.users_at_limit), + limits: counts(record.limits, LIMIT_KINDS), + }; +} + +const STATUSES = new Set(['ok', 'not_configured', 'unavailable']); + +/** The per-day answer as the table reads it; `null` for one without a known status. */ +function readDaily(body: unknown): AgentUsageDaily | null { + if (!body || typeof body !== 'object') return null; + const record = body as Record; + if (typeof record.status !== 'string' || !STATUSES.has(record.status)) return null; + const rows = Array.isArray(record.rows) ? record.rows : []; + return { + status: record.status as AgentUsageDaily['status'], + rows: rows + .filter( + (row): row is Record => + !!row && typeof row === 'object' && typeof (row as UsageDay).date === 'string' + ) + .map(row => { + const wait = row.queue_wait_p95_ms; + return { + date: row.date as string, + sessions: count(row.sessions), + plots_shipped: count(row.plots_shipped), + cost_usd: amount(row.cost_usd) ?? 0, + fanout_cost_usd: amount(row.fanout_cost_usd) ?? 0, + unpriced_calls: count(row.unpriced_calls), + limit_hits: counts(row.limit_hits, LIMIT_CODES), + queue_wait_p95_ms: typeof wait === 'number' && wait >= 0 ? wait : null, + queue_waits: count(row.queue_waits), + }; + }), + generated_at: typeof record.generated_at === 'string' ? record.generated_at : null, + }; +} + +// ============================================================================ +// Formatting +// ============================================================================ + +/** Dollars to the cent; a spend below a cent is `<$0.01`, never `$0.00`. */ +export function formatUsd(value: number): string { + if (value > 0 && value < 0.005) return '<$0.01'; + return `$${value.toFixed(2)}`; +} + +const millions = (tokens: number): string => `${(tokens / 1e6).toFixed(1)} M`; + +/** The time of day of a UTC instant, `14:05 UTC`. */ +const utcClock = (iso: string): string => `${new Date(iso).toISOString().slice(11, 16)} UTC`; + +const plural = (value: number, noun: string): string => + `${value.toLocaleString('en-US')} ${noun}${value === 1 ? '' : 's'}`; + +/** The day caps users are at, those with any: `tokens 1, runs 2`. */ +function limitBreakdown(limits: Record): string { + return LIMIT_KINDS.filter(kind => limits[kind] > 0) + .map(kind => `${kind.replace('_', ' ')} ${limits[kind]}`) + .join(', '); +} + +// ============================================================================ +// Today, live +// ============================================================================ + +interface Answer { + key: string; + value: T | null; +} + +function useAnswer( + url: string, + token: string, + reloadCounter: number, + read: (body: unknown) => T | null +): T | null | undefined { + const key = `${url}\n${token}\n${reloadCounter}`; + const [answer, setAnswer] = useState | null>(null); + useEffect(() => { + let cancelled = false; + fetchWithAuth(url, token) + .then(async response => (response.ok ? read(await response.json()) : null)) + .catch(() => null) + .then(value => { + if (!cancelled) setAnswer({ key, value }); + }); + return () => { + cancelled = true; + }; + }, [url, token, key, read]); + // `undefined` while this key's answer is out. + return answer && answer.key === key ? answer.value : undefined; +} + +export interface DayViewProps { + /** The admin token of the page (`X-Admin-Token`), or `''` behind Cloudflare Access. */ + token: string; + /** The page's reload count: a change fetches again. */ + reloadCounter: number; +} + +const subheadSx = { + ...listTextSx, + fontWeight: 600, + color: 'var(--ink-soft)', + mt: 3, + mb: 0.75, +} as const; + +export function AgentTodayLine({ token, reloadCounter }: DayViewProps) { + const budget = useAnswer( + `${DEBUG_API_URL}/debug/agent/budget`, + token, + reloadCounter, + readBudget + ); + + return ( + + today · live from the agents service + {budget === undefined ? ( + loading today… + ) : budget === null ? ( + + live numbers unavailable: the agents service did not answer + + ) : ( + + )} + + ); +} + +function TodayText({ budget }: { budget: AgentBudget }) { + const share = budget.budget_tokens > 0 ? Math.round((budget.tokens / budget.budget_tokens) * 100) : null; + const spend = + budget.usd !== null && budget.budget_usd !== null + ? `${formatUsd(budget.usd)} of ${formatUsd(budget.budget_usd)}` + : `${millions(budget.tokens)} of ${millions(budget.budget_tokens)} weighted tokens`; + const tokens = + budget.usd !== null && budget.budget_usd !== null + ? ` (${millions(budget.tokens)} of ${millions(budget.budget_tokens)} weighted tokens${share !== null ? `, ${share}%` : ''})` + : ''; + const breakdown = limitBreakdown(budget.limits); + const startedToday = budget.since === budget.started_at; + const resets = localClock(budget.resets_at) ?? utcClock(budget.resets_at); + return ( + <> + {budget.paused && ( + + paused · the day budget is spent: every turn gets service_paused until {resets} + + )} + + today: {spend} + {tokens} · {plural(budget.users, 'user')}, {budget.users_at_limit} at their own limit + {breakdown ? ` (${breakdown})` : ''} + + + {startedToday + ? `counted since ${utcClock(budget.since)}, when this instance started` + : `counted since 00:00 UTC; this instance has run since ${formatUtc(budget.started_at)}`} + {' · '}a redeploy or a scale to zero starts the counts again · the new day starts at {resets} + + + ); +} + +// ============================================================================ +// The last 14 days +// ============================================================================ + +const DAILY_EMPTY: Record<'not_configured' | 'unavailable', string> = { + not_configured: 'BigQuery telemetry is not configured yet', + unavailable: 'per-day numbers unavailable right now', +}; + +const cellSx = { + ...listTextSx, + px: 1, + py: 0.5, + borderBottom: '1px solid var(--rule)', + textAlign: 'right', + whiteSpace: 'nowrap', + fontVariantNumeric: 'tabular-nums', +} as const; + +const headSx = { ...cellSx, fontWeight: 600, color: 'var(--ink-soft)' } as const; + +const COLUMNS = [ + 'day (UTC)', + 'new chats', + 'plots', + 'cost', + 'fan-out', + 'limit hits', + 'p95 queue wait', +] as const; + +export function AgentDailyTable({ token, reloadCounter }: DayViewProps) { + const daily = useAnswer( + `${DEBUG_API_URL}/debug/agent-usage/daily`, + token, + reloadCounter, + readDaily + ); + const empty = + daily === null || daily?.status === 'unavailable' + ? 'unavailable' + : daily?.status === 'not_configured' + ? 'not_configured' + : null; + + return ( + + per day · the last 14 UTC days, newest first + {daily === undefined ? ( + loading the days… + ) : empty || !daily ? ( + + {DAILY_EMPTY[empty ?? 'unavailable']} + + ) : ( + <> + + + + + {COLUMNS.map((column, index) => ( + + {column} + + ))} + + + + {daily.rows.map((day, index) => ( + + ))} + + + + + new chats: live sessions by the day of their first line · cost: list price, the fan-out's + apart · limit hits: refusals and errors by code · — : no run on that day carried the + queue wait yet + {daily.generated_at ? ` · generated ${timeAgo(daily.generated_at)}` : ''} + + + )} + + ); +} + +function DayRow({ day, today }: { day: UsageDay; today: boolean }) { + const hits = LIMIT_CODES.filter(code => day.limit_hits[code] > 0); + const unpriced = + day.unpriced_calls > 0 ? `${plural(day.unpriced_calls, 'model call')} without a price` : ''; + return ( + + + {day.date} + {today ? ' · so far' : ''} + + + {day.sessions.toLocaleString('en-US')} + + + {day.plots_shipped.toLocaleString('en-US')} + + + {formatUsd(day.cost_usd)} + {unpriced ? '+' : ''} + + + {formatUsd(day.fanout_cost_usd)} + + + {hits.length === 0 ? ( + + 0 + + ) : ( + hits.map(code => `${code} ${day.limit_hits[code]}`).join(' · ') + )} + + 0 ? `of ${plural(day.queue_waits, 'run')}` : undefined} + > + {day.queue_wait_p95_ms === null ? '—' : `${(day.queue_wait_p95_ms / 1000).toFixed(1)} s`} + + + ); +} diff --git a/app/src/sections/debug/AgentUsageSection.tsx b/app/src/sections/debug/AgentUsageSection.tsx index 6ba79e867c5..a4fe97fe1cd 100644 --- a/app/src/sections/debug/AgentUsageSection.tsx +++ b/app/src/sections/debug/AgentUsageSection.tsx @@ -9,6 +9,8 @@ * 12"); a rate without a base reads `—`. * - Three short ranked lists: the specs, the data formats and the refusal * codes seen most often. + * - Below, apart from the window toggle (`AgentDailyUsage.tsx`): today so far, + * live from the agents service, and a table of the last 14 UTC days. * - BigQuery not set up yet (`not_configured`), a failed query (`unavailable`) * and a window without sessions are one muted line each. A failed request or * an answer the page cannot read is the `unavailable` line too: the section @@ -24,6 +26,7 @@ import Typography from '@mui/material/Typography'; import { SectionHeader } from 'src/components/SectionHeader'; import { DEBUG_API_URL } from 'src/constants'; import { fetchWithAuth } from 'src/lib/api'; +import { AgentDailyTable, AgentTodayLine } from 'src/sections/debug/AgentDailyUsage'; import { timeAgo } from 'src/sections/debug/format'; import { debugControlSx, listMutedSx, listTextSx } from 'src/sections/debug/styles'; import { colors, fontSize, semanticColors, typography } from 'src/theme'; @@ -242,6 +245,10 @@ export function AgentUsageSection({ token, reloadCounter }: AgentUsageSectionPro )} + + {/* Independent of the window toggle: today live, and one row per day. */} + + ); } diff --git a/tests/unit/agents/runtime/test_day_budget.py b/tests/unit/agents/runtime/test_day_budget.py new file mode 100644 index 00000000000..6b509330104 --- /dev/null +++ b/tests/unit/agents/runtime/test_day_budget.py @@ -0,0 +1,269 @@ +"""The service's day budget: the `service_paused` refusal, the status's pause note and the admin's live view. + +Owner request of 2026-11-02: once `AGENT_GLOBAL_DAILY_TOKEN_BUDGET` is spent, every user got the same text as for +their own spent limit, and nobody could tell that the whole service paused. Now the global budget answers +`service_paused` (a turn's refusal, a paste's `429`), `GET /v1/status` says `paused` before the user types, and +`GET /v1/budget` gives the `/debug` page the day's spend and how many users are at their own limits, counts only. +""" + +import logging +from collections.abc import AsyncIterator, Iterator +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any, get_args + +import httpx +import pytest +from google.adk.models.llm_request import LlmRequest + +from agents.anyplot import telemetry +from agents.anyplot.plugins.budget import ROOT_AGENT, BudgetPlugin +from agents.anyplot.plugins.ledger import ( + CURRENT_LEDGER, + LIMIT_KINDS, + LOCATION_SURCHARGES, + SERVICE_PAUSED, + UNIT_PRICES, + RequestLedger, + UsageBook, + day_refusal, + next_reset, + weighted_usd, +) +from agents.anyplot.plugins.scope_guard import LOCKED_OUT +from agents.anyplot.policy import refusal +from agents.anyplot.services import Services +from agents.anyplot.settings import AgentSettings, get_settings +from agents.evals import pricing +from agents.main import Runtime, app, get_runtime +from api.agent_usage import LIMIT_CODES + +from .test_service_flow import HEADERS, USER, open_session, parse_sse, permissive_queue +from .test_telemetry import lines_of +from .test_turn_limits import OTHER, chatty + + +@pytest.fixture +def runtime() -> Iterator[Runtime]: + fresh = Runtime(queue=permissive_queue()) + app.dependency_overrides[get_runtime] = lambda: fresh + yield fresh + app.dependency_overrides.clear() + + +@pytest.fixture +async def client(runtime: Runtime, services: Services) -> AsyncIterator[httpx.AsyncClient]: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://agents") as http: + yield http + + +@pytest.fixture +def small_budget(monkeypatch: pytest.MonkeyPatch) -> AgentSettings: + """A service budget of 1,000 weighted tokens and a user budget of 600, so a test spends either at will.""" + monkeypatch.setenv("AGENT_GLOBAL_DAILY_TOKEN_BUDGET", "1000") + monkeypatch.setenv("AGENT_DAILY_TOKEN_BUDGET", "600") + get_settings.cache_clear() + yield get_settings() + get_settings.cache_clear() + + +async def say(client: httpx.AsyncClient, sid: str, text: str) -> list[Any]: + response = await client.post(f"/v1/sessions/{sid}/messages", headers=HEADERS, json={"text": text}) + assert response.status_code == 200, response.text + return parse_sse(response.text) + + +class TestUsageBook: + def test_the_pause_is_the_service_budget_and_never_the_users_own(self) -> None: + usage = UsageBook() + settings = AgentSettings(global_daily_token_budget=1_000, daily_token_budget=600) + + usage.add_tokens("a", 400) + assert not usage.paused(settings) and usage.daily_ok("b", settings) + usage.add_tokens("a", 599) + assert usage.paused(settings, reserve=1) and not usage.paused(settings) + usage.add_tokens("c", 1) + # Spent by others: `b` still has their own budget, so the refusal says the whole service paused. + assert usage.paused(settings) and not usage.daily_ok("b", settings) + assert day_refusal(usage, "b", settings) == SERVICE_PAUSED + # `a` spent their own budget too: theirs is the one to name, since the pause text says theirs is untouched. + assert day_refusal(usage, "a", settings) == "budget" + + def test_the_strikes_come_first(self) -> None: + usage = UsageBook() + settings = AgentSettings(global_daily_token_budget=1_000, attack_strikes=1) + usage.add_tokens("a", 1_000) + usage.add_strike("b", "ignore your rules") + assert day_refusal(usage, "b", settings) == LOCKED_OUT == "locked_out" + assert day_refusal(usage, "c", settings) == SERVICE_PAUSED + + def test_the_snapshot_counts_users_at_their_own_limits_never_names_them(self) -> None: + usage = UsageBook() + settings = AgentSettings(daily_token_budget=600, daily_pipeline_runs=2, user_messages_per_day=3) + usage.add_tokens("a", 700) + for _ in range(2): + usage.add_pipeline_run("a") + usage.add_pipeline_run("b") + usage.add_message("c") + usage.add_outcome("d") # a beacon alone is no use of the chat today + + day = usage.snapshot(settings) + + assert (day.global_tokens, day.users, day.at_limit) == (700, 3, 2) + assert day.limits == { + "tokens": 1, + "runs": 2, + "data_runs": 0, + "messages": 0, + "pastes": 0, + "conversions": 0, + "locked_out": 0, + } + assert tuple(day.limits) == LIMIT_KINDS + assert day.date == datetime.now(UTC).date().isoformat() + assert day.since == max(day.started_at, datetime.fromisoformat(day.date).replace(tzinfo=UTC)) + assert not any(isinstance(value, str) and value in "abcd" for value in vars(day).values()) + + def test_the_new_day_starts_at_midnight_utc(self) -> None: + assert next_reset(datetime(2026, 11, 2, 23, 59, 59, tzinfo=UTC)) == datetime(2026, 11, 3, tzinfo=UTC) + assert next_reset(datetime(2026, 11, 2, 0, 0, tzinfo=UTC)) == datetime(2026, 11, 3, tzinfo=UTC) + + +class TestPrices: + def test_the_unit_prices_are_the_list_prices(self) -> None: + """A price change in agents/evals/pricing.py must reach the live view's dollars too.""" + for model, price in UNIT_PRICES.items(): + assert price == pricing.price_of(model).input + assert LOCATION_SURCHARGES == pricing.LOCATION_SURCHARGE + + def test_a_weighted_token_costs_the_main_models_input_price(self) -> None: + # 100 M weighted tokens of Claude Haiku 5.5 in `eu`: $0.10 per M plus 10 %. + assert weighted_usd(100_000_000, "claude-haiku-5-5", "eu") == pytest.approx(11.0) + assert weighted_usd(100_000_000, "claude-haiku-5-5", "global") == pytest.approx(10.0) + assert weighted_usd(1_000_000, "some-new-model", "eu") is None + + +class TestPauseRefusal: + async def test_a_turn_says_the_service_paused_when_others_spent_the_day( + self, client: httpx.AsyncClient, services: Services, swap_models, small_budget: AgentSettings + ) -> None: + fake = swap_models("gemini", chatty(1)) + sid = await open_session(client) + services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) + + refused = await say(client, sid, "Which columns do I have?") + + assert [name for name, _ in refused] == ["ready", "refusal", "done"] + assert refused[1][1] == {"code": SERVICE_PAUSED, "text": refusal(SERVICE_PAUSED, "en")} + assert "00:00 UTC" in refused[1][1]["text"] and fake.requests == [] # no model call + + async def test_the_pause_reaches_the_telemetry_under_its_own_code( + self, + client: httpx.AsyncClient, + services: Services, + swap_models, + small_budget: AgentSettings, + caplog: pytest.LogCaptureFixture, + ) -> None: + """A closed set: an unknown code would drop the line, so `service_paused` is in both sets the routes write.""" + caplog.set_level(logging.INFO) + swap_models("gemini", chatty(1)) + sid = await open_session(client, with_data=False) + services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) + + await say(client, sid, "Which columns do I have?") + await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": "x,y\n1,2\n"}) + + assert [line["refusal"] for line in lines_of(caplog, "chat_turn")] == [SERVICE_PAUSED] + assert [line["outcome"] for line in lines_of(caplog, "data_input")] == [SERVICE_PAUSED] + + def test_the_debug_table_counts_codes_the_telemetry_writes(self) -> None: + """`api/agent_usage.py` cannot import the agents package (the API image does not ship it): pinned here.""" + written = set(get_args(telemetry.RefusalCode)) | set(get_args(telemetry.ErrorCode)) + written |= set(get_args(telemetry.DataOutcome)) + assert set(LIMIT_CODES) <= written + + async def test_a_root_call_past_the_service_budget_mid_turn_says_so( + self, services: Services, small_budget: AgentSettings + ) -> None: + """The Budget plugin's halt before a root call names the pause too, not the user's own `budget`.""" + ledger = RequestLedger(user_id=USER, lang="de") + context = SimpleNamespace( + invocation_id="inv-1", agent_name=ROOT_AGENT, session=SimpleNamespace(user_id=USER) + ) + services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) + token = CURRENT_LEDGER.set(ledger) + try: + halted = await BudgetPlugin().before_model_callback(callback_context=context, llm_request=LlmRequest()) + finally: + CURRENT_LEDGER.reset(token) + + assert halted is not None and ledger.refusal == (SERVICE_PAUSED, refusal(SERVICE_PAUSED, "de")) + + async def test_a_user_past_their_own_budget_keeps_the_budget_refusal( + self, client: httpx.AsyncClient, services: Services, swap_models, small_budget: AgentSettings + ) -> None: + swap_models("gemini", chatty(1)) + sid = await open_session(client) + services.usage.add_tokens(USER, small_budget.daily_token_budget) + + refused = await say(client, sid, "Which columns do I have?") + + assert refused[1][1] == {"code": "budget", "text": refusal("budget", "en")} + + async def test_a_paste_is_429_service_paused_apart_from_the_users_daily_limit( + self, client: httpx.AsyncClient, services: Services, small_budget: AgentSettings + ) -> None: + sid = await open_session(client, with_data=False) + services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) + + paused = await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": "x,y\n1,2\n"}) + assert (paused.status_code, paused.json()) == (429, {"detail": SERVICE_PAUSED}) + + services.usage.add_tokens(USER, small_budget.daily_token_budget) + own = await client.post(f"/v1/sessions/{sid}/dataset", headers=HEADERS, json={"text": "x,y\n1,2\n"}) + assert (own.status_code, own.json()) == (429, {"detail": "daily_limit"}) + + +class TestStatusAndBudget: + async def test_the_status_tells_the_page_before_the_user_types( + self, client: httpx.AsyncClient, services: Services, small_budget: AgentSettings + ) -> None: + before = (await client.get("/v1/status", headers=HEADERS)).json() + services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) + after = (await client.get("/v1/status", headers=HEADERS)).json() + + assert before["paused"] is False and after["paused"] is True + resets = datetime.fromisoformat(after["resets_at"]) + assert after["resets_at"].endswith("T00:00:00Z") and resets == next_reset() + + async def test_the_budget_view_is_counts_and_times_only( + self, client: httpx.AsyncClient, services: Services, small_budget: AgentSettings + ) -> None: + services.usage.add_tokens(USER, 600) + services.usage.add_tokens(OTHER, 150) + services.usage.add_message(OTHER) + + body = (await client.get("/v1/budget", headers=HEADERS)).json() + + assert set(body) == { + "date", + "since", + "started_at", + "resets_at", + "paused", + "model", + "tokens", + "budget_tokens", + "usd", + "budget_usd", + "users", + "users_at_limit", + "limits", + } + assert (body["tokens"], body["budget_tokens"], body["paused"]) == (750, 1_000, False) + assert body["usd"] == pytest.approx(750 * 0.11 / 1e6, abs=1e-4) + assert body["budget_usd"] == pytest.approx(1_000 * 0.11 / 1e6, abs=1e-4) + assert (body["users"], body["users_at_limit"], body["limits"]["tokens"]) == (2, 1, 1) + assert body["model"] == "claude-haiku-5-5" and body["resets_at"].endswith("T00:00:00Z") + assert USER not in str(body) and OTHER not in str(body) diff --git a/tests/unit/agents/runtime/test_policy.py b/tests/unit/agents/runtime/test_policy.py index 82460fe2a67..b2aaed4665a 100644 --- a/tests/unit/agents/runtime/test_policy.py +++ b/tests/unit/agents/runtime/test_policy.py @@ -773,6 +773,7 @@ def test_the_language_has_its_own_fence(self) -> None: "out_of_scope", "needs_data", "budget", + "service_paused", "rate_limited", "daily_limit", "session_closed", @@ -831,6 +832,18 @@ def test_the_data_refusal_says_nothing_is_fetched_and_asks_for_pasted_data(self) assert "cannot fetch data from the internet" in english and "paste it into the data panel" in english assert "keine Daten aus dem Internet" in german and "Datenbereich" in german + def test_the_pause_names_its_end_in_utc_and_spares_the_users_own_limit(self) -> None: + """The service's day budget, not the user's own (owner request of 2026-11-02): every language names the reset + in UTC, the one time the server knows, and says the user's own limit is untouched.""" + texts = policy.refusals()["service_paused"] + + assert all("00:00" in text and "UTC" in text for text in texts.values()) + assert texts["en"] == ( + "The chat has reached today's overall limit and is paused until 00:00 UTC. Your own limit is not affected." + ) + assert "Dein eigenes Limit" in texts["de"] + assert texts["en"] != policy.refusal("budget", "en") + def test_a_missing_root_refusal_breaks_the_build(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: (tmp_path / "refusals.yaml").write_text('out_of_scope:\n en: "x"\n', encoding="utf-8") monkeypatch.setattr(policy, "PROMPTS_DIR", tmp_path) diff --git a/tests/unit/api/test_agent_router.py b/tests/unit/api/test_agent_router.py index cf378468d99..f4b3c3e15cd 100644 --- a/tests/unit/api/test_agent_router.py +++ b/tests/unit/api/test_agent_router.py @@ -43,7 +43,8 @@ get_case_bucket, load_catalogue_snapshot, ) -from api.routers.debug import AdminIdentity, require_admin_identity +from api.routers.agent import router as agent_router +from api.routers.debug import AdminIdentity, require_admin, require_admin_identity from core.agent_cases import case_object, row_from_manifest from core.config import settings from core.database import AGENT_CASE_EXPECTED, FEEDBACK_REACTIONS, FEEDBACK_STATUSES, AgentFeedback, get_db @@ -321,6 +322,7 @@ def _sse_events(body: str) -> list[tuple[str, dict]]: # Every route with a request that passes validation, for the "all dark" sweep. ALL_ROUTES = [ ("GET", "/debug/agent/status", None), + ("GET", "/debug/agent/budget", None), ("GET", "/debug/agent/eligibility?spec=scatter-basic&library=matplotlib", None), ("POST", "/debug/agent/sessions", {"spec_id": "scatter-basic", "library": "matplotlib", "locale": "en"}), ("POST", "/debug/agent/sessions/s1/library", {"spec_id": "scatter-basic", "library": "seaborn"}), @@ -2079,6 +2081,48 @@ def test_without_a_configured_database_nothing_is_fetched_or_stored(self, client rows.create_case.assert_not_awaited() +class TestBudget: + """`GET /debug/agent/budget`: the `/debug` page's live line, counts and times only.""" + + ANSWER = { + "date": "2026-11-02", + "since": "2026-11-02T14:05:12Z", + "started_at": "2026-11-02T14:05:12Z", + "resets_at": "2026-11-03T00:00:00Z", + "paused": False, + "model": "claude-haiku-5-5", + "tokens": 12_000_000, + "budget_tokens": 100_000_000, + "usd": 1.32, + "budget_usd": 11.0, + "users": 3, + "users_at_limit": 1, + "limits": {"tokens": 1, "runs": 0}, + } + + def test_forwards_the_named_fields_only(self, client, upstream) -> None: + upstream.handler = lambda request: httpx.Response(200, json={**self.ANSWER, "user_ids": ["adm_0123"]}) + response = client.get("/debug/agent/budget") + + assert response.status_code == 200 and response.json() == self.ANSWER + assert str(upstream.requests[0].url) == f"{SERVICE_URL}/v1/budget" + assert response.headers["Cache-Control"] == "private, no-store" + + def test_the_route_carries_its_own_admin_gate(self, agent_on, upstream) -> None: + """Gated on the route as well as by the router, so it stays an admin view if the chat routes open.""" + route = next(route for route in agent_router.routes if getattr(route, "path", "") == "/debug/agent/budget") + assert any(dependency.call is require_admin for dependency in route.dependant.dependencies) + with patch.object(settings, "admin_token", "supersecret"): + response = TestClient(app).get("/debug/agent/budget", headers={"X-Admin-Token": "wrong"}) + assert response.status_code == 401 and upstream.requests == [] + + def test_an_upstream_failure_keeps_the_agent_error_shape(self, client, upstream) -> None: + upstream.handler = lambda request: httpx.Response(500, json={"detail": "boom"}) + response = client.get("/debug/agent/budget") + assert response.status_code == 500 + assert response.json() == {"detail": "upstream", "ref": response.headers["X-Request-Id"]} + + class TestUpstreamMapping: @pytest.mark.parametrize( ("status", "body", "expected_status", "expected_code"), @@ -2096,6 +2140,8 @@ class TestUpstreamMapping: # The paste route's limits and guards since 2026-10-14; a 403 with a documented code is no IAM error. (429, {"detail": "daily_limit"}, 429, "daily_limit"), (429, {"detail": "locked_out"}, 429, "locked_out"), + # The service's day budget, not the user's own (2026-11-02): never the generic `rate_limited` of a 429. + (429, {"detail": "service_paused"}, 429, "service_paused"), (403, {"detail": "session_closed"}, 403, "session_closed"), (422, {"detail": "Traceback: KeyError in /app/agents/x.py"}, 422, "invalid"), (409, {"detail": ["run_active"]}, 409, "conflict"), diff --git a/tests/unit/api/test_agent_usage.py b/tests/unit/api/test_agent_usage.py index 29d6d87febe..408297988dd 100644 --- a/tests/unit/api/test_agent_usage.py +++ b/tests/unit/api/test_agent_usage.py @@ -1,11 +1,13 @@ """ -Tests for GET /debug/agent-usage and api/agent_usage.py — the agent usage highlights read from BigQuery. +Tests for GET /debug/agent-usage, GET /debug/agent-usage/daily and api/agent_usage.py — the agent usage +highlights and the per-day table read from BigQuery. No test reaches BigQuery: `get_usage_query` is overridden with a fake that records the SQL and answers rows. """ from collections.abc import Iterator from datetime import UTC, datetime, timedelta +from pathlib import Path from typing import Any from unittest.mock import patch @@ -16,8 +18,12 @@ from api.agent_usage import ( CACHE_TTL_S, DATASET_PATTERN, + LIMIT_CODES, clear_usage_cache, + daily_sql, + daily_start, decode_rows, + read_daily, read_usage, usage_query, usage_sql, @@ -286,3 +292,131 @@ def test_a_dataset_that_could_break_the_sql_is_not_configured(self, dataset) -> def test_an_empty_answer_is_zero(self) -> None: usage = read_usage(FakeQuery(rows=[]), DATASET, 7) assert usage.status == "ok" and usage.sessions == 0 and usage.kept_rate is None + + +NOW = datetime(2026, 11, 2, 14, 5, 30, tzinfo=UTC) +DAY_ROWS: list[dict[str, Any]] = [ + { + "day": "2026-11-02", + "sessions": 4, + "plots_shipped": 3, + "hits_service_paused": 2, + "hits_capacity": 1, + "queue_wait_p95_ms": 12_400, + "queue_waits": 3, + "cost_usd": 0.022837133, + "fanout_cost_usd": 0.1210845185, + "unpriced_calls": 0, + }, + # A day with lines but no shipped plot and a run before the queue-wait field: no percentile. + {"day": "2026-10-28", "sessions": 11, "cost_usd": 0.17469341, "queue_wait_p95_ms": None, "queue_waits": 0}, + {"day": None, "sessions": 99}, # never a row without its day +] + + +class TestDaily: + def test_answers_every_day_of_the_window_newest_first(self, client, fake) -> None: + fake._rows = DAY_ROWS + body = client.get("/debug/agent-usage/daily").json() + + assert body["status"] == "ok" and body["days"] == 14 and len(body["rows"]) == 14 + dates = [row["date"] for row in body["rows"]] + assert dates == sorted(dates, reverse=True) and dates[0] == datetime.now(UTC).date().isoformat() + sql, since = fake.calls[0] + assert f"`{DATASET}_logs._AllLogs`" in sql and f"`{DATASET}.model_prices`" in sql + assert since == datetime.combine(datetime.now(UTC).date() - timedelta(days=13), datetime.min.time(), UTC) + + def test_reads_a_day_and_fills_the_days_without_lines(self) -> None: + daily = read_daily(FakeQuery(rows=DAY_ROWS), DATASET, now=NOW) + today, quiet, busy = daily.rows[0], daily.rows[1], daily.rows[5] + + assert (today.date, today.sessions, today.plots_shipped) == ("2026-11-02", 4, 3) + assert (today.cost_usd, today.fanout_cost_usd, today.unpriced_calls) == (0.0228, 0.1211, 0) + assert today.limit_hits == { + "service_paused": 2, + "budget": 0, + "daily_limit": 0, + "rate_limited": 0, + "run_limit": 0, + "capacity": 1, + } + assert (today.queue_wait_p95_ms, today.queue_waits) == (12_400, 3) + # A day without a line is zeros, and its percentile is unknown, not 0. + assert quiet.date == "2026-11-01" and quiet.sessions == 0 and quiet.cost_usd == 0.0 + assert quiet.queue_wait_p95_ms is None and set(quiet.limit_hits.values()) == {0} + assert (busy.date, busy.sessions, busy.cost_usd, busy.queue_wait_p95_ms) == ("2026-10-28", 11, 0.1747, None) + assert daily.rows[-1].date == "2026-10-20" and 99 not in [row.sessions for row in daily.rows] + + def test_the_window_starts_at_midnight_and_stays_for_the_day(self) -> None: + morning = daily_start(datetime(2026, 11, 2, 0, 0, 1, tzinfo=UTC)) + night = daily_start(datetime(2026, 11, 2, 23, 59, 59, tzinfo=UTC)) + assert morning == night == datetime(2026, 10, 20, tzinfo=UTC) + assert daily_start(datetime(2026, 11, 3, tzinfo=UTC)) == datetime(2026, 10, 21, tzinfo=UTC) + + def test_an_answer_is_kept_and_a_new_day_queries_anew(self) -> None: + fake, now = FakeQuery(rows=DAY_ROWS), [100.0] + read_daily(fake, DATASET, now=NOW, clock=lambda: now[0]) + now[0] += CACHE_TTL_S - 1 + read_daily(fake, DATASET, now=NOW, clock=lambda: now[0]) + assert len(fake.calls) == 1 + read_daily(fake, DATASET, now=NOW + timedelta(days=1), clock=lambda: now[0]) + assert len(fake.calls) == 2 # the window moved: another key + + def test_the_statuses_carry_no_rows(self) -> None: + assert read_daily(FakeQuery(), None).status == "not_configured" + assert read_daily(None, DATASET).status == "unavailable" + failing = FakeQuery(error=RuntimeError("403 Access Denied")) + failed = read_daily(failing, DATASET, now=NOW) + assert failed.status == "unavailable" and failed.rows == [] + # A failure is never kept. + assert read_daily(FakeQuery(rows=[]), DATASET, now=NOW).status == "ok" + + def test_needs_the_admin_gate(self) -> None: + fastapi_app.dependency_overrides[get_usage_query] = lambda: FakeQuery() + try: + with patch.object(settings, "admin_token", "supersecret"): + response = TestClient(app).get("/debug/agent-usage/daily") + finally: + fastapi_app.dependency_overrides.clear() + assert response.status_code == 401 + + +class TestDailyQuery: + def test_prunes_to_the_window_before_it_deduplicates(self) -> None: + sql = daily_sql(DATASET) + assert sql.index("timestamp >= TIMESTAMP_SUB(@since, INTERVAL 2 DAY)") < sql.index("QUALIFY ROW_NUMBER()") + assert "CURRENT_TIMESTAMP" not in sql and "CURRENT_DATE" not in sql + # The lookback only dates the chats: every count of the window reads from its start. + assert sql.count("WHERE ts >= @since") == 3 and "WHERE first_ts >= @since" in sql + + def test_counts_live_lines_and_the_fanout_cost_apart(self) -> None: + sql = daily_sql(DATASET) + assert "JSON_VALUE(json_payload.source) IN ('live', 'fanout')" in sql and "harness" not in sql + lines = sql[sql.index("lines AS") : sql.index("steps AS")] + assert "source = 'live'" in lines + assert "SUM(IF(source = 'live', usd, 0)) AS cost_usd" in sql + assert "SUM(IF(source = 'fanout', usd, 0)) AS fanout_cost_usd" in sql + + def test_prices_as_the_cost_view_does(self) -> None: + """The same steps, the same model id clean-up and the same cache factors as `05_prices_cost.sql`.""" + view = (Path(__file__).parents[3] / "agents/telemetry/views/05_prices_cost.sql").read_text(encoding="utf-8") + sql = daily_sql(DATASET) + for fragment in ( + r"r'(@.*|-[0-9]{8})$'", + "UNNEST(JSON_QUERY_ARRAY(", + "* prices.input_usd_per_m * 0.1", + "* prices.input_usd_per_m * 1.25", + "* prices.output_usd_per_m", + ") / 1e6", + ): + assert fragment in view and fragment in sql, fragment + steps = sql[sql.index("steps AS") : sql.index("costs AS")] + assert "event = 'chat_turn'" in steps and "event = 'data_input'" in steps and "pipeline_run" not in steps + + def test_counts_every_limit_code_where_a_line_records_it(self) -> None: + sql = daily_sql(DATASET) + for code in LIMIT_CODES: + assert f"COUNTIF(code = '{code}') AS hits_{code}" in sql + assert "IFNULL(JSON_VALUE(payload.refusal), JSON_VALUE(payload.error))" in sql + assert "WHEN 'data_input' THEN JSON_VALUE(payload.outcome)" in sql + assert "APPROX_QUANTILES(queue_wait_ms, 100 IGNORE NULLS)[SAFE_OFFSET(95)]" in sql From c0bf08c3b1846161c69f4fb485e1b4e5889818c7 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:05:57 +0100 Subject: [PATCH 2/4] feat(agent): finish the per-day numbers, the live line and the pause refusal Takes the pause note's clock to UTC plus the reader's own time on /debug, keeps the limit hits on one line on a phone, counts no chat for a line without a session, drops an unused `reserve`, and documents all of it: the /debug section, the API reference, the Plausible reasons and the concept doc. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- agents/anyplot/plugins/budget.py | 7 +- agents/anyplot/plugins/ledger.py | 13 ++- agents/stream.py | 2 +- api/agent_usage.py | 2 +- app/src/hooks/useAgentSession.test.ts | 62 ++++++++---- app/src/pages/DebugPage.test.tsx | 10 +- app/src/sections/agent-chat/messages.test.ts | 4 +- app/src/sections/debug/AgentDailyUsage.tsx | 33 +++---- app/src/sections/debug/format.ts | 6 ++ changelog.d/debug-daily-usage.md | 22 +++++ docs/concepts/agent-network.md | 4 +- docs/reference/agent-telemetry.md | 49 +++++++++- docs/reference/api.md | 99 ++++++++++++++++++-- docs/reference/plausible.md | 4 +- tests/unit/agents/runtime/test_day_budget.py | 6 +- tests/unit/api/test_agent_usage.py | 2 + 16 files changed, 253 insertions(+), 72 deletions(-) create mode 100644 changelog.d/debug-daily-usage.md diff --git a/agents/anyplot/plugins/budget.py b/agents/anyplot/plugins/budget.py index 95f28c1c6e5..9e223a5b0a8 100644 --- a/agents/anyplot/plugins/budget.py +++ b/agents/anyplot/plugins/budget.py @@ -4,9 +4,10 @@ the request has spent `AGENT_MAX_LLM_CALLS` calls or `AGENT_REQUEST_TOKEN_BUDGET` cost-weighted tokens (`ledger.BUDGET_WEIGHTS`), or the user or the service has spent its daily ones: `service_paused` when only the service's day budget is spent, and - `locked_out` past the day's strikes (`ledger.day_refusal`). A halt inside the adapter or the reviewer would break their schema - parsing, so the pipeline checks `ledger.budget_allows` before each `run_node` and - finishes with `PlotResult(failed, reason=budget)` instead. Before a review it keeps + `locked_out` past the day's strikes (`ledger.day_refusal`). A halt inside the adapter + or the reviewer would break their schema parsing, so the pipeline checks + `ledger.budget_allows` before each `run_node` and finishes with + `PlotResult(failed, reason=budget)` instead. Before a review it keeps room for the review and the root's closing reply (`pipeline.REVIEW_RESERVE_TOKENS`, the measured need), and once a plot has shipped in the invocation (`ledger.plot_shipped`) the root's next call is exempt from the halt, once and with diff --git a/agents/anyplot/plugins/ledger.py b/agents/anyplot/plugins/ledger.py index e104575957c..b09b23e5a3f 100644 --- a/agents/anyplot/plugins/ledger.py +++ b/agents/anyplot/plugins/ledger.py @@ -367,10 +367,10 @@ def locked_out(self, user_id: str, settings: AgentSettings) -> bool: `daily_ok` folds into its answer, told apart from a spent token budget.""" return self.user_strikes(user_id) >= settings.attack_strikes - def paused(self, settings: AgentSettings, *, reserve: int = 0) -> bool: - """Whether the service's day budget (`AGENT_GLOBAL_DAILY_TOKEN_BUDGET`) is spent, `reserve` tokens beyond the - spend: the `service_paused` refusal for everyone until 00:00 UTC, which `daily_ok` folds into its answer.""" - return self.global_tokens() + reserve >= settings.global_daily_token_budget + def paused(self, settings: AgentSettings) -> bool: + """Whether the service's day budget (`AGENT_GLOBAL_DAILY_TOKEN_BUDGET`) is spent: the `service_paused` refusal + for everyone until 00:00 UTC, which `daily_ok` folds into its answer.""" + return self.global_tokens() >= settings.global_daily_token_budget def snapshot(self, settings: AgentSettings) -> DaySnapshot: """Today's global spend and how many users are at their own day caps, as counts (`DaySnapshot`).""" @@ -536,13 +536,12 @@ def _at_least(counts: dict[str, int], cap: int) -> set[str]: return {user for user, count in counts.items() if count >= cap} -def day_refusal(usage: UsageBook, user_id: str, settings: AgentSettings, *, reserve: int = 0) -> str: +def day_refusal(usage: UsageBook, user_id: str, settings: AgentSettings) -> str: """Which refusal a failed `daily_ok` is: `locked_out` past the day's strikes, `budget` when the user's own day budget is spent, `service_paused` when only the service's is (its text says the user's own limit is untouched).""" if usage.locked_out(user_id, settings): return "locked_out" - own_left = usage.user_tokens(user_id) + reserve < settings.daily_token_budget - if own_left and usage.paused(settings, reserve=reserve): + if usage.user_tokens(user_id) < settings.daily_token_budget and usage.paused(settings): return SERVICE_PAUSED return "budget" diff --git a/agents/stream.py b/agents/stream.py index e721c73954b..f5617d67d12 100644 --- a/agents/stream.py +++ b/agents/stream.py @@ -10,7 +10,7 @@ | `status` | `step`, `attempt`, and on `repairing` `outcome`, `findings`, `defects` | the pipeline's content-free `custom_metadata` progress events; the step that opens a repair round says how the round before it ended (`RoundOutcome`, a count) and the rubric ids it repairs (`VQ-05`, closed codes) | | `message` | `text` | a final, non-partial text response authored by the root (`anyplot`) | | `plot` | `status`, `reason`, `attempts`, `artifacts`, `changes`, `residual_defects`, `version`, `rounds` | the pipeline's `PlotResult` output event; `version`, the stored version's number, only on `ok` and `needs_attention`; `rounds`, how each attempt ended (`{attempt, outcome, findings, step, shipped, defects}`: the step it ended in, whether its render is the one shipped, and the rubric ids it passed on to the repair) | -| `refusal` | `code`, `text` | the request ledger's refusal, in place of the message: `out_of_scope`, `needs_data` (data that would have to be fetched), `other_library` (this plot in another library or language: a fixed pointer to the catalogue, not counted toward closing by default), `unsupported_content`, `locked_out` (past the day's `attack` strikes) and `session_closed` (the session's third counted refusal) from the scope guard, `budget` from the budgets (`service_paused` when only the service's day budget is spent, for every user); the refusal one before the closing one ends with the fixed `close_warning` sentence; right after `ready`, before the scope judge: `locked_out`, `budget` for a user over the daily budget, `rate_limited` or `daily_limit` past the user's message limits, `session_closed` in a closed session | +| `refusal` | `code`, `text` | the request ledger's refusal, in place of the message: `out_of_scope`, `needs_data` (data that would have to be fetched), `other_library` (this plot in another library or language: a fixed pointer to the catalogue, not counted toward closing by default), `unsupported_content`, `locked_out` (past the day's `attack` strikes) and `session_closed` (the session's third counted refusal) from the scope guard, `budget` from the budgets (`service_paused` when only the service's day budget is spent, for every user); the refusal one before the closing one ends with the fixed `close_warning` sentence; right after `ready`, before the scope judge: `locked_out`, `budget` for a user over the daily budget, `service_paused` for everyone else once the service's day budget is spent, `rate_limited` or `daily_limit` past the user's message limits, `session_closed` in a closed session | | `error` | `code`, `ref` | `guard_unavailable`, `capacity` (also when a sandbox run found the queue full or waited its maximum; the turn ends there, without a `plot`), `run_limit` (the user already has `AGENT_USER_MAX_RUNS` sandbox runs queued or running in other sessions; the turn ends the same way), `deadline` or `internal` | | `done` | `llm_calls`, `tokens`, and after a run `bindings`, `data_confirmed`, `language` (`{language, source}`, the session language and how it was set), `suggestion` (the `.to_long()` offer in the session language, or null, when the session has data) and, when the turn changed the data (`prepare_data`, `generate_data`), `dataset` | the end of every run, always last; the route adds the session's sync fields (`_session_sync` in `agents/main.py`) | diff --git a/api/agent_usage.py b/api/agent_usage.py index 3585868fa4a..4b842099fac 100644 --- a/api/agent_usage.py +++ b/api/agent_usage.py @@ -383,7 +383,7 @@ def daily_sql(dataset: str) -> str: ), chats AS ( SELECT DATE(first_ts) AS day, COUNT(*) AS sessions - FROM (SELECT session, MIN(ts) AS first_ts FROM ev WHERE source = 'live' GROUP BY session) + FROM (SELECT session, MIN(ts) AS first_ts FROM ev WHERE source = 'live' AND session IS NOT NULL GROUP BY session) WHERE first_ts >= @since GROUP BY day ), diff --git a/app/src/hooks/useAgentSession.test.ts b/app/src/hooks/useAgentSession.test.ts index bdc3427edf0..6780a284509 100644 --- a/app/src/hooks/useAgentSession.test.ts +++ b/app/src/hooks/useAgentSession.test.ts @@ -19,6 +19,7 @@ import { } from 'src/hooks/useAgentSession'; import { type EffectiveTheme, ThemeContext } from 'src/hooks/useLayoutContext'; import type { DatasetChanged, DatasetParsed, RoleSpec } from 'src/lib/agent'; +import { nextUtcMidnight } from 'src/sections/agent-chat/messages'; import { pendingReply } from 'src/sections/agent-chat/steps'; const lastItem = (items: ChatItem[]) => items[items.length - 1]; @@ -491,6 +492,7 @@ describe('useAgentSession: data and bindings', () => { it.each([ [429, 'rate_limited'], [429, 'daily_limit'], + [429, 'service_paused'], [429, 'locked_out'], [403, 'session_closed'], ])('reports the paste route’s %i %s as a guardrail block', async (status, code) => { @@ -1671,24 +1673,48 @@ describe('useAgentSession: turns', () => { expect(JSON.stringify(trackEvent.mock.calls)).not.toContain('Munich'); }); - it.each(['rate_limited', 'daily_limit', 'session_closed', 'other_library', 'locked_out'])( - 'shows the %s refusal and records it as its own guardrail reason', - async code => { - const routes = baseRoutes(); - routes['POST /sessions/S1/messages'] = () => - sse([ - ['ready', { v: 'anyplot/1', run_id: 'r' }], - ['refusal', { code, text: 'A fixed reply.' }], - ['done', {}], - ]); - const { result } = await openSession(routes); - await act(() => result.current.sendMessage('another question')); - expect(result.current.state.items).toContainEqual( - expect.objectContaining({ kind: 'refusal', code, text: 'A fixed reply.' }) - ); - expect(trackEvent).toHaveBeenCalledWith('agent_guardrail_block', { reason: code }); - } - ); + it.each([ + 'rate_limited', + 'daily_limit', + 'service_paused', + 'session_closed', + 'other_library', + 'locked_out', + ])('shows the %s refusal and records it as its own guardrail reason', async code => { + const routes = baseRoutes(); + routes['POST /sessions/S1/messages'] = () => + sse([ + ['ready', { v: 'anyplot/1', run_id: 'r' }], + ['refusal', { code, text: 'A fixed reply.' }], + ['done', {}], + ]); + const { result } = await openSession(routes); + await act(() => result.current.sendMessage('another question')); + expect(result.current.state.items).toContainEqual( + expect.objectContaining({ kind: 'refusal', code, text: 'A fixed reply.' }) + ); + expect(trackEvent).toHaveBeenCalledWith('agent_guardrail_block', { reason: code }); + }); + + it('dates the service pause to the next 00:00 UTC, and no other refusal', async () => { + const routes = baseRoutes(); + let code = 'service_paused'; + routes['POST /sessions/S1/messages'] = () => + sse([ + ['ready', { v: 'anyplot/1', run_id: 'r' }], + ['refusal', { code, text: 'A fixed reply.' }], + ['done', {}], + ]); + const { result } = await openSession(routes); + await act(() => result.current.sendMessage('first')); + code = 'budget'; + await act(() => result.current.sendMessage('second')); + + const refusals = result.current.state.items.filter(item => item.kind === 'refusal'); + expect(refusals.map(item => item.code)).toEqual(['service_paused', 'budget']); + expect(refusals[0].until).toBe(nextUtcMidnight()); + expect(refusals[1].until).toBeUndefined(); + }); it('turns an HTTP 409 and a stream without done into errors with their ref', async () => { const routes = baseRoutes(); diff --git a/app/src/pages/DebugPage.test.tsx b/app/src/pages/DebugPage.test.tsx index 289afae7f16..a85344f3e61 100644 --- a/app/src/pages/DebugPage.test.tsx +++ b/app/src/pages/DebugPage.test.tsx @@ -2,6 +2,7 @@ import { fireEvent } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DebugPage } from 'src/pages/DebugPage'; +import { localClock } from 'src/sections/agent-chat/messages'; import { render, screen, waitFor, within } from 'src/test-utils'; const mockDebugData = { @@ -634,9 +635,7 @@ describe('DebugPage agent usage', () => { const days = await screen.findByRole('group', { name: 'Per day' }); const table = await within(days).findByRole('table'); const [head, today, quiet] = within(table).getAllByRole('row'); - expect(head).toHaveTextContent( - 'day (UTC)new chatsplotscostfan-outlimit hitsp95 queue wait' - ); + expect(head).toHaveTextContent('day (UTC)new chatsplotscostfan-outlimit hitsp95 queue wait'); expect(today).toHaveTextContent( '2026-11-02 · so far43$0.02$0.12service_paused 2 · capacity 112.4 s' ); @@ -699,7 +698,10 @@ describe('DebugPage agent usage', () => { render(); expect(await screen.findByTestId('agent-today-paused')).toHaveTextContent( - 'paused · the day budget is spent: every turn gets service_paused until' + 'paused · the day budget is spent: every turn gets service_paused until 00:00 UTC (' + ); + expect(screen.getByTestId('agent-today-paused')).toHaveTextContent( + `(${localClock(BUDGET.resets_at)} your time)` ); // No price for the model: the tokens alone. expect(screen.getByTestId('agent-today')).toHaveTextContent( diff --git a/app/src/sections/agent-chat/messages.test.ts b/app/src/sections/agent-chat/messages.test.ts index 65d683b59f9..8fabc434c64 100644 --- a/app/src/sections/agent-chat/messages.test.ts +++ b/app/src/sections/agent-chat/messages.test.ts @@ -26,9 +26,7 @@ describe('the service pause', () => { }); it('is the line of a paused paste or file, never the daily limit of the user', () => { - expect(describeDataFailure('service_paused', 'r-9')).toBe( - `${servicePausedText()} · ref r-9` - ); + expect(describeDataFailure('service_paused', 'r-9')).toBe(`${servicePausedText()} · ref r-9`); expect(describeConvertFailure('a.xlsx', 'service_paused', null, null, null)).toBe( `"a.xlsx" ${servicePausedText()}` ); diff --git a/app/src/sections/debug/AgentDailyUsage.tsx b/app/src/sections/debug/AgentDailyUsage.tsx index 1c3e6a60172..90ca587718f 100644 --- a/app/src/sections/debug/AgentDailyUsage.tsx +++ b/app/src/sections/debug/AgentDailyUsage.tsx @@ -22,7 +22,7 @@ import Typography from '@mui/material/Typography'; import { DEBUG_API_URL } from 'src/constants'; import { fetchWithAuth } from 'src/lib/api'; import { localClock } from 'src/sections/agent-chat/messages'; -import { formatUtc, timeAgo } from 'src/sections/debug/format'; +import { formatUsd, formatUtc, timeAgo } from 'src/sections/debug/format'; import { listMutedSx, listTextSx } from 'src/sections/debug/styles'; import { colors } from 'src/theme'; @@ -159,12 +159,6 @@ function readDaily(body: unknown): AgentUsageDaily | null { // Formatting // ============================================================================ -/** Dollars to the cent; a spend below a cent is `<$0.01`, never `$0.00`. */ -export function formatUsd(value: number): string { - if (value > 0 && value < 0.005) return '<$0.01'; - return `$${value.toFixed(2)}`; -} - const millions = (tokens: number): string => `${(tokens / 1e6).toFixed(1)} M`; /** The time of day of a UTC instant, `14:05 UTC`. */ @@ -229,12 +223,7 @@ const subheadSx = { } as const; export function AgentTodayLine({ token, reloadCounter }: DayViewProps) { - const budget = useAnswer( - `${DEBUG_API_URL}/debug/agent/budget`, - token, - reloadCounter, - readBudget - ); + const budget = useAnswer(`${DEBUG_API_URL}/debug/agent/budget`, token, reloadCounter, readBudget); return ( @@ -253,7 +242,8 @@ export function AgentTodayLine({ token, reloadCounter }: DayViewProps) { } function TodayText({ budget }: { budget: AgentBudget }) { - const share = budget.budget_tokens > 0 ? Math.round((budget.tokens / budget.budget_tokens) * 100) : null; + const share = + budget.budget_tokens > 0 ? Math.round((budget.tokens / budget.budget_tokens) * 100) : null; const spend = budget.usd !== null && budget.budget_usd !== null ? `${formatUsd(budget.usd)} of ${formatUsd(budget.budget_usd)}` @@ -264,7 +254,9 @@ function TodayText({ budget }: { budget: AgentBudget }) { : ''; const breakdown = limitBreakdown(budget.limits); const startedToday = budget.since === budget.started_at; - const resets = localClock(budget.resets_at) ?? utcClock(budget.resets_at); + // UTC like the rest of the line, then the reader's own clock. + const local = localClock(budget.resets_at); + const resets = `${utcClock(budget.resets_at)}${local ? ` (${local} your time)` : ''}`; return ( <> {budget.paused && ( @@ -289,7 +281,8 @@ function TodayText({ budget }: { budget: AgentBudget }) { {startedToday ? `counted since ${utcClock(budget.since)}, when this instance started` : `counted since 00:00 UTC; this instance has run since ${formatUtc(budget.started_at)}`} - {' · '}a redeploy or a scale to zero starts the counts again · the new day starts at {resets} + {' · '}a redeploy or a scale to zero starts the counts again · the new day starts at{' '} + {resets} ); @@ -378,9 +371,9 @@ export function AgentDailyTable({ token, reloadCounter }: DayViewProps) { - new chats: live sessions by the day of their first line · cost: list price, the fan-out's - apart · limit hits: refusals and errors by code · — : no run on that day carried the - queue wait yet + new chats: live sessions by the day of their first line · cost: list price, the + fan-out's apart · limit hits: refusals and errors by code · — : no run on that day + carried the queue wait yet {daily.generated_at ? ` · generated ${timeAgo(daily.generated_at)}` : ''} @@ -412,7 +405,7 @@ function DayRow({ day, today }: { day: UsageDay; today: boolean }) { {formatUsd(day.fanout_cost_usd)} - + {hits.length === 0 ? ( 0 diff --git a/app/src/sections/debug/format.ts b/app/src/sections/debug/format.ts index 97418a724a0..74f2cca1ba5 100644 --- a/app/src/sections/debug/format.ts +++ b/app/src/sections/debug/format.ts @@ -17,6 +17,12 @@ export function timeAgo(iso: string): string { return `${Math.floor(diffMon / 12)}y ago`; } +/** Dollars to the cent; a spend below a cent is `<$0.01`, never `$0.00`. */ +export function formatUsd(value: number): string { + if (value > 0 && value < 0.005) return '<$0.01'; + return `$${value.toFixed(2)}`; +} + /** A UTC timestamp as `2026-10-12 14:30:05 UTC`; the input unchanged when it does not parse. */ export function formatUtc(iso: string): string { const date = new Date(iso); diff --git a/changelog.d/debug-daily-usage.md b/changelog.d/debug-daily-usage.md new file mode 100644 index 00000000000..a03d4de8622 --- /dev/null +++ b/changelog.d/debug-daily-usage.md @@ -0,0 +1,22 @@ +### Added + +- **Per-day agent numbers on `/debug`.** The usage card gains a table of the last + 14 UTC days, newest first: new chats, plots shipped, cost in dollars (the `cost` + view's logic, the admin fan-out's beside it), limit hits by code and the 95th + percentile of the run queue's wait, `—` on a day before the field. A new + `GET /debug/agent-usage/daily` reads it from the telemetry under the card's own + discipline: filtered on `timestamp` before it deduplicates, kept for ten minutes, + aggregates only. +- **A live line for today on `/debug`.** `GET /debug/agent/budget` shows the service's + spend against its day budget and how many users are at their own limits, counts + only, from the agents service's memory. The line says since when it counts, because + a redeploy or a scale-to-zero starts the counts again. + +### Changed + +- **A spent service budget has its own refusal.** When the service's day budget is + spent, not the user's own, the chat now answers `service_paused` in every language, + a paste or a file conversion `429 service_paused`, instead of the text for a user's + own limit. It names the reset at 00:00 UTC, the page adds that time in the user's + own time zone, and `GET /v1/status` answers `paused` and `resets_at`, so the chat + shows the note before the user types. diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 560bd923c2d..63ed6f5ec17 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -183,6 +183,7 @@ The owner decided the queue, the rate and the serial renders on 2026-10-09, afte | File conversions per user | at most `AGENT_USER_CONVERSIONS_PER_MINUTE` (5) in any 60 s and `AGENT_USER_CONVERSIONS_PER_DAY` (30) per UTC day (owner decision of 2026-10-15), across sessions, counted when the file goes to the sandbox and checked after the daily token budget and the strikes; the paste that follows counts as one paste. One conversion per session at a time (`409 run_active`; per user before 2026-10-21), at most 5 MB, 10 s in the sandbox, behind every pipeline render in the render slot (`503 capacity` after 20 s) | the `/dataset/convert` route (`UsageBook.convert_limit`, `Runtime.converting`) | | Scope refusals per session | the third counted refusal of a session (`AGENT_SESSION_REFUSALS`: `out_of_scope` or `attack`; `needs_data` never counts, `other_library` only with `AGENT_OTHER_LIBRARY_COUNTS`) closes it: that refusal and every later turn of the session answer `session_closed` ("reload the page to start a new session"), the later ones before the judge. The counted refusal before the closing one carries the fixed `close_warning` sentence (`AGENT_SESSION_CLOSE_WARNING`), after a live test of 2026-10-14 closed a session unwarned. Counted from the session's withheld messages, so the count ends with the session; a closed session's data is not carried into the next one (follow-up). The per-user strikes (`AGENT_ATTACK_STRIKES`, 3 `attack` verdicts per day across sessions and the dataset judge) end in the `locked_out` refusal for the day (the `budget` text before 2026-10-14, which read as a spent usage limit); the strikes live in the instance's memory and reset at midnight UTC or on a restart (a redeploy or a scale-to-zero) | the ScopeGuard, the `/messages` and `/dataset` routes | | Daily budget | a user already over the daily token budget gets the `budget` refusal at once, before the judge | the `/messages` route | +| Service's day budget | once `AGENT_GLOBAL_DAILY_TOKEN_BUDGET` is spent, every user who still has their own budget gets the `service_paused` refusal (a paste or a conversion `429 service_paused`) until the usage book's new day at 00:00 UTC. Owner request of 2026-11-02: the text says the whole chat paused, not the user's own limit, and `GET /v1/status` answers `paused` and `resets_at`, so the page shows the note before the user types | `ledger.day_refusal`, `UsageBook.paused` | | Concurrent renders per instance | 1 (`AGENT_RENDER_CONCURRENCY`), one theme per slot, for every backend and every route; a waiting pipeline render gets a freed slot before a waiting theme toggle | `SerialRenderer` (`render/serial.py`), applied in `Services.backend` | | Theme toggle | one render; it waits for a render slot at most `AGENT_REQUEST_DEADLINE_S` - `AGENT_RENDER_TIMEOUT_S` (120 s), then answers `503 capacity`; a theme that failed the host gates twice is answered from its record | `theme_render.py` | | Dataset size | at most 1,048,576 bytes of paste (204,800 until 2026-10-14; computed data 2,097,152), 20,000 rows, 50 columns and 200 characters per cell; one past any of them is refused before any model call (`413 too_long` for the bytes, `422 unparseable` for the rest). The canonical `data.csv` can grow to about 3.8 times the input (`7e15` becomes `7000000000000000.0`); the parser refuses a parse whose canonical CSV passes `MAX_CANONICAL_BYTES` (4 MiB, the renderer's `data_csv` limit, 2 MiB before), so the 1 MB `stress-csv-expansion` case (3,954,696 bytes canonical) still fits. Measured on a 914,795-byte paste of 19,999 rows: the parse 0.5 s, the pre-filter over every cell 0.22 s, one dataset-judge part (the judge reads headers and a sample, never every row). The dataset store accounts each version at its canonical CSV (up to 4 MiB) and profile, and a session keeps its current version and five undo versions, about 25 MiB at worst, so its cap rose from 64 MiB to 256 MiB (about ten such sessions per instance, inside the service's 4 GiB) | the `/dataset` route and `parse.py` (stress cases `stress-cap-*`, `stress-over-*`, `stress-csv-expansion`) | @@ -251,7 +252,8 @@ Callable only with Cloud Run IAM; the BFF mirrors it under `/debug/agent`. | Route | Body and result | Errors | |---|---|---| -| `GET /v1/status` | `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs}`: `waiting` entries in the run queue, runs `in_flight`, and `AGENT_USER_MAX_RUNS` | | +| `GET /v1/status` | `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs, paused, resets_at}`: `waiting` entries in the run queue, runs `in_flight`, `AGENT_USER_MAX_RUNS`, and whether the service's day budget is spent until `resets_at`, the next 00:00 UTC | | +| `GET /v1/budget` | `{date, since, started_at, resets_at, paused, model, tokens, budget_tokens, usd, budget_usd, users, users_at_limit, limits}`: the admin's live view of the day from the usage book (`UsageBook.snapshot`), counts only, never a user id, for the instance's lifetime; the BFF's `/debug/agent/budget` serves it behind the admin gate | | | `POST /v1/sessions` | `{user, spec_id, library, locale, snapshot: CatalogueSnapshot{spec_id, title, description, data_roles[], notes[], code, library_version}, source?}` (at most 64 KB; `source`: `fanout` only, for the admin fan-out's sessions, whose usage telemetry it files apart from `live`) → `{session_id, eligibility}` | `422 not_eligible` | | `POST /v1/sessions/{sid}/library` | `{library, snapshot}`; keeps dataset and bindings | `422`, `409 run_active` | | `POST /v1/sessions/{sid}/dataset` | `{text, confirmed?, source?}` → `{preview, profile, bindings, warnings, context_lines, roles, version, language}`; `source` is the name of a dropped file (at most 1,024 characters), screened on its own and answered as `file_name`, never a note; `context_lines` are the judged notes pasted around the table; `language` is the session language as `{language, source}` after the paste, which sets it (`pasted`) when nothing typed or chosen has (see [Session language](#session-language)); `roles` lists the spec's data roles as `{name, kinds, required, variadic, description, non_negative}`; a paste is `version` 1 and drops an earlier paste's undo history | `413 too_long`, `422 unparseable`, `422 formula` (text that asks for a formula, which the root's `generate_data` computes in the chat; never transcribed), `403 data_refused`, `422 code` (a program that generates data, which the root's `run_data_code` runs in the chat; checked before any parse) | diff --git a/docs/reference/agent-telemetry.md b/docs/reference/agent-telemetry.md index e60a8881b97..c1f9a36a417 100644 --- a/docs/reference/agent-telemetry.md +++ b/docs/reference/agent-telemetry.md @@ -98,7 +98,9 @@ session it opens for the fan-out (`useAgentSession` with `fanout`, `POST /debug/ `source: "fanout"`), the BFF forwards the value, and the agents service accepts `fanout` only (any other value is `422`). The service keeps the mark in session state (`anyplot.telemetry_source`), and every line of the session, whatever its event, carries `source = "fanout"` instead of `live`. Every existing `source = 'live'` filter (the -`/debug` card, `api/agent_usage.py`) leaves them out, and the views keep them apart by `source`. +`/debug` card, `api/agent_usage.py`) leaves them out, and the views keep them apart by `source`. The one exception +is the per-day table's `fan-out` column ([The /debug card](#the-debug-card)): these sessions spend the same day +budget, so their cost is shown beside the live cost, never inside it. Only an admin can open the chat today, so the value labels the owner's own test sessions. Once the chat is public, a user who sends the mark by hand only files their own lines as tests. @@ -350,6 +352,51 @@ minutes, so reloading `/debug` runs no query at all. The answer holds aggregates `not_configured` while the variable is unset, and `unavailable` when the query fails (no table yet, no permission, a timeout); both come with zeros, the card shows its empty state, and a failure is never kept. +Below the tiles the card shows two views of the day that do not follow the 7 or 30 days toggle. + +### Per day + +A table of the last 14 UTC days, newest first, reads `GET /debug/agent-usage/daily` (`daily_sql` and `read_daily` +in `api/agent_usage.py`). A day without a line shows zeros. + +| Column | What it counts | +|---|---| +| new chats | Live sessions whose first line falls on that day. A chat that was opened and never sent anything wrote no line and does not count. | +| plots | `pipeline_run` lines with the status `ok` or `needs_attention`. | +| cost | Live `chat_turn` steps and `data_input` judge calls, priced as `agents/telemetry/views/05_prices_cost.sql` prices them, from the views dataset's `model_prices`. `pipeline_run.steps` repeats its turn's tokens and is left out. A `+` means a step's model has no price row, so the sum is too low; the cell's tooltip says how many steps. | +| fan-out | The same cost for `source = 'fanout'` sessions. | +| limit hits | The counts of `service_paused`, `budget`, `daily_limit`, `rate_limited`, `run_limit` and `capacity`, from a `chat_turn`'s `refusal` or `error` and a `data_input`'s `outcome`. | +| p95 queue wait | The 95th percentile of `queue_wait_ms` on live `pipeline_run` lines. It reads `—` on a day whose runs do not carry the field. | + +`locked_out` is a safety lockout, not a usage limit, so it stays in the card's refusal list and out of this column. +`service_paused` exists since 2026-11-02. Before that, a spent service budget read `budget` in a turn and +`daily_limit` in a paste, so those two columns hold both causes on older days. + +Which lines count depends on their `source`: + +- `live` lines fill every column. +- `fanout` lines fill the `fan-out` column only. +- `harness` lines are never read. + +The query keeps the card's scan rules: it filters `timestamp` before it deduplicates, and the API keeps each answer +for 10 minutes. Its window is floored to the day instead of to 10 minutes, because the table's oldest row has to be +a whole day: it starts at 00:00 UTC 13 days ago, so the query text and its parameter stay the same all day. It also +reads the 2 days before the window, only to find each chat's first line, so a chat that began before the window is +not new on the window's first day. `model_prices` is a literal `UNNEST`, so the join reads no bytes. + +### Today, live + +The line above the table comes from the agents service's memory, not from BigQuery, through +`GET /debug/agent/budget`. It shows the service's spend against `AGENT_GLOBAL_DAILY_TOKEN_BUDGET` and how many +users are at their own limits, counts only, never an id. The line says since when it counts, because a redeploy or +a scale-to-zero starts the usage book again at zero: `since` is 00:00 UTC, or the instance's start if that came +later. A cold agents service also starts at the moment the page asks, so right after one the line reads zero. + +The dollar amounts are the weighted tokens at the main model's uncached input list price in the service's location, +so they carry `eu`'s 10 %: 100 M weighted tokens of Claude Haiku 5.5 read `$11.00`, not `$10.00`. Both the spend and +the budget use the same price, so the share is exact. The per-day table's cost comes from the telemetry's model and +token counts instead, so the two can differ for today. + ## On the legal page The telemetry is content-free and keyed by a salted, per-instance session hash, but it is still a record of diff --git a/docs/reference/api.md b/docs/reference/api.md index 78e3c432a4a..eb4840eade6 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -491,14 +491,15 @@ The routes mirror the agents service's `/v1` API. All paths below start with | Route | Request | Response | |---|---|---| -| `GET /status` | None | The service's `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs}` plus `"enabled": true`; `waiting` counts the turns in the agents service's run queue, `in_flight` the runs it is running, and `user_max_runs` is how many sandbox runs one user may have queued or running at once (`AGENT_USER_MAX_RUNS`) | +| `GET /status` | None | The service's `{libraries, model, location, version, provider, waiting, in_flight, user_max_runs, paused, resets_at}` plus `"enabled": true`; `waiting` counts the turns in the agents service's run queue, `in_flight` the runs it is running, `user_max_runs` is how many sandbox runs one user may have queued or running at once (`AGENT_USER_MAX_RUNS`), and `paused` says the service's day budget is spent, so every turn gets `service_paused` until `resets_at`, the next 00:00 UTC. The chat page shows that note before you type | +| `GET /budget` | None | The `/debug` page's live line for today: `{date, since, started_at, resets_at, paused, model, tokens, budget_tokens, usd, budget_usd, users, users_at_limit, limits}`. Counts and times from the agents service's memory, never a user id; see [Today's spend](#todays-spend) | | `GET /eligibility?spec=&library=` | Spec id and library id | The agents service's answer, passed through | | `POST /sessions` | `{spec_id, library, locale, source?}` (`source`: `fanout` only, sent by the admin's "plot in all libraries" for each session it opens, so the usage telemetry files the session apart from `live` ones) | `{session_id, eligibility}`; `404 not_found` when the spec has no implementation for the library | | `POST /sessions/{sid}/library` | `{spec_id, library}` | Switches the library; the dataset and bindings stay | -| `POST /sessions/{sid}/dataset` | `{text, confirmed?, source?}`, at most 1 MB (1,048,576 bytes) of UTF-8 (200 KB before 2026-10-14); `source` is the name of the file the page read `text` from (1 to 1,024 characters), which the agents service sanitises, screens on its own and answers as `file_name` (`(name withheld)` when its screen refused it), never as a note and never in the text | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version, language}` for a table, stored as version 1 of the data (an earlier paste's undo history is dropped; `suggestion` is the wide-table offer or `null`, see [Data changes](#data-changes); `language` is the session language as `{language, source}` after the paste, see [Session language](#session-language)): CSV, TSV, semicolon or pipe text; JSON as an array of records, an object of column lists, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`; a repeated key keeps its last value with a warning); JSON Lines, one JSON object per line, read as JSON records (source format `json_lines`; up to 10 lines above the first object, such as a title, become notes; a `source` named `.jsonl` or `.ndjson` is read as JSON Lines first, so a file of one record is one row; a line that is no object is `422 unparseable` with its line number); a JSON-stat 2.0 dataset, as statistics offices' APIs answer it (or a JSON-stat 1.x bundle of one dataset), flattened into a long table with one column per dimension of more than one category, named by its label and holding the category labels as text, and a typed `value` column, the one-category dimensions and the dataset's label, source and notes coming back in `context_lines` (source format `json_stat`; an inconsistent dataset is `422 unparseable`); the same shapes as a Python literal, also behind an assignment such as `cat_day = {...}`, with single quotes, trailing commas and tuples (text that starts with a bare `(` and is no literal is read as delimited text); a column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, also with a no-break space) is read as numbers and named ` ()` with a warning, unless the header already names the unit. One column of values (integers, numbers, dates or booleans, such as a header and 800 commute times for a histogram) is a table, stored as any paste; without a header line its first line is kept as the first value, the column is named `value`, and a warning says so (a list of 20,001 values without a header is `422 unparseable`, never cut). A column of digit codes such as postal codes with leading zeros is a table too, kept as text, and so is a column of categories under a header where at least one category repeats; when that header has the categories' form (no `_`, brackets, `:`, `/`, `%`, `#` or digit, and no more words than a category), the warning `read '
' as the column name` says which line became the name, since `Tier` above `Hund` and `Apfel` above `Birne` look alike. For text that is no table (prose, `label value` lines without a separator, a list of names typed one per line) of at most 4,000 characters and 200 non-blank lines, `{transcribed: true, text, preview, profile, warnings, note}`, the table the judge model read, with its CSV in `text` and nothing stored: the page shows it for confirmation, and confirming posts that CSV through this route again with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and keeps every row (no line is left out as a note or a total), never the parse, the pre-filter, the judge or the store. A pasted table may carry other lines: a title, metadata or notes above it, a source line or notes below it, and up to three summary rows at the end (`Total`, `Summe`…) are left out, each kind with one entry in `warnings`. The lines left out above and below the table (not the summary rows) come back in `context_lines`, one text line each and at most 2,000 characters in all: the notes the assistant reads with the data. They pass the same pre-filter as the cells and then the scope judge; when either refuses them, or the judge does not answer, `context_lines` is empty and `warnings` starts with the reason, while the table is stored all the same and no strike is counted. `413 too_long` above the limit, `422 unparseable` when neither the parser nor the model finds a table (with a `cause` that names what the data fails on: the lines with too many or too few fields, the line and column of a JSON error, the cap a table breaks, free text past the transcriber's character or line limit, which is never stored as it parsed, or the model running out of output room on the text, or out of time on a text of at least half those limits, with its line and character count), `422 formula` when the text asks for a formula such as `sin(x) von -pi bis pi` or `y = x**2` (no model reads it; ask for the formula in the chat, where the assistant computes the table, see [Data changes](#data-changes)), `422 code` when the text is a program that generates data, such as Python, JavaScript or R (checked before any parse; send it to the chat, where the assistant runs it; only text within the chat's 2,000 characters can move there), `403 data_refused` when a header, a cell or the raw text addresses an AI (personal data such as names, addresses, e-mail addresses, phone numbers and web addresses is allowed), `403 session_closed` in a session the scope check closed, `409 run_active` while a turn, a theme render or another paste is in flight in this session (your other sessions never block it), `429 rate_limited` past 10 pastes in a minute and `429 daily_limit` past 100 in a UTC day or past the daily token budget (only a paste that reaches the checks counts; one refused as a formula, code or unparseable text does not), `429 locked_out` for the rest of the UTC day after three messages or pastes classed as attacks, `503 guard_unavailable` when the judge model does not answer (an error, a quota, an answer that does not parse, or no answer in time on a short text, a stalled endpoint) | +| `POST /sessions/{sid}/dataset` | `{text, confirmed?, source?}`, at most 1 MB (1,048,576 bytes) of UTF-8 (200 KB before 2026-10-14); `source` is the name of the file the page read `text` from (1 to 1,024 characters), which the agents service sanitises, screens on its own and answers as `file_name` (`(name withheld)` when its screen refused it), never as a note and never in the text | `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version, language}` for a table, stored as version 1 of the data (an earlier paste's undo history is dropped; `suggestion` is the wide-table offer or `null`, see [Data changes](#data-changes); `language` is the session language as `{language, source}` after the paste, see [Session language](#session-language)): CSV, TSV, semicolon or pipe text; JSON as an array of records, an object of column lists, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`; a repeated key keeps its last value with a warning); JSON Lines, one JSON object per line, read as JSON records (source format `json_lines`; up to 10 lines above the first object, such as a title, become notes; a `source` named `.jsonl` or `.ndjson` is read as JSON Lines first, so a file of one record is one row; a line that is no object is `422 unparseable` with its line number); a JSON-stat 2.0 dataset, as statistics offices' APIs answer it (or a JSON-stat 1.x bundle of one dataset), flattened into a long table with one column per dimension of more than one category, named by its label and holding the category labels as text, and a typed `value` column, the one-category dimensions and the dataset's label, source and notes coming back in `context_lines` (source format `json_stat`; an inconsistent dataset is `422 unparseable`); the same shapes as a Python literal, also behind an assignment such as `cat_day = {...}`, with single quotes, trailing commas and tuples (text that starts with a bare `(` and is no literal is read as delimited text); a column whose every cell carries the same unit (`65 %`, `15,6 h`, `€ 5`, also with a no-break space) is read as numbers and named ` ()` with a warning, unless the header already names the unit. One column of values (integers, numbers, dates or booleans, such as a header and 800 commute times for a histogram) is a table, stored as any paste; without a header line its first line is kept as the first value, the column is named `value`, and a warning says so (a list of 20,001 values without a header is `422 unparseable`, never cut). A column of digit codes such as postal codes with leading zeros is a table too, kept as text, and so is a column of categories under a header where at least one category repeats; when that header has the categories' form (no `_`, brackets, `:`, `/`, `%`, `#` or digit, and no more words than a category), the warning `read '
' as the column name` says which line became the name, since `Tier` above `Hund` and `Apfel` above `Birne` look alike. For text that is no table (prose, `label value` lines without a separator, a list of names typed one per line) of at most 4,000 characters and 200 non-blank lines, `{transcribed: true, text, preview, profile, warnings, note}`, the table the judge model read, with its CSV in `text` and nothing stored: the page shows it for confirmation, and confirming posts that CSV through this route again with `confirmed: true`, which skips the free-text fallback (a one-column table would otherwise be read again) and keeps every row (no line is left out as a note or a total), never the parse, the pre-filter, the judge or the store. A pasted table may carry other lines: a title, metadata or notes above it, a source line or notes below it, and up to three summary rows at the end (`Total`, `Summe`…) are left out, each kind with one entry in `warnings`. The lines left out above and below the table (not the summary rows) come back in `context_lines`, one text line each and at most 2,000 characters in all: the notes the assistant reads with the data. They pass the same pre-filter as the cells and then the scope judge; when either refuses them, or the judge does not answer, `context_lines` is empty and `warnings` starts with the reason, while the table is stored all the same and no strike is counted. `413 too_long` above the limit, `422 unparseable` when neither the parser nor the model finds a table (with a `cause` that names what the data fails on: the lines with too many or too few fields, the line and column of a JSON error, the cap a table breaks, free text past the transcriber's character or line limit, which is never stored as it parsed, or the model running out of output room on the text, or out of time on a text of at least half those limits, with its line and character count), `422 formula` when the text asks for a formula such as `sin(x) von -pi bis pi` or `y = x**2` (no model reads it; ask for the formula in the chat, where the assistant computes the table, see [Data changes](#data-changes)), `422 code` when the text is a program that generates data, such as Python, JavaScript or R (checked before any parse; send it to the chat, where the assistant runs it; only text within the chat's 2,000 characters can move there), `403 data_refused` when a header, a cell or the raw text addresses an AI (personal data such as names, addresses, e-mail addresses, phone numbers and web addresses is allowed), `403 session_closed` in a session the scope check closed, `409 run_active` while a turn, a theme render or another paste is in flight in this session (your other sessions never block it), `429 rate_limited` past 10 pastes in a minute and `429 daily_limit` past 100 in a UTC day or past your daily token budget (only a paste that reaches the checks counts; one refused as a formula, code or unparseable text does not), `429 service_paused` when the service's day budget is spent, `429 locked_out` for the rest of the UTC day after three messages or pastes classed as attacks, `503 guard_unavailable` when the judge model does not answer (an error, a quota, an answer that does not parse, or no answer in time on a short text, a stalled endpoint) | | `PUT /sessions/{sid}/bindings` | `[{role, column}]`, at most 50 | The agents service's `{bindings, complete, missing_roles, warnings}`; `422 no_dataset` before a parse, `422 invalid` with the check's `errors` lines for a binding the spec roles or the columns refuse, `409 run_active` while a turn or a render is in flight. A change of the bindings drops the data confirmation below | | `PUT /sessions/{sid}/language` | `{language}`: a short language name or code such as `de`, `Deutsch` or `Swedish`, 1 to 40 characters that match `^[A-Za-z][A-Za-z -]{0,39}$` | The agents service's `{language, source}`, the stored form (`English` comes back as `en`, `Swedish` as typed) with the source now `chosen`, plus, when the session has data, `suggestion`: the `.to_long()` offer (or null) with its new column names in the new language. The replies and the plot text of the next version follow it (the chip sets the plot's language too), and the language a typed message implies never replaces it (see [Session language](#session-language)). Versions made before keep their text. `422` for a value outside the pattern, `409 run_active` while a turn or a render is in flight | -| `POST /sessions/{sid}/dataset/convert` | `{data_base64, sheet?}`: a dropped binary data file as base64, at most 5 MB (5,242,880 bytes) raw; `sheet` a sheet (or `.npz` array) name from a previous answer's `sheets` | `{text, format, kind, rows, columns, sheet, sheets, sheet_count, warnings, dropped}`: the file read on the server, in the renderer's sandbox, into the text the paste box takes, and nothing stored. Values keep the precision they came with: every fraction digit of a timestamp and its offset in its own zone (UTC, counted as `utc_timestamps`, only where the zone cannot be read), decimals and 64-bit integers exactly, float32 in its shortest text; `dropped` names the columns left out (durations, intervals, binary, map, union, nested; at most 20). `kind` is the format found from the first bytes (`xlsx`, `xlsm`, `xlsb`, `xls`, `ods`, `parquet`, `feather`, `arrow`, `npy`, `npz`); `text` is CSV, or JSON records when a Parquet or Arrow table has list columns (`format` `json`), which the dataset route expands into one row per value; a spreadsheet comes as its raw cell grid, so a title above the header stays a note. `sheet` is the sheet read (the first visible one with a cell unless one was asked for), `sheets` at most ten visible ones with cells (an empty sheet is not offered) and `sheet_count` all of them. A plain 1-D `.npy` array is one column, `value`; an `.npz` of two or more plain 1-D arrays of one length (what `np.savez(path, t=t, y=y)` writes) is one table, a column per array named after it, with `sheet` null and `sheets` empty, and its 0-D members are left out and named in `dropped`; any other `.npz` is read one array at a time, a 2-D or structured array before a 1-D one, which is the column of its name (`value` where the name could read as a value, such as `2024`). An array row whose cells are all missing comes as `NaN`, so the paste keeps it, and an array without any value is `file_empty`; `warnings` counts `formulas_without_value` (empty cells, xlsx and xlsm only), `hidden_sheets`, `dropped_columns`, `list_columns`, `utc_timestamps` and `renamed_columns` (a repeated column name given a `_2` suffix). Sheet names come whole, so a name from `sheets` can be sent back as is; a name over 255 characters, with a control or bidi character, or with a path separator is left out of the list. Send `text` to `POST /sessions/{sid}/dataset` with the file name and sheet as `source`. `413 too_long` above 5 MB, `422 invalid` for a body that is not this shape or no base64, `422 file_unsupported`, `file_encrypted` (remove the password), `file_pickle`, `file_hdf5` (never read), `file_corrupt` (also a zip member that inflates to another size than it declares, or a Parquet footer whose row groups do not add up), `file_archive_limits` (a zip declaring more than 100 MB, 250 times its size or 2,000 entries), `file_too_many_rows` (20,000, or 20,200 raw rows of a sheet, measured from the span of its cells before the sheet is loaded), `file_too_many_columns` (50), `file_formulas_only` (a sheet whose cells are all formulas without saved values: open it in Excel and save it once), `file_too_large` (more than 1 MB of text), `file_empty`, `file_sheet_not_found`, `file_array_shape` (no 1-D or 2-D array and no 1-D structured one), `file_object_array` (a pickled NumPy array), `file_timeout` (10 seconds), `file_too_complex` (the sandbox's memory or disk limit, or a reader that died without an answer) and `file_unreadable`, each with `measured` and `limit` where a cap was broken; `403 session_closed`, `409 run_active` (another conversion in flight in this session), `429 rate_limited` past 5 conversions in a minute and `429 daily_limit` past 30 in a UTC day or past the daily token budget, `429 locked_out`, `503 capacity` (no render slot within 20 seconds) and `503 convert_unavailable` (the renderer cannot convert files, or its sandbox could not install the process filter every run requires) | +| `POST /sessions/{sid}/dataset/convert` | `{data_base64, sheet?}`: a dropped binary data file as base64, at most 5 MB (5,242,880 bytes) raw; `sheet` a sheet (or `.npz` array) name from a previous answer's `sheets` | `{text, format, kind, rows, columns, sheet, sheets, sheet_count, warnings, dropped}`: the file read on the server, in the renderer's sandbox, into the text the paste box takes, and nothing stored. Values keep the precision they came with: every fraction digit of a timestamp and its offset in its own zone (UTC, counted as `utc_timestamps`, only where the zone cannot be read), decimals and 64-bit integers exactly, float32 in its shortest text; `dropped` names the columns left out (durations, intervals, binary, map, union, nested; at most 20). `kind` is the format found from the first bytes (`xlsx`, `xlsm`, `xlsb`, `xls`, `ods`, `parquet`, `feather`, `arrow`, `npy`, `npz`); `text` is CSV, or JSON records when a Parquet or Arrow table has list columns (`format` `json`), which the dataset route expands into one row per value; a spreadsheet comes as its raw cell grid, so a title above the header stays a note. `sheet` is the sheet read (the first visible one with a cell unless one was asked for), `sheets` at most ten visible ones with cells (an empty sheet is not offered) and `sheet_count` all of them. A plain 1-D `.npy` array is one column, `value`; an `.npz` of two or more plain 1-D arrays of one length (what `np.savez(path, t=t, y=y)` writes) is one table, a column per array named after it, with `sheet` null and `sheets` empty, and its 0-D members are left out and named in `dropped`; any other `.npz` is read one array at a time, a 2-D or structured array before a 1-D one, which is the column of its name (`value` where the name could read as a value, such as `2024`). An array row whose cells are all missing comes as `NaN`, so the paste keeps it, and an array without any value is `file_empty`; `warnings` counts `formulas_without_value` (empty cells, xlsx and xlsm only), `hidden_sheets`, `dropped_columns`, `list_columns`, `utc_timestamps` and `renamed_columns` (a repeated column name given a `_2` suffix). Sheet names come whole, so a name from `sheets` can be sent back as is; a name over 255 characters, with a control or bidi character, or with a path separator is left out of the list. Send `text` to `POST /sessions/{sid}/dataset` with the file name and sheet as `source`. `413 too_long` above 5 MB, `422 invalid` for a body that is not this shape or no base64, `422 file_unsupported`, `file_encrypted` (remove the password), `file_pickle`, `file_hdf5` (never read), `file_corrupt` (also a zip member that inflates to another size than it declares, or a Parquet footer whose row groups do not add up), `file_archive_limits` (a zip declaring more than 100 MB, 250 times its size or 2,000 entries), `file_too_many_rows` (20,000, or 20,200 raw rows of a sheet, measured from the span of its cells before the sheet is loaded), `file_too_many_columns` (50), `file_formulas_only` (a sheet whose cells are all formulas without saved values: open it in Excel and save it once), `file_too_large` (more than 1 MB of text), `file_empty`, `file_sheet_not_found`, `file_array_shape` (no 1-D or 2-D array and no 1-D structured one), `file_object_array` (a pickled NumPy array), `file_timeout` (10 seconds), `file_too_complex` (the sandbox's memory or disk limit, or a reader that died without an answer) and `file_unreadable`, each with `measured` and `limit` where a cap was broken; `403 session_closed`, `409 run_active` (another conversion in flight in this session), `429 rate_limited` past 5 conversions in a minute and `429 daily_limit` past 30 in a UTC day or past your daily token budget, `429 service_paused`, `429 locked_out`, `503 capacity` (no render slot within 20 seconds) and `503 convert_unavailable` (the renderer cannot convert files, or its sandbox could not install the process filter every run requires) | | `POST /sessions/{sid}/dataset/confirm` | none | Step 1 of two: you confirm that the stored dataset and its bindings fit; `{confirmed: true, rows, columns}`. The plot pipeline runs only on confirmed data (`create_plot` before it answers `plot{status: "not_ready", reason: "data_unconfirmed"}`); a new paste, a transcription's `.use()`, a binding change, a change of the data by the assistant or an undo drops the confirmation, a library switch keeps it. `422 no_dataset` before a parse, `409 run_active` while a turn or a render is in flight | | `POST /sessions/{sid}/dataset/undo` | none | Undo the assistant's last change to the data (see [Data changes](#data-changes)): the version before it is the current dataset again, `{preview, profile, bindings, warnings, context_lines, roles, suggestion, version}` like the dataset route (the wide-table offer comes back with a wide version), unconfirmed. The service keeps the last five versions. `422 nothing_to_undo` when the paste was never changed or the first data came from a formula, `409 run_active` while a turn or a render is in flight | | `POST /sessions/{sid}/dataset/copy` | `{from_session}`: the session id of the page that opened the fan-out, and nothing else | The admin's "plot in all libraries": the agents service copies the confirmed current data of `from_session` into this fan-out session as it is stored, whatever its origin (pasted, opened, computed or changed by the assistant), with its bindings, their confirmation, the session language and the language of its plot's text, and answers like the dataset route (version 1) plus `confirmed: true`, `complete`, `missing_roles` and `copied_from` (the source's version number). Nothing is parsed or judged again, and the copy counts as no paste. Both sessions are looked up under your user id: another user's session is `404 session_expired`. `409 has_dataset` when this session already has data, `422 no_dataset` or `422 unconfirmed` when `from_session` has no confirmed data, `422 invalid` for a session that is no fan-out session or another spec, `409 run_active`, `429 rate_limited` / `daily_limit` past 30 copies a minute or 150 a day, `503 capacity`. An agents service without the route answers `404 not_found`, on which the page posts the paste again | @@ -910,6 +911,88 @@ feedback; a saved session alone is no keep. `status` is `not_configured` while the variable is unset and `unavailable` when the query fails; both come with zeros, `null` rates and empty lists. See [Agent usage telemetry](agent-telemetry.md). +### Usage per day + +`GET /debug/agent-usage/daily` is the admin-gated, read-only source of the per-day +table on `/debug`. Like `/debug/agent-usage`, it runs one query over the BigQuery +telemetry and never calls the agents service. It answers the last 14 UTC days, +newest first, including days without a line: + +```json +{ + "status": "ok", + "days": 14, + "rows": [ + { + "date": "2026-11-02", + "sessions": 4, + "plots_shipped": 3, + "cost_usd": 0.0228, + "fanout_cost_usd": 0.1211, + "unpriced_calls": 0, + "limit_hits": { + "service_paused": 2, + "budget": 0, + "daily_limit": 0, + "rate_limited": 0, + "run_limit": 0, + "capacity": 1 + }, + "queue_wait_p95_ms": 12400, + "queue_waits": 3 + } + ], + "generated_at": "2026-11-02T14:05:30Z" +} +``` + +`cost_usd` is the live sessions' model cost at list price and `fanout_cost_usd` the +same for the admin fan-out's sessions. `unpriced_calls` counts the steps whose model +has no price row, so their cost is missing from both sums. `queue_wait_p95_ms` is +`null` on a day whose runs do not carry the field, and `queue_waits` is the number of +runs behind it. `status` is `not_configured` or `unavailable` as above, with no +`rows`. See [Agent usage telemetry](agent-telemetry.md). + +### Today's spend + +`GET /debug/agent/budget` is the admin-gated source of the live line on `/debug`. +Unlike the two routes above it reads the agents service, not BigQuery, and needs the +service to answer; an upstream failure keeps the agent routes' error shape: + +```json +{ + "date": "2026-11-02", + "since": "2026-11-02T14:05:12Z", + "started_at": "2026-11-02T14:05:12Z", + "resets_at": "2026-11-03T00:00:00Z", + "paused": false, + "model": "claude-haiku-5-5", + "tokens": 12000000, + "budget_tokens": 100000000, + "usd": 1.32, + "budget_usd": 11.0, + "users": 3, + "users_at_limit": 1, + "limits": { + "tokens": 1, + "runs": 0, + "data_runs": 0, + "messages": 0, + "pastes": 0, + "conversions": 0, + "locked_out": 0 + } +} +``` + +The counts live in the instance's memory: `since` is when today's counts began, the +instance's start or 00:00 UTC, whichever came later, and a redeploy or a scale-to-zero +starts them again at zero. `usd` and `budget_usd` are the weighted tokens at the main +model's input list price in the service's location, `null` for a model without a +price. `users` counts the users seen today, `users_at_limit` those at one or more of +their own day limits, each once, and `limits` those at each limit, so a user at two +limits counts under both. + ### SSE protocol `anyplot/1` | Event | Data | @@ -919,7 +1002,7 @@ the variable is unset and `unavailable` when the query fails; both come with zer | `status` | `{step, attempt}` for a pipeline step: `adapting`, `checking`, `rendering`, `reviewing`, or `repairing`; `repairing` adds `outcome` and `findings`, how the round before it ended (see `rounds` below), and `defects`, the rubric ids the repair works on (`VQ-01`, `VQ-05`) | | `message` | `{text}`: the assistant's reply as plain text, at most 1,200 characters; a longer reply arrives cut to 1,200 characters when the turn ran the plot pipeline, and as the `out_of_scope` refusal otherwise | | `plot` | `{status, reason, attempts, artifacts, changes, residual_defects, version, rounds}`; `version` only on `ok` and `needs_attention`; `rounds` lists each attempt as `{attempt, outcome, findings, step, shipped, defects}` (`step` is the pipeline step the round ended in, `shipped` marks the round whose render the result shipped, `defects` are the rubric ids the round passed on to the repair), where `outcome` is `ok` (the review passed), `review` (the review named `findings`), `measured` (measured findings went to the repair), `render_failed`, `check_failed`, `answer_unusable`, `unreviewed` or `error` | -| `refusal` | `{code, text}`; `code` is `out_of_scope`, `needs_data` (the request needs data the service would have to fetch, also code that reads its data from a URL or a file), `other_library` (this plot in another library or language: the fixed answer points to the plot's catalogue page and, by default, does not count toward closing the session; with `AGENT_OTHER_LIBRARY_COUNTS=true` it does), `unsupported_content`, `budget`, `locked_out` (paused until the next UTC day after three messages or pastes classed as attacks), `rate_limited` (more than 10 messages in a minute), `daily_limit` (more than 200 messages in a UTC day) or `session_closed` (the session's third `out_of_scope` or `attack` refusal, and every turn after it: reload the page to start a new session; the refusal before it ends with a fixed warning sentence), and `text` is the fixed reply for it in the language the scope judge detects in your message, or in the session language for a refusal sent before the judge runs (`budget`, `locked_out`, `unsupported_content`, `rate_limited`, `daily_limit`, and `session_closed` after the closing turn), which is English until your pasted data, your first message or your choice sets another (see [Session language](#session-language)): English, German, French, Spanish, Italian, Portuguese, Dutch, Polish, Chinese, Vietnamese or Russian, and English for any other language | +| `refusal` | `{code, text}`; `code` is `out_of_scope`, `needs_data` (the request needs data the service would have to fetch, also code that reads its data from a URL or a file), `other_library` (this plot in another library or language: the fixed answer points to the plot's catalogue page and, by default, does not count toward closing the session; with `AGENT_OTHER_LIBRARY_COUNTS=true` it does), `unsupported_content`, `budget` (your own daily token budget is spent), `service_paused` (the service's day budget is spent, not yours: the whole chat pauses until 00:00 UTC), `locked_out` (paused until the next UTC day after three messages or pastes classed as attacks), `rate_limited` (more than 10 messages in a minute), `daily_limit` (more than 200 messages in a UTC day) or `session_closed` (the session's third `out_of_scope` or `attack` refusal, and every turn after it: reload the page to start a new session; the refusal before it ends with a fixed warning sentence), and `text` is the fixed reply for it in the language the scope judge detects in your message, or in the session language for a refusal sent before the judge runs (`budget`, `service_paused`, `locked_out`, `unsupported_content`, `rate_limited`, `daily_limit`, and `session_closed` after the closing turn), which is English until your pasted data, your first message or your choice sets another (see [Session language](#session-language)): English, German, French, Spanish, Italian, Portuguese, Dutch, Polish, Chinese, Vietnamese or Russian, and English for any other language | | `error` | `{code, ref}`; `code` is `capacity` (the run queue was full when the turn's plot run or data snippet asked for a place, the run waited the queue's maximum of 600 seconds, or the BFF's turn cap ran out while it waited), `run_limit` (you already have three plot runs or data snippets queued or running in other sessions, `AGENT_USER_MAX_RUNS`), `deadline`, `guard_unavailable`, `upstream`, or `internal`; `ref` is the request id | | `done` | `{llm_calls, tokens, bindings?, data_confirmed?, dataset?, language?, suggestion?}`: after a run, the session's bindings and whether the data is confirmed, so the page syncs its dropdowns and its step after a turn in which the root's `set_bindings` changed them; `dataset` only after a turn in which the assistant changed the data or computed it from a formula (see [Data changes](#data-changes)): `{preview, profile, bindings, warnings, context_lines, roles, changes, version}`, the dataset route's shape (`roles` at most 50; `context_lines`, the pasted notes, carried over to every version made from the paste) plus the turn's change lines, for the page's preview table; `version` is 1 when a formula or the user's code made the session's first data; `code`, only on a version computed from the user's code, holds the Python that ran; `language` after every run: `{language, source}`, the session language and how it was set (`default`, `pasted`, `detected` or `chosen`), so the page syncs its chip; `suggestion` after every run on a session with data: the `.to_long()` offer, or null, with its new column names in the session language, so the data panel's card names what `.to_long()` applies | @@ -959,8 +1042,10 @@ Before the scope check, and without any model call, the stream sends `ready`, a `refusal` and `done` at once when you are paused for the day after three messages or pastes the checks classed as attacks (`locked_out`; the count lives in the agents service's memory and resets at midnight UTC or when the service -restarts), over the daily -token budget (`budget`), past 10 messages in a minute (`rate_limited`) or 200 +restarts), over your daily +token budget (`budget`), when the whole service has spent its day budget +(`service_paused`, until 00:00 UTC; the page adds that time in your own time +zone), past 10 messages in a minute (`rate_limited`) or 200 in a UTC day (`daily_limit`), counting typed messages and the page's actions alike, or when the session is closed (`session_closed`). The scope check closes a session at its third `out_of_scope` refusal; a `needs_data` refusal never @@ -998,7 +1083,7 @@ upstream `4xx` or `5xx` keeps its status. Its code is one the agents service documents (`not_eligible`, `run_active`, `too_long`, `unparseable`, `formula`, `code`, `data_refused`, `session_expired`, `guard_unavailable` and `capacity` (both `503`, worth a retry), `no_dataset`, -`nothing_to_undo`, `rate_limited`, `daily_limit`, `locked_out`, `session_closed`, +`nothing_to_undo`, `rate_limited`, `daily_limit`, `service_paused`, `locked_out`, `session_closed`, `convert_unavailable`, the file conversion's `file_*` refusals, and, for a JavaScript plot's `plot.html`, `page_unavailable` and `page_failed`) or a generic one for the status (`bad_request`, `not_found`, `conflict`, diff --git a/docs/reference/plausible.md b/docs/reference/plausible.md index 985e5c974e4..0015c58a891 100644 --- a/docs/reference/plausible.md +++ b/docs/reference/plausible.md @@ -145,7 +145,7 @@ with `VITE_ENABLE_AGENT_CHAT=true`, and like every event they fire only on | `agent_data_undo` | `version` | useAgentSession.ts | The user took back the assistant's last change to the data with `.undo()`, and the server restored the version before it. `version` ∈ `1` (back at the paste), `2`, `3+`: the restored version | | `agent_data_file` | `kind`, `encoding`, `outcome` | DataPanel.tsx | A file reached the data box: dropped on it, chosen with `.open()` or pasted as a file (since 2026-10-15). A text file is read in the browser; a spreadsheet, Parquet, Feather, Arrow or NumPy file goes to the server, which converts it in the renderer's sandbox (since the binary-file step of 2026-10-15). `.parse()` afterwards reports `agent_data_parsed` as for a paste. `kind` ∈ `text`, `spreadsheet`, `parquet`, `arrow`, `numpy`, `hdf5`, `pickle`, `pdf`, `archive`, `binary` (by the file's first bytes in the browser; `binary` for anything else that is not text), `unknown` (refused before it was read), and for a file the server converted the format it found: `xlsx`, `xlsm`, `xlsb`, `xls`, `ods`, `parquet`, `feather`, `arrow`, `npy`, `npz`; `encoding` ∈ `utf-8`, `utf-8-bom`, `utf-16le`, `utf-16be`, `windows-1252`, `none` (a binary file); `outcome` ∈ `filled` (into an empty box), `replaced` (the box had text and the user chose `.replace()`), `kept` (the user chose `.keep()`), `sheet` (another sheet or `.npz` array chosen in the sheet picker and read again), `refused` (a format never read, an empty file, or a file the server refused: encrypted, damaged, over a cap), `failed` (the server could not convert it now: rate limit, no render slot, renderer unavailable, network), `too_large` (over 4 MB as a text file or 5 MB as a binary one, or over 1 MB as text), `busy` (a step was running, or another file was still being read), `unreadable` (the browser could not read it). The file's name and its sheet names are never sent | | `agent_plot_rendered` | `library`, `status`, `repaired`, `spec` | useAgentSession.ts | A turn's `plot` event arrived. `status` ∈ `ok`, `needs_attention`, `failed`, `not_ready`; `repaired` is `yes` when the run needed its repair round (two attempts), else `no` | -| `agent_guardrail_block` | `reason` | useAgentSession.ts | A guardrail stopped a request. `reason` ∈ `out_of_scope`, `needs_data` (the request needs data the service would have to fetch), `budget`, `unsupported_content`, `rate_limited` and `daily_limit` (the user's message limits per minute and per day, and since 2026-10-14 the paste limits, which the data panel reports from the dataset route's `429`), `session_closed` (the session closed after three scope refusals), `other_library` (a request for this plot in another library or language, answered with a pointer to the catalogue), `locked_out` (the user is paused for the day after three attack verdicts; these are the stream's fixed refusals), `data_refused` (the pre-filter or the dataset judge refused the pasted data: text addressed to an AI, or the judge's verdict; personal data is never a reason), `guard_unavailable` (the scope check could not answer and failed closed), `other` (a refusal code the page does not know yet) | +| `agent_guardrail_block` | `reason` | useAgentSession.ts | A guardrail stopped a request. `reason` ∈ `out_of_scope`, `needs_data` (the request needs data the service would have to fetch), `budget` (the user's own daily token budget), `service_paused` (since 2026-11-02: the whole service spent its day budget, from a turn's refusal or the dataset route's `429`), `unsupported_content`, `rate_limited` and `daily_limit` (the user's message limits per minute and per day, and since 2026-10-14 the paste limits, which the data panel reports from the dataset route's `429`), `session_closed` (the session closed after three scope refusals), `other_library` (a request for this plot in another library or language, answered with a pointer to the catalogue), `locked_out` (the user is paused for the day after three attack verdicts; these are the stream's fixed refusals), `data_refused` (the pre-filter or the dataset judge refused the pasted data: text addressed to an AI, or the judge's verdict; personal data is never a reason), `guard_unavailable` (the scope check could not answer and failed closed), `other` (a refusal code the page does not know yet) | | `agent_session_saved` | `include_data` | useAgentSession.ts | The user saved the session as a JSON file with `.save()` in the chat header, and the download started. `include_data` is `yes` when the `# include data.csv` box was ticked (the default), so the file holds each version's data file, else `no`. A failed save fires nothing | | `agent_result_feedback` | `reaction`, `include_data`, `subject` | useAgentSession.ts | The user sent quick feedback with `.send()`, on a result version from its result card or on the chat itself from the control below the composer, and the BFF stored it as a feedback case. `reaction` ∈ `thumbs_up`, `thumbs_down`, `bug`, `idea`, the site feedback's set; `include_data` is `yes` when the `# include data.csv` box was ticked (the default), so the case holds the data file, else `no`; `subject` is `result` for a version and `chat` for the chat (since 2026-10-12, for a session that made no plot). An edit that replaces a sent case fires again. A failed send fires nothing; the text never travels | @@ -516,7 +516,7 @@ To see event properties in Plausible dashboard, you **MUST** register them as cu | `encoding` | The text encoding the browser read a dropped file with (`utf-8`, `utf-8-bom`, `utf-16le`, `utf-16be`, `windows-1252`, `none`) | `agent_data_file` | | `outcome` | What happened to a dropped file (`filled`, `replaced`, `kept`, `sheet`, `refused`, `failed`, `too_large`, `busy`, `unreadable`) | `agent_data_file` | | `repaired` | Whether an agent run needed its repair round (`yes` / `no`) | `agent_plot_rendered` | -| `reason` | Why a guardrail stopped an agent chat request (`out_of_scope`, `needs_data`, `budget`, `unsupported_content`, `rate_limited`, `daily_limit`, `session_closed`, `other_library`, `locked_out`, `data_refused`, `guard_unavailable`, `other`) | `agent_guardrail_block` | +| `reason` | Why a guardrail stopped an agent chat request (`out_of_scope`, `needs_data`, `budget`, `service_paused`, `unsupported_content`, `rate_limited`, `daily_limit`, `session_closed`, `other_library`, `locked_out`, `data_refused`, `guard_unavailable`, `other`) | `agent_guardrail_block` | | `include_data` | Whether a saved agent chat session, or a feedback case, carries the data file (`yes` / `no`) | `agent_session_saved`, `agent_result_feedback` | | `reaction` | The reaction of a feedback entry or of an agent result's quick feedback (`thumbs_up`, `thumbs_down`, `bug`, `idea`) | `feedback_submitted`, `agent_result_feedback` | | `platform` | Bot/platform name (twitter, whatsapp, teams, etc.) | `og_image_view` | diff --git a/tests/unit/agents/runtime/test_day_budget.py b/tests/unit/agents/runtime/test_day_budget.py index 6b509330104..92cfe62e062 100644 --- a/tests/unit/agents/runtime/test_day_budget.py +++ b/tests/unit/agents/runtime/test_day_budget.py @@ -81,7 +81,7 @@ def test_the_pause_is_the_service_budget_and_never_the_users_own(self) -> None: usage.add_tokens("a", 400) assert not usage.paused(settings) and usage.daily_ok("b", settings) usage.add_tokens("a", 599) - assert usage.paused(settings, reserve=1) and not usage.paused(settings) + assert not usage.paused(settings) usage.add_tokens("c", 1) # Spent by others: `b` still has their own budget, so the refusal says the whole service paused. assert usage.paused(settings) and not usage.daily_ok("b", settings) @@ -188,9 +188,7 @@ async def test_a_root_call_past_the_service_budget_mid_turn_says_so( ) -> None: """The Budget plugin's halt before a root call names the pause too, not the user's own `budget`.""" ledger = RequestLedger(user_id=USER, lang="de") - context = SimpleNamespace( - invocation_id="inv-1", agent_name=ROOT_AGENT, session=SimpleNamespace(user_id=USER) - ) + context = SimpleNamespace(invocation_id="inv-1", agent_name=ROOT_AGENT, session=SimpleNamespace(user_id=USER)) services.usage.add_tokens(OTHER, small_budget.global_daily_token_budget) token = CURRENT_LEDGER.set(ledger) try: diff --git a/tests/unit/api/test_agent_usage.py b/tests/unit/api/test_agent_usage.py index 408297988dd..36370e56160 100644 --- a/tests/unit/api/test_agent_usage.py +++ b/tests/unit/api/test_agent_usage.py @@ -388,6 +388,8 @@ def test_prunes_to_the_window_before_it_deduplicates(self) -> None: assert "CURRENT_TIMESTAMP" not in sql and "CURRENT_DATE" not in sql # The lookback only dates the chats: every count of the window reads from its start. assert sql.count("WHERE ts >= @since") == 3 and "WHERE first_ts >= @since" in sql + # As the card's `COUNT(DISTINCT session)`: a line without a session is no chat. + assert "source = 'live' AND session IS NOT NULL GROUP BY session" in sql def test_counts_live_lines_and_the_fanout_cost_apart(self) -> None: sql = daily_sql(DATASET) From 4f9fc5adf21f8377850c712fe84e2dc5cef365ae Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:57:10 +0100 Subject: [PATCH 3/4] docs(changelog): name the PR in the debug usage fragment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- changelog.d/debug-daily-usage.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/debug-daily-usage.md b/changelog.d/debug-daily-usage.md index a03d4de8622..38b0efc7e47 100644 --- a/changelog.d/debug-daily-usage.md +++ b/changelog.d/debug-daily-usage.md @@ -6,11 +6,11 @@ percentile of the run queue's wait, `—` on a day before the field. A new `GET /debug/agent-usage/daily` reads it from the telemetry under the card's own discipline: filtered on `timestamp` before it deduplicates, kept for ten minutes, - aggregates only. + aggregates only. (#12518) - **A live line for today on `/debug`.** `GET /debug/agent/budget` shows the service's spend against its day budget and how many users are at their own limits, counts only, from the agents service's memory. The line says since when it counts, because - a redeploy or a scale-to-zero starts the counts again. + a redeploy or a scale-to-zero starts the counts again. (#12518) ### Changed @@ -19,4 +19,4 @@ a paste or a file conversion `429 service_paused`, instead of the text for a user's own limit. It names the reset at 00:00 UTC, the page adds that time in the user's own time zone, and `GET /v1/status` answers `paused` and `resets_at`, so the chat - shows the note before the user types. + shows the note before the user types. (#12518) From 4501e65906dd6399936997c67a568dab2f469049 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Mon, 2 Nov 2026 07:08:13 +0100 Subject: [PATCH 4/4] fix(agent): the pause note claims nothing about the user's own limit `/status` reports the pause for everyone, so the page cannot know that a user still has their own budget; one who does not reads a false claim and then gets `budget`. The server's refusal keeps the sentence, because `day_refusal` sends it only to users with their own budget left. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H7HSkGyea1k2v293XiAMo8 --- app/src/pages/AgentChatPage.test.tsx | 3 ++- app/src/sections/agent-chat/messages.test.ts | 4 ++-- app/src/sections/agent-chat/messages.ts | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/src/pages/AgentChatPage.test.tsx b/app/src/pages/AgentChatPage.test.tsx index c1a4a7163a7..909f41e6f84 100644 --- a/app/src/pages/AgentChatPage.test.tsx +++ b/app/src/pages/AgentChatPage.test.tsx @@ -1791,7 +1791,8 @@ describe('AgentChatPage service pause', () => { const note = await screen.findByTestId('service-paused'); expect(note).toHaveAttribute('role', 'note'); expect(note).toHaveTextContent(`# ${servicePausedText(resetsAt)}`); - expect(note).toHaveTextContent(/your own limit is not affected$/); + // `/status` is the same for everyone: a user whose own budget is spent too must not read otherwise. + expect(note).not.toHaveTextContent('your own limit'); // The composer stays: the note may go stale at midnight, the server decides each turn. expect(screen.getByLabelText('Message')).toBeInTheDocument(); }); diff --git a/app/src/sections/agent-chat/messages.test.ts b/app/src/sections/agent-chat/messages.test.ts index 8fabc434c64..23fde1718c4 100644 --- a/app/src/sections/agent-chat/messages.test.ts +++ b/app/src/sections/agent-chat/messages.test.ts @@ -15,11 +15,11 @@ describe('the service pause', () => { expect(nextUtcMidnight(Date.parse('2026-12-31T00:00:00Z'))).toBe('2027-01-01T00:00:00.000Z'); }); - it("names the end in the user's own time, and says their own limit is untouched", () => { + it("names the end in the user's own time, and claims nothing about their own limit", () => { const until = '2026-11-03T00:00:00Z'; expect(localClock(until)).toMatch(/^\d{2}:\d{2}/); expect(servicePausedText(until)).toBe( - `the chat has reached today's overall limit and is paused until ${localClock(until)} · your own limit is not affected` + `the chat has reached today's overall limit and is paused until ${localClock(until)}` ); expect(localClock('not a time')).toBeNull(); expect(servicePausedText('not a time')).toContain('paused until 00:00 UTC'); diff --git a/app/src/sections/agent-chat/messages.ts b/app/src/sections/agent-chat/messages.ts index b78488bd154..22389d35743 100644 --- a/app/src/sections/agent-chat/messages.ts +++ b/app/src/sections/agent-chat/messages.ts @@ -66,12 +66,14 @@ export function nextUtcMidnight(now: number = Date.now()): string { } /** - * The whole chat paused for the day (`service_paused`): the service's day budget is - * spent, not the user's own. `until` is the reset in the user's time; the server's - * own refusal says 00:00 UTC, since it does not know the user's time zone. + * The whole chat paused for the day: the service's day budget is spent. `until` is + * the reset in the user's time; the server's own refusal says 00:00 UTC, since it + * does not know the user's time zone. Only that refusal says the user's own limit + * is untouched: `/status` reports the pause for everyone, so a note from it cannot + * know, and a user whose own budget is spent too gets `budget` on the next turn. */ export function servicePausedText(until: string = nextUtcMidnight()): string { - return `the chat has reached today's overall limit and is paused until ${localClock(until) ?? '00:00 UTC'} · your own limit is not affected`; + return `the chat has reached today's overall limit and is paused until ${localClock(until) ?? '00:00 UTC'}`; } /** Failures of the dataset, bindings, confirmation and undo calls. */