Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ Before the first case the harness renders the catalogue file of the first case o
| `--max-attempts N` | Adapter attempts per run, 1 to 3 (`AGENT_MAX_ATTEMPTS`, 2 by default), so 1 measures the first attempt alone and 3 a second repair round; the report's stamp records the value as `max_attempts` |
| `--thinking-budget ROOT,ADAPTER` | Claude only: adaptive thinking for the root and the adapter, each value (0 to 8,192, 0 off) added to that agent's output cap (`AGENT_ROOT_THINKING_BUDGET`, `AGENT_ADAPTER_THINKING_BUDGET`); the stamp records it as `thinking_budget`, the summary names it, and the thinking tokens are priced as output |
| `--budget-usd X` | Stop before a case that could take the estimated list-price cost past X (the spend so far plus the costliest case so far), or once the cost has passed X (exit code 3) |
| `--theme light\|dark` | The site theme every turn sends (`?theme=`), as the chat page does, so `dark` measures dark first renders; without it, no theme is sent and the plots render light, as in every committed baseline. The stamp records it as `theme`; compare a dark run with `--no-baseline` or a dark baseline. The preflight stays light |
| `--out DIR` | Output directory, `agents/evals/reports/` by default (git-ignored) |
| `--baseline FILE`, `--no-baseline` | The report to compare with; by default `evals/baselines/claude-haiku-5-5.json` when it exists |
| `--tolerance F` | The pass-rate drop that counts as a regression, 0.05 (5 percentage points) by default |
Expand Down
45 changes: 30 additions & 15 deletions agents/anyplot/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
`run_pipeline` is the single node of the `plot_pipeline` workflow (`tools/session.py`).
It reads spec, library, dataset and bindings from server-set session state, never from
its input: `PipelineArgs` carries only `change_request`, `base` and `theme` (the one
theme to render; omitted, a change keeps the previous version's theme and a new plot
is light). The session language is read once per run, so the adapter, which writes the
plot's own text in it, and the reviewer, which judges that text against it, share one
value; each prompt gets it fenced as `<plot_language>`. It runs at most
theme to render; omitted, the theme `run_theme` picks, the site's unless the user
asked for one). The session language is read once per run, so the adapter, which
writes the plot's own text in it, and the reviewer, which judges that text against it,
share one value; each prompt gets it fenced as `<plot_language>`. It runs at most
`AGENT_MAX_ATTEMPTS` attempts, two by default (the first and one repair round; the
eval harness may set one or three):

Expand All @@ -29,7 +29,7 @@
literal text the plan replaces, up to `MAX_NEW_LITERAL_CHARS`;
3. **render**: `normalise`, the loader substitution (the runtime's `loader_block`, typed
from the parser's dtypes and the column kinds), then the one
theme the call asks for (`PipelineArgs.theme`, light by default) through the
theme of the run (`run_theme`: the one the user asked for, else the site's) through the
render backend, and the host gates (`render/gates.py`) on exactly that theme; a failed
render's summary is the runtime's error grammar over the code that ran
(`RuntimeAdapter.error_summary` with the run form, so Julia's names the working form's
Expand Down Expand Up @@ -415,6 +415,8 @@ class Run:
view: SessionView
dataset: StoredDataset
theme: Theme = "light"
theme_asked: bool = False
"""The user asked for `theme` (`run_theme`); the shipped version keeps it as `CodeVersion.theme_asked`."""
language: str = DEFAULT_LANGUAGE
"""The session language, read once per run: the adapter writes the plot text in it, the reviewer judges it."""
attempts: int = 0
Expand Down Expand Up @@ -1249,6 +1251,7 @@ def _store_version(ctx: Context, services: Services, run: Run, result: PlotResul
title=shipped.plan.title,
library=run.view.library,
theme=run.theme,
theme_asked=run.theme_asked,
language=run.language,
requests=standing_requests([*run.user_requests, run.change_request]),
themes={run.theme: ThemeRender("needs_attention", PADDED_REASON) if shipped.padded else ThemeRender("ok")},
Expand Down Expand Up @@ -1387,17 +1390,23 @@ def _reraise_unless_schema(exc: Exception) -> None:
logger.warning("sub-agent answer failed its schema: %s", type(exc).__name__)


def _default_theme(services: Services, session_id: str, library: str, base: str) -> Theme:
"""An omitted theme keeps the previous version's theme on a change; a new plot is light.
def run_theme(asked: Theme | None, site: Theme | None, previous: CodeVersion | None) -> tuple[Theme, bool]:
"""The one theme a run renders, and whether the user asked for it (`CodeVersion.theme_asked`).

The root may leave `theme` out of a refinement call, so a dark plot must not
silently come back light.
In order: the theme the root passed (`PipelineArgs.theme`), which the user asked for in the chat; on a
change, the theme of the version it changes (`previous`) when the user asked for that one, because the root
leaves `theme` out of a refinement and a dark plot asked for must not silently come back light; the site
theme the page sent with the turn (`RequestLedger.site_theme`; owner decision of 2026-11-01: a user on the
dark site gets a dark plot, the other theme one click away on the toggle), so a change after a switch of the
site theme follows the switch; without a page (`adk web`, the evals) the version it changes, else light.
"""
if base == "previous":
previous = services.versions.latest_rendered(session_id, library=library)
if previous is not None:
return previous.theme
return "light"
if asked is not None:
return asked, True
if previous is not None and previous.theme_asked:
return previous.theme, True
if site is not None:
return site, False
return (previous.theme if previous is not None else "light"), False


@node(name="run_pipeline", rerun_on_resume=True)
Expand Down Expand Up @@ -1434,11 +1443,17 @@ async def run_pipeline(ctx: Context, node_input: PipelineArgs) -> AsyncGenerator
return
# Taken only once the run has its slot: a run that never starts leaves the user's note for the next one.
node_input = take_pending_request(ctx.state, node_input)
theme = node_input.theme or _default_theme(services, ctx.session.id, view.library, node_input.base)
previous = (
services.versions.latest_rendered(ctx.session.id, library=view.library)
if node_input.base == "previous"
else None
)
theme, theme_asked = run_theme(node_input.theme, ledger.site_theme, previous)
run = Run(
view=view,
dataset=dataset,
theme=theme,
theme_asked=theme_asked,
language=view.language,
change_request=node_input.change_request,
expected_texts=list(node_input.expected_texts),
Expand Down
4 changes: 4 additions & 0 deletions agents/anyplot/plugins/ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
from datetime import UTC, datetime
from typing import Any, Literal

from ..schemas import Theme
from ..settings import AgentSettings


Expand Down Expand Up @@ -158,6 +159,9 @@ class RequestLedger:
"""The operation names of every `prepare_data` call that changed the data (closed: `telemetry.PREP_OPS`)."""
reply_question: bool = False
"""The root's last reply sent to the page ends with a question mark (`stream.Translator`); the text is never kept."""
site_theme: Theme | None = None
"""The theme the page showed the site in when the user sent the turn (`/messages?theme=`), which a plot run takes
when neither the user nor the version it changes asked for one (`pipeline.run_theme`); None without a page."""

def book_step(self, agent: str, *, model: str | None, usage: dict[str, int], cached: int) -> None:
"""Add one model call to the agent's step usage (`usage` as `usage_breakdown` counts it)."""
Expand Down
2 changes: 1 addition & 1 deletion agents/anyplot/prompts/root.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ The session has one language for your replies and for the text in the plot. The
- When the data is not confirmed yet (or roles are still unbound) and this library has no plot yet, a typed message that asks for something about the plot ("Setz den Titel auf ...", "log scale on y") is the first plot's request: call `keep_plot_request` with it as `change_request` (and `expected_texts`), which keeps it without running anything. Its `request_kept` `true` means the next `.create_plot()` draws it: say that, and what the user still has to do (`.confirm()`, or the open roles).
- Only a `request_kept` `true` from `keep_plot_request` or `plot_pipeline` lets you say that a request is noted, kept or will be used. The "Create plot" action runs `plot_pipeline` with no arguments, so a request only you remember never reaches the plot; `request_kept` `false` or no answer means it is not kept: say so plainly.
- `expected_texts`: when the user wrote out a text or a number format the plot must show (a title, a legend entry, an axis title, a unit, a format example such as "6,5 Mio." or "12 %"), list each one, verbatim as the user wrote it (never translated), at most five of at most 120 characters. The server checks the render's drawn texts for each. Leave it out when the user named none; never list a text the user wants removed or replaced, only the new one.
- `theme` is `"light"` or `"dark"`. Leave it out unless the user asks for a theme: a new plot is then light, and a change keeps the theme of the latest version. Set `"dark"` only when the user asks for a dark plot or a dark background, and `"light"` when they ask to go back to light.
- `theme` is `"light"` or `"dark"`. Leave it out unless the user asks for a theme: the plot is then rendered in the theme the user's page shows, and a change keeps a theme the user asked for earlier. Set `"dark"` only when the user asks for a dark plot or a dark background, and `"light"` when they ask for a light one or to go back to light.
- When the user only wants to see a finished plot in the other theme, do not call `plot_pipeline`: tell them to use the light and dark switch on the result, which shows the other theme without changing the plot.
- When a tool answers `not_ready`, tell the user what is missing: a dataset, bindings for the roles it names, or their confirmation of the data (`data_unconfirmed`: the user checks the preview and the bindings in the data panel and presses `.confirm()` there; you cannot confirm for them).

Expand Down
10 changes: 5 additions & 5 deletions agents/anyplot/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,11 +297,11 @@ class Binding(_ServerContract):
class PipelineArgs(_ServerContract):
"""The `plot_pipeline` tool input. Spec, library and dataset come from server state.

`theme` is the one theme the run renders and the reviewer sees. Omitted (None), a
change on `base="previous"` keeps the previous version's theme and a new plot is
light; the root passes `dark` only when the user asks for a dark plot. The other
theme of a finished version is rendered on demand by the theme toggle route,
without any model call.
`theme` is the one theme the run renders and the reviewer sees; the root passes it
only when the user asks for a theme. Omitted (None), the run takes the site theme
the page sent with the turn, unless it changes a version whose theme the user asked
for (`pipeline.run_theme`). The other theme of a finished version is rendered on
demand by the theme toggle route, without any model call.
"""

change_request: str = Field(default="", max_length=MAX_CHANGE_REQUEST_CHARS)
Expand Down
3 changes: 3 additions & 0 deletions agents/anyplot/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ class CodeVersion:
"""The library the version was adapted for; a session can switch library."""
theme: Theme = "light"
"""The theme the run rendered and the reviewer saw."""
theme_asked: bool = False
"""The user asked for `theme` in the chat, for this version or for the one it changed, so a change of it keeps
the theme; otherwise a change follows the site theme of its turn (`pipeline.run_theme`)."""
language: str = "en"
"""The session language the version's plot text was written in."""
requests: list[str] = field(default_factory=list)
Expand Down
7 changes: 4 additions & 3 deletions agents/anyplot/tools/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -716,9 +716,10 @@ def _fenced_lines(lines: list[str]) -> str:
description=(
"Adapt the plot to the user's dataset and bindings, render it in one theme, review it, and repair it "
"once if needed. Call it with no arguments for 'Create plot'; for a change, pass change_request (English, "
"at most 600 characters) and base='previous'. Leave theme out unless the user asks for a theme: a new plot "
"is light and a change keeps the latest version's theme; pass theme='dark' only when the user asks for a "
"dark plot. Pass expected_texts (at most five) with the texts and number formats the user wrote out for "
"at most 600 characters) and base='previous'. Leave theme out unless the user asks for a theme: the plot "
"then follows the theme of the user's page, and a change keeps a theme the user asked for; pass "
"theme='dark' only when the user asks for a dark plot, and theme='light' only when they ask for a light "
"one. Pass expected_texts (at most five) with the texts and number formats the user wrote out for "
"the plot, verbatim. Returns the PlotResult, with what the shipped plot shows under shows."
),
input_schema=PipelineArgs,
Expand Down
15 changes: 13 additions & 2 deletions agents/evals/matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,11 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
type=float,
help="stop before a case that could take the estimated list-price cost past this (spent + costliest case)",
)
parser.add_argument(
"--theme",
choices=("light", "dark"),
help="the site theme every turn sends (`?theme=`), the theme the plots render in (default: none, light)",
)
parser.add_argument("--out", type=Path, default=DEFAULT_OUT, help="output directory (default agents/evals/reports)")
parser.add_argument("--baseline", type=Path, help="baseline report (default: baselines/<pinned model>.json)")
parser.add_argument("--no-baseline", action="store_true", help="do not compare with any baseline")
Expand Down Expand Up @@ -359,6 +364,8 @@ class MatrixConfig:
preflight: bool = True
render_token: "RenderToken | None" = None
keep_attempts: bool = False
theme: str | None = None
"""`--theme`: the site theme every turn sends, as the page does; None sends none, and the plots render light."""
argv: list[str] = field(default_factory=list)


Expand Down Expand Up @@ -1011,6 +1018,7 @@ class CaseRunner:
settings: "AgentSettings"
renders_dir: Path
recorder: AttemptRecorder | None = None
theme: str | None = None

async def run(self, case: "EvalCase", repeat: int) -> dict[str, Any]:
"""One case and repeat, from opening the session to deleting it; never raises for a broken case."""
Expand Down Expand Up @@ -1155,7 +1163,8 @@ async def _turn(
) -> dict[str, Any] | None:
self.timed.chunks.clear()
started = time.monotonic()
response = await self.client.post(f"/v1/sessions/{sid}/messages", headers=headers, json=body)
params = {"theme": self.theme} if self.theme else None
response = await self.client.post(f"/v1/sessions/{sid}/messages", headers=headers, json=body, params=params)
turn = Turn(
response.status_code,
None if response.status_code == 200 else _detail(response),
Expand Down Expand Up @@ -1278,6 +1287,7 @@ def build_stamp(
"tolerance": config.tolerance,
"gallery_seed": config.seed,
"baseline": str(config.baseline_path) if config.baseline_path else None,
"theme": config.theme,
"argv": list(config.argv),
}

Expand Down Expand Up @@ -1395,7 +1405,7 @@ async def _run_matrix(
try:
transport = httpx.ASGITransport(app=timed)
async with httpx.AsyncClient(transport=transport, base_url="http://agents", timeout=None) as client:
runner = CaseRunner(client, timed, collector, settings, config.out / "renders", recorder)
runner = CaseRunner(client, timed, collector, settings, config.out / "renders", recorder, config.theme)
for index, (case, repeat) in enumerate(plan, start=1):
if config.budget_usd is not None and runs and total + costliest > config.budget_usd:
stopped = "budget"
Expand Down Expand Up @@ -1667,6 +1677,7 @@ def _prepare_and_run(args: argparse.Namespace, settings: "AgentSettings", argv:
save_baseline=args.save_baseline,
preflight=not args.no_preflight,
keep_attempts=args.keep_attempts,
theme=args.theme,
render_token=RenderToken(os.environ) if args.gcloud_token else None,
argv=list(sys.argv[1:] if argv is None else argv),
)
Expand Down
Loading
Loading