From 4a66014a3f8bfae17392cc3601d19135b0da39e0 Mon Sep 17 00:00:00 2001 From: Jimmy Yeh Date: Mon, 26 Oct 2026 15:29:20 +0800 Subject: [PATCH 1/7] Let the sidebar hand the agent loop to pi, behind one Harness interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What runs the loop is now a choice, like what runs the model: the built-in harness, or pi, a coding agent that asks a model it has credentials for (Claude, GPT…). The page draws pi's steps in the same diagram, step cards and event list, from the events pi reports, without tokens or a prompt. Harness - harness/harnesses/harness.py: Harness, what the page asks of whatever runs the loop (run, stop, decide), and which events every harness sends and which only one that knows them. - harness/agent.py moves to harness/harnesses/simple_harness.py, and Run becomes SimpleHarness, beside SimpleBackend; run_start says which harness ran the turn ("simple" or "pi"). - LiveRun takes a function that makes the harness (harness_for()). pi - harness/harnesses/pi_harness.py: PiHarness starts pi in RPC mode for each question, in the conversation's folder and on its session, without the user's extensions, skills or context files; it sets the model and the thinking level and turns pi's retries off before the question goes, and turns pi's records into the page's events. One of pi's turns is a step; its streamed text is a new event, delta. pi has no step limit, so the page aborts it past Max steps. - harness/harnesses/pi_approve.ts: a pi extension that asks before every tool call; its dialog becomes the page's Allow and Reject, and a refusal reaches the model in the turn's language. - The sidebar: What runs the loop, then pi's models (pi.probe), its tools, its thinking level, Ask me before running a tool and Max steps. The Context tab says pi shows no prompt; the Model tab shows pi's model, settings and tools, which run on this computer without a sandbox. - render.py draws a run without a family, a context or tokens: the streamed text by channel, the provider's token counts, and pi's own wording for its stops and its diagram. Tests and docs - tests/fake_pi.py speaks pi 1.4's RPC protocol from a script, so the default suite needs neither Node nor a model; test_pi_harness.py covers PiHarness, and with RUN_PI_TESTS=1 runs the real pi against a scripted model on 127.0.0.1 (passes with pi 1.4.0). test_render.py and test_app.py cover pi's runs and the page with pi picked. - How it works gets a section on pi and the delta event; the READMEs, AGENTS.md and the guide's excerpts follow the move and the rename. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 39 +- README.md | 46 +- README.zh-TW.md | 46 +- app/assets/style.css | 3 + app/guide.py | 204 +++++--- app/page.py | 218 ++++++-- app/render.py | 251 +++++++++- harness/__init__.py | 10 +- harness/backends/backend.py | 6 +- harness/families/__init__.py | 2 +- harness/harnesses/__init__.py | 7 + harness/harnesses/harness.py | 55 ++ harness/harnesses/pi_approve.ts | 22 + harness/harnesses/pi_harness.py | 468 ++++++++++++++++++ .../{agent.py => harnesses/simple_harness.py} | 9 +- tests/fake_pi.py | 243 +++++++++ tests/fakes.py | 4 +- tests/guide_excerpts.txt | 175 +++++-- tests/test_app.py | 153 +++++- tests/test_chat_template.py | 8 +- tests/test_guide.py | 18 +- tests/test_pi_harness.py | 408 +++++++++++++++ tests/test_real_model.py | 12 +- tests/test_render.py | 149 +++++- .../{test_agent.py => test_simple_harness.py} | 14 +- tests/test_vllm_backend.py | 16 +- 26 files changed, 2328 insertions(+), 258 deletions(-) create mode 100644 harness/harnesses/__init__.py create mode 100644 harness/harnesses/harness.py create mode 100644 harness/harnesses/pi_approve.ts create mode 100644 harness/harnesses/pi_harness.py rename harness/{agent.py => harnesses/simple_harness.py} (97%) create mode 100644 tests/fake_pi.py create mode 100644 tests/test_pi_harness.py rename tests/{test_agent.py => test_simple_harness.py} (97%) diff --git a/AGENTS.md b/AGENTS.md index f86494a..abc1315 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,8 @@ Qwen3.5 0.8B, through Hugging Face `transformers`, or on vLLM: in Docker, starte server at `VLLM_URL`) and draws every step of its agent loop: context, model, harness, environment, stop, down to each token's probability. A question can carry images, sound and video for the model, and, when the sidebar lets the model write code, data files for a `run_python` -tool that runs in a Docker sandbox. It is a teaching project for the +tool that runs in a Docker sandbox. The sidebar can also hand the loop to pi, a coding agent that asks a +model it has credentials for, and draw what pi reports doing, without tokens. It is a teaching project for the Agent Atlas course. The page and its 運作說明 (How it works) guide are in Traditional Chinese and English. README.md (English) and README.zh-TW.md (Traditional Chinese) are the full account of what the page shows, the architecture, the tests and the known limits; read the parts you are about to change. @@ -22,6 +23,7 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py # the mod VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # vllm_backend.py against that real server docker compose -f docker-compose/compose.yaml up -d # the vLLM server the sidebar's vLLM (Docker) starts, by hand (+ -f docker-compose/compose.gpu.yaml on NVIDIA GPUs) RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # starts that server with Qwen3.5 and asks it through VLLMBackend, ~2 min +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # the real pi (on the PATH, or PI_COMMAND; pi 1.4 needs Node 22.19+) against a scripted model on 127.0.0.1 uv run python -m app.guide _site # the guide: _site/zh/, _site/en/ and a redirecting _site/index.html UPDATE_GUIDE_SNAPSHOT=1 uv run pytest tests/test_guide.py # after rereading the guide sections a code change touched ``` @@ -42,9 +44,13 @@ app/ the Streamlit app guide.py the 運作說明 page: content in both languages, code references by text, static HTML for GitHub Pages assets/ style.css (palette and app), dark.css, guide.css, guide.js (the guide's one inlined script), logo.svg, examples/ (files the examples attach, also offered above the input box) harness/ the agent, the model and what runs it; no Streamlit - agent.py the harness (Run): prompt → generate → parse → validate → (ask) → run tool → back into context → stop? + harnesses/ + harness.py Harness: what the page asks of whatever runs the loop (run, stop, decide), and the events every harness sends + simple_harness.py this project's harness (SimpleHarness, RunSettings): prompt → generate → parse → validate → (ask) → run tool → back into context → stop? + pi_harness.py PiHarness: pi (PI_COMMAND) in RPC mode for each question, its JSON records turned into the page's events; probe() lists its models + pi_approve.ts the pi extension behind Ask me before running a tool: a confirm dialog that pi_harness.py answers on stdin backends/ - backend.py Backend: what agent.py and the page ask of whatever runs the model (SimpleBackend, VLLMBackend, tests' FakeLLM); Sampling, Encoded, the token event's fields + backend.py Backend: what SimpleHarness and the page ask of whatever runs the model (SimpleBackend, VLLMBackend, tests' FakeLLM); Sampling, Encoded, the token event's fields simple_backend.py SimpleBackend: the hand-written decoding loop (KV cache, temperature / top-k / top-p, MTP draft and verify) vllm_backend.py VLLMBackend (VLLM_URL): the tokenizer and chat template here, the token ids to a vLLM server (/v1/completions, or /inference/v1/generate with --enable-scale-out: media, sampling masks), what it says about each token back vllm_docker.py the sidebar's vLLM (Docker): downloads the model, `docker compose up -d` with it, waits until it answers, `docker compose down`; DockerError in both languages @@ -70,14 +76,17 @@ import Backend`, `from app import render`); pytest puts that folder on the path rerun, so everything a rerun does is inside functions, and what the module reads from the environment (`VLLM_URL`, `MODEL_ID`, `AGENT_LOOP_UPLOADS`) is read once per process. Tests change those on the module (`page.VLLM_URL`, `page.UPLOADS`, `page.SimpleBackend`), not in the environment. -- **Events are the interface.** `agent.Run` runs in a background thread and appends one event per +- **Events are the interface.** The page runs a turn only through a `Harness` (harnesses/harness.py): `SimpleHarness`, or + `PiHarness` with pi picked under What runs the loop. It runs in a background thread and appends one event per stage; the order is in the READMEs' Architecture (架構) section. `app/render.py` is a pure function of that list and the page only draws it, which is why stop and approve can rerun the script without interrupting the loop. A new or changed event needs render.py, the `EVENTS` table in app/guide.py (tests check that it lists - every event the harness sends and points at the line that sends it) and the event list in both READMEs. + every event either harness sends and points at the line that sends it) and the event list in both READMEs. + Most events are optional (`harness.SOMETIMES`): pi sends no `context`, `prefill`, `token` or family but `delta`, and + render.py draws a step from what it has, never filling in what the harness did not see. The harness and the page reach the model only through `backend.Backend`; a token event's probabilities, rank, candidates and MTP draft (`backend.TOKEN_DETAILS`) are optional, and render.py leaves out what is missing. -- **Nothing model-specific outside `harness/families/`.** agent.py, simple_backend.py and render.py have no "if Gemma". +- **Nothing model-specific outside `harness/families/`.** simple_harness.py, simple_backend.py and render.py have no "if Gemma". Parsing output, the tokens that switch to thinking / tool call / reply, stop tokens, special-token explanations, notes on the files of its checkpoints and recommended sampling all live on the `Family`. Another checkpoint of a supported family is one line in `MODELS` and one in `REVISIONS`, @@ -94,8 +103,8 @@ import Backend`, `from app import render`); pytest puts that folder on the path something else: a processed --logprobs-mode is the sampling distribution, never the model's p. - **Escape everything the model writes** before it reaches `st.html`. tests/test_render.py checks it, and also that every prefix of a run's events renders, because the page redraws mid-run. -- **app/guide.py finds code by text, not line number.** `Code("harness/agent.py", "def run(self):")` and inline - `[[harness/agent.py|def run(self):]]` are looked up when the page is built, by their path from the project's folder, +- **app/guide.py finds code by text, not line number.** `Code("harness/harnesses/simple_harness.py", "def run(self):")` and inline + `[[harness/harnesses/simple_harness.py|def run(self):]]` are looked up when the page is built, by their path from the project's folder, and each must match exactly one line. Before renaming, moving or rewriting a line, grep app/guide.py for it; if a reference breaks, tests/test_guide.py fails. Fix the reference, and when the behavior changed, the prose around it too. A file that is not Python (docker-compose/compose.yaml) is quoted line by line, through the line with `end`. Where a part of a section differs by what @@ -120,7 +129,8 @@ import Backend`, `from app import render`); pytest puts that folder on the path when the two languages drift apart (different refs, links or tokens). - **The model reads harness and tool errors in the run's language.** `RunSettings.lang` is the page's language when the turn started; `Tool.validate`, `family.parse` and the harness's own messages word - errors in it, and `ToolError` / `ParseError` carry both texts. The system prompts in agent.py are the + errors in it, and `ToolError` / `ParseError` carry both texts (with pi, the refusal pi_approve.ts gives the model, through + `AGENT_LOOP_REFUSAL`). The system prompts in simple_harness.py are the same for both languages, so the model's behavior does not depend on the page's language. - **The language switch.** The app starts in the language in the `agent_loop_lang` cookie, else the browser's (`st.context.locale` through `i18n.from_locale`); picking one writes the cookie with an @@ -153,12 +163,21 @@ import Backend`, `from app import render`); pytest puts that folder on the path image build the new one, and keep compose.gpu.yaml's command the same as compose.yaml's but `--skip-mm-profiling`, plus its two flags on GPU memory and two on speculative decoding (tests/test_vllm_docker.py checks it). The app stops the container it started when it quits (`_stop_at_exit`), so a test that fakes a start leaves `vllm_docker._chosen` empty. +- **pi only when asked for, and as itself.** The page starts pi only once What runs the loop is pi, and then never + `docker`. It starts pi without the user's extensions, skills, prompt templates and context files (`pi_harness.FLAGS`), + so the page draws pi's own loop; the credentials are pi's. Each command waits for pi's response before the next (pi + handles them concurrently: a prompt sent beside `set_model` goes to the old model), auto-retry stays off (a retry is + a turn of its own), and Max steps is kept by aborting pi at the turn past it. pi's tools run unsandboxed in the + conversation's folder; the page says so wherever they are offered. Read pi's docs (docs/rpc.md, json.md, + rpc-extension-ui.md in its package) before changing what is sent or read. ## Tests - Most tests use `FakeLLM` from tests/fakes.py: same interface as `SimpleBackend`, prompts in a simplified form of each family's chat format, scripted replies streamed token by token. Cover new harness or UI - behavior with it; tests/test_app.py drives the whole page through Streamlit's `AppTest`. + behavior with it; tests/test_app.py drives the whole page through Streamlit's `AppTest`. In pi's place they start + tests/fake_pi.py (`FakePi`), which speaks pi 1.4's RPC protocol from a script; tests/test_pi_harness.py runs the real + pi only with `RUN_PI_TESTS=1`, against a scripted OpenAI-compatible server on 127.0.0.1. - tests/test_chat_template.py checks each family against the model's real tokenizer and chat template and skips models not yet downloaded into models/. tests/test_real_model.py loads the real weights and runs only with `RUN_MODEL_TESTS=1`. After changing parsing, prompts, stop tokens or the diff --git a/README.md b/README.md index 5047401..7393ec6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ English | [繁體中文](README.zh-TW.md) This project is for learning: it shows how an agent loop works underneath, in a way more intuitive and visual than reading about it. Behind an agent's chat box runs a loop: step after step, the model proposes an action and the harness carries it out, until it stops. A chat app shows only the question and the answer; this page draws every step as it happens, down to each token the model writes, so the whole mechanism is laid out in front of you. Each part of the page matches a section of the Unit 01 reading of [Agent Atlas](https://cyyeh.github.io/agent-atlas-learning/) (in Traditional Chinese), the course this project belongs to, and [Experiments to try](#experiments-to-try) turns those ideas into things you can try yourself. - Models: pick [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) or [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B) in the sidebar; they run on your machine with Hugging Face `transformers`, or on [vLLM](https://docs.vllm.ai/): pick vLLM (Docker) under What runs the model in the sidebar and the page starts a vLLM server in Docker for you, on the GPU if there is an NVIDIA one and on the CPU otherwise, or set `VLLM_URL` to use a server of your own, on this machine or on a Linux machine with a GPU ([Running the model on a vLLM server](#running-the-model-on-a-vllm-server)) +- The harness: the loop is this project's own (`harness/harnesses/simple_harness.py`), or, picked under What runs the loop in the sidebar, that of [pi](https://pi.dev), a coding agent that asks Claude, GPT or another model it has credentials for; the page then draws pi's steps from the events pi reports, without tokens ([pi as the harness](#pi-as-the-harness)) - MTP speculative decoding: one forward pass produces several tokens. With `transformers` you can switch it on in the sidebar for Gemma 4 E2B (it is off by default), with [google/gemma-4-E2B-it-assistant](https://huggingface.co/google/gemma-4-E2B-it-assistant) as the drafter; on vLLM with a GPU both models decode speculatively, Gemma 4 E2B with that drafter and Qwen3.5 with the MTP layer in its own checkpoint, and vLLM in Docker does so by default - Files: attach images and videos (both models) and sound (Gemma 4 E2B, which also takes a recording from the microphone) with the + in the input box, or pick one of the sample files above it (a receipt, a recording, a video, sales data) and ask your own question about it; each becomes a run of tokens in the context, and the page shows them - Writing code (optional): Let the model write code in the sidebar gives the model a `run_python` tool, which runs its Python in a Docker sandbox, reads the data files you attach (CSV, TSV, JSON, text, Markdown, Excel) and draws charts with matplotlib, which appear in the chat. Docker is needed only for this and for vLLM (Docker) @@ -30,7 +31,7 @@ On the left is a chat like ChatGPT's; on the right the agent loop is drawn step | Environment | What the tool really returned and how long it took; for `run_python`, the code is written out above and what it printed, or its traceback, below | §2 The environment and observable state | | Stop | Why it stopped and at which token. Gemma stops at `` after an answer (like end_turn) and at `<\|tool_response>` when it wants a tool (like tool_use); Qwen stops at `<\|im_end\|>` either way. Then there are the step limit, the token limit and the user pressing stop | §5 When to stop | | Model tab | The parameter breakdown, the number and kinds of layers, attention heads, the context limit, what the model reads (text, images, sound, video), the MTP drafter, what each decoding setting does, each step's prefill time and tok/s (with MTP, also how many guesses were kept and how many decoding forward passes ran), the id and purpose of every special token of the tokenizer (24 for Gemma, 33 for Qwen), and every file in the model's folder in `models/`: its size, what it is for, what it holds for that model and the values of its key fields | §1, §7 An agent is a system | -| How it works | The 18 stages of one turn (including how an image becomes tokens, how MTP guesses and checks, and what can run the model), each with the code that does it and its line numbers, which open those lines on GitHub, in English and in Chinese. It is on GitHub Pages; the line numbers are looked up in the source when the page is built, the links go to that commit, and it is rebuilt on every push to main, so it follows the code | All of it | +| How it works | The 19 stages of one turn (including how an image becomes tokens, how MTP guesses and checks, what can run the model, and pi in place of the harness), each with the code that does it and its line numbers, which open those lines on GitHub, in English and in Chinese. It is on GitHub Pages; the line numbers are looked up in the source when the page is built, the links go to that commit, and it is rebuilt on every push to main, so it follows the code | All of it | ## Quick start @@ -43,6 +44,7 @@ You need: - [uv](https://docs.astral.sh/uv/) 0.13.4 or newer (it installs Python 3.12 for you). Older versions still trust eMudhra's compromised emSign certificates ([the announcement](https://discuss.python.org/t/109576)), so `uv sync` refuses to run with them; `uv self update` (or `brew upgrade uv`) updates it - A network connection: to download the models the first time, and for the Wikipedia tools - Optional: [Docker](https://docs.docker.com/get-started/get-docker/) (Docker Desktop, OrbStack or Colima), only to let the model write code or to run it on vLLM (Docker) +- Optional: [pi](https://pi.dev) and Node.js 22.19 or newer, only to hand the loop to pi ([pi as the harness](#pi-as-the-harness)) ```bash uv sync @@ -91,7 +93,11 @@ app/ what a person sees, built on the harness guide.py the How it works page: its content in both languages, line lookups, and the static HTML for GitHub Pages assets/ the page's styles and the guide's script; logo.svg; examples/, the files the examples attach and the sample files above the input box harness/ what runs a turn; it never imports Streamlit - agent.py the loop: build the prompt → generate → parse → validate → (ask you) → run the tool → back into the context → stop? + harnesses/ what runs the loop + harness.py the Harness the page runs a turn through: run, stop, decide, and the events every harness sends + simple_harness.py this project's loop: build the prompt → generate → parse → validate → (ask you) → run the tool → back into the context → stop? + pi_harness.py pi as the harness: pi in RPC mode, its JSON records turned into the page's events + pi_approve.ts the pi extension through which the page asks you before pi runs a tool backends/ what runs the model backend.py the Backend that the harness and the page run the model through: render, encode, generate, and the fields of a token event simple_backend.py local inference with a hand-written decoding loop; MTP's drafting and checking are here too @@ -113,11 +119,11 @@ tests/ pytest tests; see Tests below models/ the downloaded models (git ignores it; AGENT_LOOP_MODELS moves it) ``` -The agent runs in a background thread and appends an event at every stage; the UI only draws the event list. So when you press Stop or Allow and Streamlit reruns the script, the loop in the background is not interrupted. +The agent runs in a background thread and appends an event at every stage; the UI only draws the event list. So when you press Stop or Allow and Streamlit reruns the script, the loop in the background is not interrupted. The page reaches the loop only through a `Harness` (`harnesses/harness.py`): `run()`, `stop()`, `decide()` and the events it sends. `SimpleHarness` is this project's own, and `PiHarness` hands the loop to pi ([pi as the harness](#pi-as-the-harness)). How it works is a static page: `uv run python -m app.guide _site` writes `_site/en/index.html` and `_site/zh/index.html` (styles and a small script inlined, so they open straight in a browser), and `_site/index.html`, which sends a reader to the language they last picked on the page's switch, or else their browser's. On a push to main, `.github/workflows/pages.yml` runs `tests/test_guide.py`, then does the same and publishes it to GitHub Pages; if the tests fail, the page already up stays as it was. -The events of one run, in order: `run_start`, then for each step `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`, with a tool call also `tool_call` → (`approval_request` → `approval`) → (`tool_start`, when the tool runs) → `tool_result`, and finally `run_end`, after an `error` if something failed: its `message` is the exception's, and a backend's error (a vLLM server's, say) also comes in both languages (`text`), so the page shows the one it is in. `context` carries the prompt as the model reads it, with each attachment expanded into its tokens, and how many tokens each kind of media took (`media_tokens`). `token` carries the token and, when the backend knows them, its probability, its rank, the top 5 candidates and what MTP's drafter guessed (`backend.TOKEN_DETAILS`): `simple_backend.py` knows them all, and the page leaves out whatever a backend does not report rather than showing a number nobody measured. A backend that learns which tokens were MTP's guesses only when the step ends (a vLLM server) says so in `model_end`'s MTP counts (`kept_at`, `turned_down_at`), and the page marks those tokens then. `tool_result` carries the observation the model reads and, apart from it, the images the tool made for you (`images`: the paths of `run_python`'s charts), which the model never reads. +The events of one run, in order: `run_start`, then for each step `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`, with a tool call also `tool_call` → (`approval_request` → `approval`) → (`tool_start`, when the tool runs) → `tool_result`, and finally `run_end`, after an `error` if something failed: its `message` is the exception's, and a backend's error (a vLLM server's, say) also comes in both languages (`text`), so the page shows the one it is in. `context` carries the prompt as the model reads it, with each attachment expanded into its tokens, and how many tokens each kind of media took (`media_tokens`). `token` carries the token and, when the backend knows them, its probability, its rank, the top 5 candidates and what MTP's drafter guessed (`backend.TOKEN_DETAILS`): `simple_backend.py` knows them all, and the page leaves out whatever a backend does not report rather than showing a number nobody measured. A backend that learns which tokens were MTP's guesses only when the step ends (a vLLM server) says so in `model_end`'s MTP counts (`kept_at`, `turned_down_at`), and the page marks those tokens then. `tool_result` carries the observation the model reads and, apart from it, the images the tool made for you (`images`: the paths of `run_python`'s charts), which the model never reads. `run_start` says which harness ran the turn (`harness`: `simple` or `pi`). A harness that does not run the model itself, pi, sends no `context`, `prefill` or `token`, and no family: instead it sends `delta`, a piece of the model's output and its channel (thinking, reply, tool call) as it streams, and its `model_end` carries the token counts the provider reported (`usage`). ### Two languages @@ -234,9 +240,26 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py - **On a DGX Spark** (NVIDIA GB10, whose GPU shares the machine's 128 GB): on the GPU, Qwen3.5 0.8B starts in about 2 minutes and decodes about 113 tok/s, Gemma 4 E2B starts in about 3 minutes and decodes about 41 tok/s, and the real-server tests pass for both, every kind of media Gemma reads included. At vLLM's default `gpu_memory_utilization` (0.92) the server refuses to start there: vLLM takes the machine's available memory for the GPU's free memory, about 108 of 122 GiB. On the same machine's CPU, Qwen3.5 0.8B starts in about 1.3 minutes and decodes about 23 tok/s. - **On a Mac** (Apple M5, 10 cores given to Docker): Qwen3.5 0.8B starts in about 1.5 minutes, takes about 9 GB of Docker's memory and decodes about 10 tok/s. Gemma 4 E2B needs more memory than Docker Desktop's default: with 16 GB it is killed while loading (the page says so and how to give Docker more), with 24 GB it starts in 3 to 5 minutes and decodes about 4.6 tok/s, but checking at load which media the server takes runs its image, audio and video encoders on the CPU and takes about 10 minutes the first time. +### pi as the harness + +The page can also hand the loop to [pi](https://pi.dev), a coding agent with a loop, a system prompt and tools of its own (read, bash, edit, write, grep, find, ls). Pick pi under What runs the loop at the top of the sidebar: no model loads here; pi asks a model it has credentials for (Claude, GPT, Gemini, or any compatible endpoint in its `models.json`), and the page draws what pi says it does, in the same diagram, step cards and event list. What it cannot show is what only a harness that runs the model sees: an API sends back text, not tokens, so there are no token probabilities, candidates or special tokens, and pi does not say which prompt it sends, so a step has no context section and the Context tab stays empty. A step shows the text as it streams, the token counts the provider reported (with how many came from the cache) and the cost at pi's prices. + +```bash +npm install -g @earendil-works/pi-coding-agent # pi 1.4 needs Node.js 22.19 or newer +pi # then /login, once; or set an API key such as ANTHROPIC_API_KEY before starting the app +PI_COMMAND=/path/to/pi uv run streamlit run streamlit_app.py # if pi is not on the PATH (any command works, npx included) +``` + +- **How the page runs it.** `harness/harnesses/pi_harness.py` starts pi for each question in RPC mode (`pi --mode rpc`): commands go to its stdin and records come from its stdout, one JSON object per line. It picks the model and the thinking level, turns pi's own retries off (a retry would be another step, so a failed request is shown as an error instead) and sends the question, each command only once pi has answered the last; one of pi's turns (one request to the model, and the tools its reply calls) is one step. pi keeps the conversation in a session file in the conversation's folder, so the next question continues it; New chat removes it. +- **What it starts pi without.** The user's own extensions, skills, prompt templates and context files (AGENTS.md, CLAUDE.md) stay out (`--no-extensions` and the like), so the page draws pi's own loop, the same on every machine; the models and the credentials are still the user's, in `~/.pi/agent`. +- **Tools.** The sidebar picks which of pi's tools the model sees (read, bash, edit and write by default, as pi offers). They run on this computer, in the conversation's folder, without a sandbox: unlike `run_python`, bash reaches whatever you can. A file you attach is saved in that folder and named in the question; an image also goes with the question to a model that reads images. +- **Ask me before running a tool.** pi does not ask by itself; with this on, the page has pi load `harness/harnesses/pi_approve.ts`, a small extension that opens a confirm dialog before every tool call. In RPC mode the dialog comes out on stdout, the page shows it as Allow and Reject, and the answer goes back on stdin; a rejected call does not run, and the model reads the refusal in the turn's language. +- **Stopping.** Stop sends pi `abort`. pi has no step limit of its own, so the page keeps Max steps: when pi starts a turn past it, the page aborts pi. An unanswered confirmation stops pi after 5 minutes, as with the built-in harness. +- **Tested** with pi 1.4.0, through a stand-in that speaks its protocol from a script (`tests/fake_pi.py`) and, with `RUN_PI_TESTS=1`, the real pi pointed at a scripted model on 127.0.0.1. + ### Supporting more models -The harness (`agent.py`), the decoding loop (`simple_backend.py`) and the page (`render.py`) are written once, without a single "if Gemma" branch. Whatever differs between models is gathered in one `Family` object in `families/`: +The harness (`harnesses/simple_harness.py`), the decoding loop (`simple_backend.py`) and the page (`render.py`) are written once, without a single "if Gemma" branch. Whatever differs between models is gathered in one `Family` object in `families/`: - Reading the output: `parse` (splits the output into thinking, tool calls and the reply), `thought_prefix`, `channel_switch` (which tokens switch to thinking, a tool call or the reply), `stop_tokens` - Writing the conversation back: `reasoning_key` (the field the thinking is kept in, so the chat template can read it), `text_with_calls` (whether text in front of a tool call is kept) @@ -281,9 +304,10 @@ uv run pytest # about 40 s; loads no we RUN_MODEL_TESTS=1 uv run pytest tests/test_real_model.py # loads the real Gemma (with its MTP drafter) and Qwen, about 4 minutes; add -k Qwen for one VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # asks a running vLLM server RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # starts vLLM in Docker with Qwen3.5 and asks it +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # starts the real pi (on the PATH, or PI_COMMAND) against a scripted model on 127.0.0.1 ``` -Most tests use the fake model in `tests/fakes.py`: it replays scripted output one token at a time and is a `Backend` (`backend.py`) like `SimpleBackend`, so the harness and the page run from start to finish. +Most tests use the fake model in `tests/fakes.py`: it replays scripted output one token at a time and is a `Backend` (`backend.py`) like `SimpleBackend`, so the harness and the page run from start to finish. In pi's place they start `tests/fake_pi.py`, which speaks pi 1.4's RPC protocol from a script, so they need neither Node nor a model. Every file format the page accepts has a test: `tests/samples.py` makes a small file of each one from the examples' files (the receipt in every image format, the recording in every audio format, the rolling ball in every video container, the sales data in every data format). `test_media.py` fails if `media.py` accepts a format without a sample, and `test_real_model.py` asks both models about each one. The tests that need Docker run only when Docker and the sandbox image are there. @@ -296,9 +320,10 @@ How it works quotes code, and `tests/guide_excerpts.txt` keeps everything it quo | `test_families.py` | Every family explains the tokens it uses, in both languages and for the same tokens; `model_type` finds the family; a token shows as the text it stands for, and a byte that is only part of a character as `<0xF0>`; every family explains the files of its checkpoints in both languages, naming the same fields and tokens; every model and drafter downloads at a pinned commit; vLLM drafts for a listed model with the MTP layer in its checkpoint or the drafter transformers uses | | `test_tools.py` | Schema validation and defaults, the calculator's AST allowlist (rejecting `__import__` and the like), timezones, Wikipedia (with fake HTTP); `run_python`'s locked-down `docker run`, its errors (a traceback, no output, too much memory, too long, no image) and what is missing when Docker is not ready or the image is out of date, in both languages; the charts after the marker saved without overwriting a file, only PNGs, at most four, none from code that failed; the image's version label and its pip installed by hash; with Docker, the real sandbox reads `/data` but cannot change it or reach the network, reads every data format, sends back its charts with Chinese labels drawn, and has a pip (with its certifi) past the emSign fix | | `test_simple_backend.py` | `SimpleBackend` has every method and attribute of a `Backend`, and its tokens carry every field and detail; the hand-written decoding loop: KV cache, stop tokens, the token limit, interruption, channel switches, Chinese characters split across tokens; `encode()` handing the processor each kind of media in order with the family's settings, and a short video all its frames; images and videos through the vision encoder before the prefill, and no position offset left from the last prompt; greedy taking the lower id of two tied tokens, as `generate()` does; the candidates and probabilities of temperature / top-k / top-p; MTP with a scripted fake drafter: kept and rejected guesses, the extra token when all hold, how far the KV cache is cut back, the drafter never seeing dropped guesses, and a check that guesses do not change the distribution when sampling; models downloading at their pinned revision into `models/`, which `AGENT_LOOP_MODELS` moves (a relative one from where the app started) and `HF_HUB_CACHE` does not, letting a model go on CUDA freeing cuBLAS's workspaces before emptying the cache, and a model folder's files, their sizes (through the cache's links) and the fields its notes name | -| `test_agent.py` | The fake model is a `Backend` whose tokens carry what `SimpleBackend`'s do; the harness: event order, tool results back in the context, validation errors, repeated calls, truncated results, every stop reason, Allow / Reject / Stop while waiting; the model reads errors in the turn's language, and the system prompt is the same in both; attachments as parts of the question and a note for `run_python`, the media of every turn going to the processor, the expanded context still marking only the tool result as new, `run_python` getting the conversation's folder, and a turn without `run_python`; the model reading where the charts are while the page gets the images | -| `test_render.py` | Every prefix of a run's events renders in both languages (the page redraws every 0.5 s mid-run), HTML tags pair up, the loop diagram is valid SVG, model output is always escaped, special token popovers and lists; MTP's underlines, popovers, counts and drafter card; a run of one special token as one chip with its count, a cut snippet never splitting a token, how many tokens the media took, code and output over several lines written out, tools left out of a turn, the charts of a run and of each step; the model folder card: the files in order with their sizes and notes, a file the family notes but the checkpoint lacks, the drafter's folder, escaping; tokens from a backend that reports no probabilities, or only log probabilities, drawn with what they carry and no more; no Chinese left on an English page | -| `test_app.py` | Drives the whole page with Streamlit's `AppTest`: Streamlit not collecting garbage after every script run, and a rerun waiting for the run in progress; sending a question, the example buttons, sidebar settings reaching the model, multi-turn chat, the Context and Model tabs, Allow / Reject / Stop, a new chat, resetting the settings, two tabs never running at once, switching models, How it works linking to GitHub Pages in the page's language; MTP off by default, the drafter loaded only when it is switched on, and switched off and disabled for Qwen, which has none; an English browser gets English, the last pick beats the browser's language, switching languages keeps the conversation and the settings, and the model's errors follow the language; each page shows only its own language's examples, an example attaches its file and shows only when the model reads that kind, a sample file picked above the input box goes with the next question typed and is offered only when the model can use it, Let the model write code looks for Docker only once it is on and says what is missing, the sales example runs Python on the attached file, a chart shows in the chat and in the step that drew it, and New chat removes the conversation's files; with `VLLM_URL`, the menu lists what the server serves, the sidebar and the Model tab name the server, MTP is the server's, a server with sampling masks gets no greedy decoding or unlimited top-k from the sidebar, the model reads the media the server takes and the page says which it refused and why, the menu leaves out the LoRA adapters the server serves, marks the models of a family the page does not support and starts on the first it does, and picking one of those says why and offers a GitHub issue filled in in the page's language, naming the model (a folder by its name alone) and never the server's address, an error the server sends mid-run is shown in the page's language, and an unreachable server, at load or later, is said in the page's language; without `VLLM_URL`, What runs the model starts on transformers and leaves Docker alone, picking vLLM (Docker) lets go of the local model (nothing keeps it) and starts the server with it, a rerun starts nothing, another model starts the server with that one, picking transformers again stops the container, a new tab or a reload while vLLM runs joins its server with its model and a tab on transformers follows another that picks vLLM and back (the model loaded here once), on a GPU vLLM starts with MTP on and the switch starts it again without (and back, with the reset button), on a CPU it runs without MTP and the switch says why, a tab follows the MTP another tab picked, and a start that failed is said in the page's language, tried again only on Try again, with the switch back to transformers still there | +| `test_simple_harness.py` | The fake model is a `Backend` whose tokens carry what `SimpleBackend`'s do; the harness: event order, tool results back in the context, validation errors, repeated calls, truncated results, every stop reason, Allow / Reject / Stop while waiting; the model reads errors in the turn's language, and the system prompt is the same in both; attachments as parts of the question and a note for `run_python`, the media of every turn going to the processor, the expanded context still marking only the tool result as new, `run_python` getting the conversation's folder, and a turn without `run_python`; the model reading where the charts are while the page gets the images | +| `test_pi_harness.py` | pi as the harness, with `tests/fake_pi.py` in pi's place: both harnesses are a `Harness`, and pi sends only the events it names; a turn becomes the page's events in the loop's order (the thinking, text and tool-call arguments streamed by channel, the provider's token counts with the cache, the calls and their results); pi started in the conversation's folder on its session, with the sidebar's tools and without the user's extensions, the model set and its retries off before the question goes, no tools at all, attachments named for the model and images sent inline, a repeated call marked, a tool that fails, the token limit, the next question on the same session; asking first through the extension: Allow, Reject with the refusal in the turn's language, Stop while a call waits, a timeout; Stop mid-reply and before pi has started, and the step limit drawing no step past it; a failed request, a refused question, a model pi does not have, pi dying as it starts and pi not installed, said in both languages; which models pi has; with `RUN_PI_TESTS=1`, the real pi against a scripted model on 127.0.0.1, with and without asking first | +| `test_render.py` | Every prefix of a run's events renders in both languages (the page redraws every 0.5 s mid-run), HTML tags pair up, the loop diagram is valid SVG, model output is always escaped, special token popovers and lists; MTP's underlines, popovers, counts and drafter card; a run of one special token as one chip with its count, a cut snippet never splitting a token, how many tokens the media took, code and output over several lines written out, tools left out of a turn, the charts of a run and of each step; the model folder card: the files in order with their sizes and notes, a file the family notes but the checkpoint lacks, the drafter's folder, escaping; tokens from a backend that reports no probabilities, or only log probabilities, drawn with what they carry and no more; a run pi ran: the text it streamed by channel and no token numbers, escaped, joined without changing the events, the diagram, the stop card and the step table saying pi runs the loop, and pi's cards in both languages; no Chinese left on an English page | +| `test_app.py` | Drives the whole page with Streamlit's `AppTest`: Streamlit not collecting garbage after every script run, and a rerun waiting for the run in progress; sending a question, the example buttons, sidebar settings reaching the model, multi-turn chat, the Context and Model tabs, Allow / Reject / Stop, a new chat, resetting the settings, two tabs never running at once, switching models, How it works linking to GitHub Pages in the page's language; MTP off by default, the drafter loaded only when it is switched on, and switched off and disabled for Qwen, which has none; an English browser gets English, the last pick beats the browser's language, switching languages keeps the conversation and the settings, and the model's errors follow the language; each page shows only its own language's examples, an example attaches its file and shows only when the model reads that kind, a sample file picked above the input box goes with the next question typed and is offered only when the model can use it, Let the model write code looks for Docker only once it is on and says what is missing, the sales example runs Python on the attached file, a chart shows in the chat and in the step that drew it, and New chat removes the conversation's files; with `VLLM_URL`, the menu lists what the server serves, the sidebar and the Model tab name the server, MTP is the server's, a server with sampling masks gets no greedy decoding or unlimited top-k from the sidebar, the model reads the media the server takes and the page says which it refused and why, the menu leaves out the LoRA adapters the server serves, marks the models of a family the page does not support and starts on the first it does, and picking one of those says why and offers a GitHub issue filled in in the page's language, naming the model (a folder by its name alone) and never the server's address, an error the server sends mid-run is shown in the page's language, and an unreachable server, at load or later, is said in the page's language; without `VLLM_URL`, What runs the model starts on transformers and leaves Docker alone, picking vLLM (Docker) lets go of the local model (nothing keeps it) and starts the server with it, a rerun starts nothing, another model starts the server with that one, picking transformers again stops the container, a new tab or a reload while vLLM runs joins its server with its model and a tab on transformers follows another that picks vLLM and back (the model loaded here once), on a GPU vLLM starts with MTP on and the switch starts it again without (and back, with the reset button), on a CPU it runs without MTP and the switch says why, a tab follows the MTP another tab picked, and a start that failed is said in the page's language, tried again only on Try again, with the switch back to transformers still there; with pi picked under What runs the loop, its models and settings in the sidebar and no decoding settings, a question drawn as pi's loop, run in the conversation's folder, the sidebar's settings reaching pi, Allow through the page, Stop, the Context tab saying pi shows no prompt and the Model tab showing pi's model and tools, pi without a model or not installed said with a Try again, Docker never looked for, the built-in harness's model and settings kept across a switch, a switch starting a new conversation, and an English page with no Chinese | | `test_chat_template.py` | With each model's real tokenizer and chat template (only when downloaded into `models/`): every special token is a single token and has a note, every token's popover shows the text it stands for, tool calls the template writes into the history parse back; a harness run with the real template checks that what the step card marks as new is exactly the tool result (two arguments, thinking mode, text before a tool call), and with the real processor, with an image and a video in the question too; every file in each model's folder (and its drafter's) has a note, and every field a note names is in the file | | `test_vllm_backend.py` | The vLLM backend against a fake server that speaks the streams of vLLM 0.32: `/v1/completions` gets the rendered prompt's token ids, every sidebar setting with every other sampling setting neutral, the stop tokens and a `max_tokens` that fits the server's context; each token's probability and the top 5, the rank only within them; with `--enable-scale-out`, `/inference/v1/generate` and every token's rank, the top 5 by rank though the chosen token comes first; on either route a token tied for first ranked #1, though vLLM's rank counts the tie; with sampling masks, which vLLM sends only with processed log probabilities, the candidates left and the sampling probability but not the model's own probabilities, and temperature 0 and top-k 0 refused on every route, as vLLM refuses them, which the page finds out at load on a server with masks but without `--enable-scale-out` too, while a server that is not ready is no such refusal; the server's `--logprobs-mode` worked out at load with one token, processed log probabilities without masks showing how many were left when some of the top were cut, logits showing only the rank; greedy as one candidate drawn for sure; which media the server takes, each sent as a file with the placeholders and the Context tab showing the server's expansion, a different expansion at generation time an error, a refused kind a plain file; MTP's counts on either route, the per-step ones marking the guesses only when they add up and counting as kept only those up to the stop token; the stop token, the token limit, Stop closing the stream; an unreachable server, a model it does not serve, one started from a folder of its own or with a model of a family the page (or its transformers) does not know or without a config.json, a missing or failing `/inference/v1/generate`, a rank the server took from the logits staying true to the listed probabilities where they tie, its refusals and errors, in both languages, and an error mid-run shown in the page's language; which family a served model is of, from its config.json alone (a folder on this machine read where it is, a repo without one of no family, and with no way to the Hub or from a folder only on the server not told), and the model id and `model_type` an unsupported model's error carries for the page's GitHub issue; warming up, and a step that ends before its first token; the model card's specs, the server, what its numbers are in each logprobs mode, its sampling masks and the media it refused; a whole turn with a tool call and one with an image, every moment of each rendered; with the real tokenizer and chat template (when in `models/`), the server gets exactly the prompt the Context tab shows; with `VLLM_TEST_URL`, a real server, each feature it was started with | | `test_vllm_docker.py` | Starting vLLM in Docker, with Docker faked at the level of `subprocess`: the weights downloaded at their pinned commit before `docker compose up` runs `docker-compose/compose.yaml` with the model, its revision and `models/`, and `compose.gpu.yaml` on top where Docker has the nvidia runtime (or `AGENT_LOOP_VLLM_DEVICE` says so); a server already up for the model not started again, another model replacing it; no Docker, Docker not running, no Compose, `docker compose up` failing, a container killed for memory, one that exits (the exceptions in its log, cause first) and one that never answers, each said in both languages; a rerun while Compose is busy stopping it too; `docker compose down`; the model the app asked for kept through a failed start until it is stopped, and quitting the app stopping the server it started (and leaving Docker alone without one); on the GPU, MTP with Gemma's drafter at its pinned commit (downloaded first) or Qwen's own MTP layer, never on the CPU or unasked, and switching it starting the server again. The compose files: on 127.0.0.1 only, `models/` read-only and offline, Qwen3.5 at its pinned commit by default, the GPU file the same vLLM with the same flags but the CPU-only one, plus a KV cache of the CPU's size, a low `--gpu-memory-utilization`, and speculative decoding off by hand, its counts in detail only with a config (vLLM refuses them without one), Docker Compose handing vLLM the page's config as it is, and Docker Compose reading both (when installed) and mounting the folder a relative `AGENT_LOOP_MODELS` names; with `RUN_VLLM_DOCKER_TESTS=1`, the real server in Docker answering through `VLLMBackend`, with MTP and then without where Docker has a GPU | @@ -333,6 +358,9 @@ How it works quotes code, and `tests/guide_excerpts.txt` keeps everything it quo - `vllm_backend.py` follows the protocols of vLLM 0.32, read from its source, and is tested against a fake server that speaks them, with the real tokenizers and in the real page; `/inference/v1/generate` and its sampling masks and per-step counts are marked experimental in vLLM and may change. Its tests against a real server (`VLLM_TEST_URL`) have been run on one GPU only, a DGX Spark's GB10. - vLLM in Docker runs on the CPU on a Mac, and slowly: Gemma 4 E2B decodes about 4.6 tok/s, needs Docker to have about 20 GB of memory or more, and its first load spends about 10 minutes running its media encoders to see which kinds the server takes. `compose.gpu.yaml` has been run on one NVIDIA GPU only, a DGX Spark's GB10. The CPU image patches vLLM 0.32.0 (`docker-compose/vllm-cpu/patch_vllm.py`); a newer vLLM may need neither fix, and the build stops if the code it patches is not the one it was written for. - The app stops vLLM's container when it quits with Ctrl+C or a SIGTERM. Stopped any other way (its terminal closed, `kill -9`), it leaves the container running and holding the model; `docker compose -p agent-loop-vllm down` stops it. +- With pi, the page sees only what pi reports: no tokens, probabilities or special tokens, and not the prompt pi sends. The token counts are the provider's, tok/s is measured as the reply arrives (the network included), and the cost is at the prices in pi's catalog, an estimate. pi's models are a provider's: they cost money, and the conversation goes to that provider. +- pi's tools run on this computer, without a sandbox; with bash on, the model can do whatever you can. Ask me before running a tool shows each call first. +- `pi_harness.py` follows pi 1.4's RPC protocol, from its documentation and the records pi 1.4.0 sends; the real-pi tests have been run with pi 1.4.0 only. pi 1.4 needs Node.js 22.19 or newer: under an older Node it does not start, and the page shows what it said. The page asks pi which models it has at most once a minute; after logging in to a provider in pi, Try again lists them. - The `MODEL_ID` environment variable picks the model selected at first; it may also name a checkpoint the menu does not list, as long as it belongs to a supported family (Gemma 4 or Qwen3.5); another family needs one added first, as in Supporting more models. ## License diff --git a/README.zh-TW.md b/README.zh-TW.md index f45f734..c12e7fc 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -9,6 +9,7 @@ 這個專案是學習用的：用更直覺、更視覺化的方式，讓你看懂 agent 迴圈在底層怎麼運作。Agent 的聊天框背後跑著一個迴圈：一步接著一步，模型提出動作、harness 去執行，直到停止。一般的聊天 app 只讓你看到問題和答案；這個網頁在每一步發生的當下把它畫出來，細到模型寫下的每一個 token，整個機制攤開在你眼前。畫面上的每個部分都對應 [Agent Atlas 技術與實戰交流](https://cyyeh.github.io/agent-atlas-learning/) unit 01 導讀的一節，本專案就是它的專案之一；[可以試試看的實驗](#可以試試看的實驗)則把這些概念變成可以自己動手試的實驗。 - 模型：側欄可以選 [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) 或 [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B)，在本機用 Hugging Face `transformers` 推論，也可以用 [vLLM](https://docs.vllm.ai/) 跑：在側欄的「用什麼跑模型」選 vLLM (Docker)，網頁會替你在 Docker 裡啟動一台 vLLM 伺服器，有 NVIDIA GPU 就用 GPU，否則用 CPU；或是設定 `VLLM_URL`，連到你自己的伺服器，在這台電腦上或有 GPU 的 Linux 機器上都行（見[在 vLLM 伺服器上跑模型](#在-vllm-伺服器上跑模型)） +- Harness：迴圈是這個專案自己寫的（`harness/harnesses/simple_harness.py`），也可以在側欄的「用什麼跑迴圈」選 [pi](https://pi.dev)：這個 coding agent 會問它登入過的 Claude、GPT 或其他模型，網頁照 pi 回報的事件畫出它的每一步，只是看不到 token（「[pi 當 harness](#pi-當-harness)」） - MTP 推測解碼：一次 forward 產生好幾個 token。用 `transformers` 時，Gemma 4 E2B 可以在側欄打開（預設關閉），用 [google/gemma-4-E2B-it-assistant](https://huggingface.co/google/gemma-4-E2B-it-assistant) 當草稿模型；在有 GPU 的 vLLM 上兩個模型都會推測解碼，Gemma 4 E2B 用同一個草稿模型，Qwen3.5 用它自己 checkpoint 裡的 MTP 層，由網頁在 Docker 裡啟動的 vLLM 預設就打開 - 檔案：用輸入框左邊的 ＋ 附上圖片和影片（兩個模型都行）、聲音（Gemma 4 E2B，也可以直接用麥克風錄音），或是在輸入框上方選一個範例檔案（收據、錄音、影片、銷售資料），自己問它問題；每個檔案在脈絡裡變成一串 token，畫面上看得到 - 寫程式（選用）：在側欄打開「讓模型寫程式」，模型多一個 `run_python` 工具，它寫的 Python 在 Docker 沙箱裡執行，讀得到你附上的資料檔（CSV、TSV、JSON、純文字、Markdown、Excel），也能用 matplotlib 畫圖，圖會出現在聊天區。只有這個功能和 vLLM (Docker) 需要 Docker @@ -30,7 +31,7 @@ | 環境 | 工具實際回傳的結果與耗時；`run_python` 的程式碼寫在上面，印出來的東西或 traceback 寫在下面 | §2 環境與可觀察的狀態 | | 停止 | 停止原因和停止 token。Gemma 回答完停在 ``（相當於 end_turn）、要工具時停在 `<\|tool_response>`（相當於 tool_use）；Qwen 兩種都停在 `<\|im_end\|>`。另外還有步數上限、token 上限、使用者中斷 | §5 什麼時候停 | | 模型分頁 | 參數量拆解、層數與層的種類、注意力頭數、脈絡上限、模型讀得懂什麼（文字、圖片、聲音、影片）、MTP 草稿模型、解碼設定的說明、每一步的 prefill 時間與 tok/s（開 MTP 時加上草稿收下幾個、解碼 forward 幾次），tokenizer 全部特殊 token 的 id 與用途（Gemma 24 個、Qwen 33 個），以及 `models/` 裡這個模型資料夾的每個檔案：大小、用途、在這個模型裡裝了什麼、關鍵欄位的值 | §1、§7 Agent 是系統 | -| 運作說明頁 | 一輪對話的 18 個階段（包括圖片怎麼變成 token、MTP 怎麼猜、怎麼驗證，以及模型可以交給誰跑），每段附上做這件事的程式碼片段和行號，點了會在 GitHub 上打開那幾行，有中文和英文兩版。放在 GitHub Pages，行號在產生頁面時從原始碼查出來，連結指向那個 commit；每次推上 main 都會重新產生，程式改了會跟著更新 | 全部 | +| 運作說明頁 | 一輪對話的 19 個階段（包括圖片怎麼變成 token、MTP 怎麼猜、怎麼驗證、模型可以交給誰跑，以及把迴圈交給 pi），每段附上做這件事的程式碼片段和行號，點了會在 GitHub 上打開那幾行，有中文和英文兩版。放在 GitHub Pages，行號在產生頁面時從原始碼查出來，連結指向那個 commit；每次推上 main 都會重新產生，程式改了會跟著更新 | 全部 | ## 快速開始 @@ -43,6 +44,7 @@ - [uv](https://docs.astral.sh/uv/) 0.13.4 以上（會自動裝好 Python 3.12）。更舊的版本還信任 eMudhra 那批外洩的 emSign 憑證（[公告](https://discuss.python.org/t/109576)），所以 `uv sync` 會拒絕執行；用 `uv self update`（或 `brew upgrade uv`）更新 - 網路：第一次下載模型，以及 Wikipedia 工具 - 選用：[Docker](https://docs.docker.com/get-started/get-docker/)（Docker Desktop、OrbStack 或 Colima），只有讓模型寫程式、或用 vLLM (Docker) 跑模型時才需要 +- 選用：[pi](https://pi.dev) 和 Node.js 22.19 以上，只有要把迴圈交給 pi 時才需要（「[pi 當 harness](#pi-當-harness)」） ```bash uv sync @@ -91,7 +93,11 @@ app/ 你看到的東西，站在 harness 上面 guide.py 「運作說明」頁：中英文內容、行號查詢，以及產生放上 GitHub Pages 的靜態 HTML assets/ 網頁的樣式和運作說明的 script；logo.svg；examples/ 是範例附上的檔案，也是輸入框上方的範例檔案 harness/ 跑一輪的東西；從不 import Streamlit - agent.py 迴圈：組 prompt → 生成 → 解析 → 驗證 →（先問你）→ 執行工具 → 接回脈絡 → 判斷要不要停 + harnesses/ 跑迴圈的那一層 + harness.py 網頁透過它跑一輪的 Harness：run、stop、decide，以及每個 harness 送出的事件 + simple_harness.py 這個專案的迴圈：組 prompt → 生成 → 解析 → 驗證 →（先問你）→ 執行工具 → 接回脈絡 → 判斷要不要停 + pi_harness.py pi 當 harness：用 RPC 模式跑 pi，把它的 JSON 紀錄翻成網頁的事件 + pi_approve.ts 讓網頁在 pi 執行工具前先問你的 pi extension backends/ 跑模型的那一層 backend.py harness 和畫面透過它使用模型的 Backend：render、encode、generate，以及 token 事件的欄位 simple_backend.py 本機推論，自己寫的解碼迴圈；MTP 的草稿與驗證也在這裡 @@ -113,11 +119,11 @@ tests/ pytest 測試，見下面「測試」 models/ 下載的模型（git 會忽略它；AGENT_LOOP_MODELS 可以換位置） ``` -agent 在背景 thread 執行，每走到一個階段就 append 一個事件；UI 只負責把事件列表畫出來。所以按「停止」或「允許」時 Streamlit 重跑 script，背景的迴圈不會被打斷。 +agent 在背景 thread 執行，每走到一個階段就 append 一個事件；UI 只負責把事件列表畫出來。所以按「停止」或「允許」時 Streamlit 重跑 script，背景的迴圈不會被打斷。網頁只透過 `Harness`（`harnesses/harness.py`）跑迴圈：`run()`、`stop()`、`decide()`，以及它送出的事件。`SimpleHarness` 是這個專案自己的，`PiHarness` 則把迴圈交給 pi（「[pi 當 harness](#pi-當-harness)」）。 「運作說明」是一個靜態頁：`uv run python -m app.guide _site` 會寫出 `_site/zh/index.html` 和 `_site/en/index.html`（樣式和一小段 script 都內嵌在裡面，可以直接用瀏覽器打開），以及 `_site/index.html`：它把讀者送到上次在頁面右上角選過的語言，沒選過就照瀏覽器的語言。推上 main 時，`.github/workflows/pages.yml` 先跑 `tests/test_guide.py`，再做同一件事並發佈到 GitHub Pages；測試沒過的話，網站上維持原來那一版。 -一次執行的事件依序是：`run_start`，然後每一步 `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`，有工具呼叫時再加上 `tool_call` →（`approval_request` → `approval`）→（真的執行工具時的 `tool_start`）→ `tool_result`，最後是 `run_end`；出錯時它前面還有一個 `error`：`message` 是例外的訊息，backend 的錯誤（例如 vLLM 伺服器的）另外帶著兩種語言的版本（`text`），畫面顯示目前語言的那一個。`context` 帶的是模型實際讀到的 prompt（每個附件都展開成它的 token），以及每種媒體佔了幾個 token（`media_tokens`）。`token` 帶著這個 token，以及 backend 知道的話，它的機率、排名、前 5 名候選和 MTP 草稿模型猜了什麼（`backend.TOKEN_DETAILS`）：`simple_backend.py` 全都知道；backend 沒有回報的，畫面就不畫，不會拿沒量過的數字來填。等這一步結束才知道哪些 token 是 MTP 猜中的 backend（vLLM 伺服器），把它們放在 `model_end` 的 MTP 統計裡（`kept_at`、`turned_down_at`），畫面到那時才標出來。`tool_result` 帶著模型讀的觀察，另外還有工具做給你看的圖（`images`：`run_python` 畫的圖的路徑），模型讀不到。 +一次執行的事件依序是：`run_start`，然後每一步 `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`，有工具呼叫時再加上 `tool_call` →（`approval_request` → `approval`）→（真的執行工具時的 `tool_start`）→ `tool_result`，最後是 `run_end`；出錯時它前面還有一個 `error`：`message` 是例外的訊息，backend 的錯誤（例如 vLLM 伺服器的）另外帶著兩種語言的版本（`text`），畫面顯示目前語言的那一個。`context` 帶的是模型實際讀到的 prompt（每個附件都展開成它的 token），以及每種媒體佔了幾個 token（`media_tokens`）。`token` 帶著這個 token，以及 backend 知道的話，它的機率、排名、前 5 名候選和 MTP 草稿模型猜了什麼（`backend.TOKEN_DETAILS`）：`simple_backend.py` 全都知道；backend 沒有回報的，畫面就不畫，不會拿沒量過的數字來填。等這一步結束才知道哪些 token 是 MTP 猜中的 backend（vLLM 伺服器），把它們放在 `model_end` 的 MTP 統計裡（`kept_at`、`turned_down_at`），畫面到那時才標出來。`tool_result` 帶著模型讀的觀察，另外還有工具做給你看的圖（`images`：`run_python` 畫的圖的路徑），模型讀不到。`run_start` 說這一輪是哪個 harness 跑的（`harness`：`simple` 或 `pi`）。自己不跑模型的 harness，也就是 pi，不送 `context`、`prefill` 和 `token`，也沒有 family：它改送 `delta`，模型邊寫邊傳回來的一段輸出和它的 channel（思考、回覆、工具呼叫），`model_end` 帶的則是 provider 報的 token 數（`usage`）。 ### 兩種語言 @@ -234,9 +240,26 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py - **在 DGX Spark 上**（NVIDIA GB10，GPU 和整台電腦共用 128 GB 記憶體）：在 GPU 上，Qwen3.5 0.8B 啟動約 2 分鐘、解碼約 113 tok/s，Gemma 4 E2B 啟動約 3 分鐘、解碼約 41 tok/s，兩個模型都通過接真的伺服器的測試，包括 Gemma 讀得懂的每種媒體。照 vLLM 預設的 `gpu_memory_utilization`（0.92），伺服器在這台電腦上不肯啟動：vLLM 把整台電腦可用的記憶體當成 GPU 空著的記憶體，大約是 122 GiB 裡的 108 GiB。在同一台電腦的 CPU 上，Qwen3.5 0.8B 啟動約 1.3 分鐘、解碼約 23 tok/s。 - **在 Mac 上**（Apple M5，給 Docker 10 個核心）：Qwen3.5 0.8B 啟動約 1.5 分鐘，佔 Docker 約 9 GB 記憶體，解碼約 10 tok/s。Gemma 4 E2B 要的記憶體比 Docker Desktop 預設的多：16 GB 時載入到一半會被停掉（頁面會說明，也說怎麼給 Docker 更多記憶體），24 GB 時 3 到 5 分鐘啟動、解碼約 4.6 tok/s，但載入時問伺服器收哪些媒體，要在 CPU 上跑它的圖片、聲音和影片編碼器，第一次要花大約 10 分鐘。 +### pi 當 harness + +網頁也可以把迴圈交給 [pi](https://pi.dev)，一個有自己的迴圈、system prompt 和工具（read、bash、edit、write、grep、find、ls）的 coding agent。在側欄最上面的「用什麼跑迴圈」選 pi：這台電腦不載入模型，pi 問它登入過的模型（Claude、GPT、Gemini，或它 `models.json` 裡任何相容的端點），網頁用同一張迴圈圖、同樣的 step 卡片和事件列表，畫出 pi 說它做了什麼。畫不出來的，是只有自己跑模型的 harness 才看得到的東西：API 傳回來的是文字，不是 token，所以沒有 token 的機率、候選和特殊 token；pi 也不說它送出的 prompt 是什麼，所以 step 沒有脈絡段，「脈絡」分頁是空的。一個 step 顯示一段段傳回來的文字、provider 報的 token 數（以及其中多少來自 cache），和照 pi 的價目估的費用。 + +```bash +npm install -g @earendil-works/pi-coding-agent # pi 1.4 需要 Node.js 22.19 以上 +pi # 再輸入 /login 登入一次；或在啟動 app 之前設好 API key，例如 ANTHROPIC_API_KEY +PI_COMMAND=/path/to/pi uv run streamlit run streamlit_app.py # pi 不在 PATH 上的話（任何指令都行，npx 也可以） +``` + +- **網頁怎麼跑它。** `harness/harnesses/pi_harness.py` 每個問題都用 RPC 模式啟動 pi（`pi --mode rpc`）：指令寫進它的 stdin，紀錄從它的 stdout 讀出來，一行一個 JSON。它先選好模型和思考等級、關掉 pi 自己的重試（重試會變成另一個 step，所以請求失敗就當成錯誤顯示），再送出問題，每個指令都等 pi 回應了才送下一個；pi 的一個 turn（向模型送一次請求，加上回覆裡要執行的工具）就是一個 step。pi 把對話存在這段對話資料夾裡的 session 檔，下一個問題就接著它；按「新對話」會刪掉。 +- **啟動 pi 時不帶的東西。** 你自己的 extension、skill、prompt 範本和 context 檔（AGENTS.md、CLAUDE.md）都不載入（`--no-extensions` 之類），所以網頁畫的是 pi 本身的迴圈，每台電腦上都一樣；模型和登入資訊還是你的，在 `~/.pi/agent`。 +- **工具。** 側欄選模型看得到 pi 的哪些工具（預設是 read、bash、edit、write，和 pi 一樣）。它們在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：不像 `run_python`，bash 碰得到你碰得到的一切。附上的檔案存在那個資料夾裡，問題裡會寫出檔名；讀得懂圖片的模型，圖片也會跟著問題一起送過去。 +- **執行工具前先問我。** pi 自己不會問；打開時，網頁讓 pi 載入 `harness/harnesses/pi_approve.ts`，一個在每次執行工具前開確認對話框的小 extension。在 RPC 模式下，對話框出現在 stdout，網頁把它顯示成「允許」和「拒絕」，你的回答再從 stdin 送回去；被拒絕的呼叫不會執行，模型讀到的是用這一輪語言寫的拒絕原因。 +- **停止。** 按停止會送給 pi `abort`。pi 自己沒有步數上限，所以「步數上限」由網頁來守：pi 開始超過上限的那個 turn 時，網頁就請它中止。確認一直沒人回答的話，5 分鐘後停掉 pi，和內建 harness 一樣。 +- **測試**用的是 pi 1.4.0：透過照劇本說它的協定的替身（`tests/fake_pi.py`），以及 `RUN_PI_TESTS=1` 時，指向 127.0.0.1 上一個照劇本回答的模型的真 pi。 + ### 支援更多模型 -harness（`agent.py`）、解碼迴圈（`simple_backend.py`）和畫面（`render.py`）都只寫一次，裡面沒有任何「如果是 Gemma」的分支。每個模型不一樣的地方集中在 `families/` 的一個 `Family` 物件裡： +harness（`harnesses/simple_harness.py`）、解碼迴圈（`simple_backend.py`）和畫面（`render.py`）都只寫一次，裡面沒有任何「如果是 Gemma」的分支。每個模型不一樣的地方集中在 `families/` 的一個 `Family` 物件裡： - 讀輸出：`parse`（把輸出切成思考、工具呼叫、回覆）、`thought_prefix`、`channel_switch`（哪些 token 切到思考、工具呼叫、回覆）、`stop_tokens` - 寫回對話：`reasoning_key`（思考存在哪個欄位，chat template 才讀得到）、`text_with_calls`（工具呼叫前的文字留不留） @@ -281,9 +304,10 @@ uv run pytest # 約 40 秒，不載入 RUN_MODEL_TESTS=1 uv run pytest tests/test_real_model.py # 載入真的 Gemma（含 MTP 草稿模型）和 Qwen，約 4 分鐘；加 -k Qwen 只跑一個 VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # 問一台正在跑的 vLLM 伺服器 RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # 用 Qwen3.5 在 Docker 裡啟動 vLLM，問它一個問題 +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # 啟動真的 pi（PATH 上的，或 PI_COMMAND），對上 127.0.0.1 上照劇本回答的模型 ``` -大部分測試用 `tests/fakes.py` 的假模型：它照劇本一個 token 一個 token 吐出預先寫好的輸出，和 `SimpleBackend` 一樣是一個 `Backend`（`backend.py`），所以 harness 和網頁可以從頭跑到尾。 +大部分測試用 `tests/fakes.py` 的假模型：它照劇本一個 token 一個 token 吐出預先寫好的輸出，和 `SimpleBackend` 一樣是一個 `Backend`（`backend.py`），所以 harness 和網頁可以從頭跑到尾。需要 pi 的地方，測試啟動的是 `tests/fake_pi.py`，它照劇本說 pi 1.4 的 RPC 協定，所以不需要 Node，也不需要模型。 網頁收的每一種檔案格式都有測試：`tests/samples.py` 用範例的檔案做出每種格式的小檔案（收據的每種圖片格式、錄音的每種聲音格式、滾動的球的每種影片格式、銷售資料的每種資料格式）。`media.py` 多收了一種格式卻沒有對應的範例檔，`test_media.py` 就會失敗；`test_real_model.py` 拿每一種格式問兩個模型。需要 Docker 的測試只在 Docker 和沙箱 image 都在時才跑。 @@ -296,9 +320,10 @@ RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real | `test_families.py` | 每個家族都說明了它用到的 token、中英文說明列出同樣的 token、`model_type` 找得到家族；token 顯示成它代表的文字，只是一個字一部分的 byte 顯示成 `<0xF0>`；每個家族都用中英文說明它 checkpoint 裡的檔案，兩種語言提到同樣的欄位和 token；每個模型和草稿模型都照固定的 commit 下載；vLLM 替清單裡的模型推測解碼時，用的是它 checkpoint 裡的 MTP 層，或 transformers 用的那個草稿模型 | | `test_tools.py` | schema 驗證與預設值、計算機的 AST 白名單（拒絕 `__import__` 之類）、時區、Wikipedia（HTTP 用假的）；`run_python` 鎖住的 `docker run`、各種錯誤（traceback、沒有輸出、記憶體太多、跑太久、沒有 image）、Docker 沒準備好或 image 過舊時缺什麼，都有中英文兩種；記號後面的圖存檔時不蓋掉現有檔案、只收 PNG、最多四張、程式出錯就不收；image 的版本 label 和用 hash 安裝的 pip；有 Docker 時，真的沙箱讀得到 `/data` 但改不了、連不上網路，讀得了每一種資料格式，送得回中文標籤畫得出來的圖，pip（和它的 certifi）也是修正 emSign 之後的版本 | | `test_simple_backend.py` | `SimpleBackend` 有 `Backend` 的每個方法和屬性，它的 token 帶著每個欄位和細節；自己寫的解碼迴圈：KV cache、停止 token、token 上限、中斷、channel 切換、被拆開的中文字元；`encode()` 照順序把每種媒體和家族的設定交給 processor、短影片給它全部畫格；圖片和影片在 prefill 前先跑視覺編碼器，上一個 prompt 的位置偏移不會留下來；greedy 遇到兩個 token 打平時和 `generate()` 一樣拿 id 小的；temperature / top-k / top-p 的候選與機率；MTP 用照劇本猜的假草稿模型：猜測的收下與退回、全部收下時多一個 token、KV cache 退回多少、草稿模型看不到作廢的猜測，以及抽樣時檢查猜測不改變分布；模型照固定的 revision 下載到 `models/`（`AGENT_LOOP_MODELS` 可以換位置，相對路徑從啟動 app 的地方算起；`HF_HUB_CACHE` 不行），在 CUDA 上放掉模型時，先放掉 cuBLAS 的工作區再清快取，模型資料夾的檔案、大小（經過快取的連結）和說明提到的欄位 | -| `test_agent.py` | 假模型是一個 `Backend`，它的 token 帶的欄位和 `SimpleBackend` 的一樣；harness：事件順序、工具結果接回脈絡、驗證錯誤、重複呼叫、結果截斷、每一種停止原因、允許 / 拒絕 / 等待時按停止；附件變成問題裡的 part 和給 `run_python` 的說明、每一輪的媒體都交給 processor、展開後的脈絡仍然只把工具結果標成新加入、`run_python` 拿到這段對話的資料夾、沒有 `run_python` 的一輪；模型讀到圖在哪裡，圖本身只給頁面 | -| `test_render.py` | 一次執行的每一個事件前綴都能用兩種語言畫出來（執行中每 0.5 秒重畫）、HTML 標籤成對、迴圈圖是合法 SVG、模型輸出一律跳脫、特殊 token 的說明浮窗與列表；MTP 的底線、浮窗、統計和草稿模型卡片；一串同樣的特殊 token 畫成一個標籤加個數、截斷的片段不會切開 token、媒體佔了幾個 token、好幾行的程式碼和輸出照行寫出來、這一輪沒提供的工具、一次執行和每一步的圖；模型資料夾卡片：檔案的順序、大小和說明、家族有說明但 checkpoint 沒有的檔案、草稿模型的資料夾、跳脫；backend 沒回報機率、或只回報 log 機率時，token 照它帶的內容畫，不多畫；英文畫面上沒有留下中文 | -| `test_app.py` | 用 Streamlit `AppTest` 操作整個網頁：Streamlit 不在每次執行完腳本後做垃圾回收，重跑會等正在跑的那一次；送出問題、範例按鈕、側欄設定傳到模型、多輪對話、脈絡與模型分頁、允許 / 拒絕 / 停止、新對話、還原設定、兩個分頁不能同時跑、換模型、運作說明連到 GitHub Pages 上同語言的那一版；MTP 開關預設關閉、打開才載入草稿模型、換到沒有草稿模型的 Qwen 會關掉並停用；英文瀏覽器看到英文、上次選的語言優先於瀏覽器、切換語言時對話和設定都留著、模型讀到的錯誤跟著語言；每種語言的頁面只有那種語言的範例、範例會附上它的檔案，模型讀得懂那一種才出現、輸入框上方選的範例檔案會附在接下來打的問題，模型用得上的才列出來、「讓模型寫程式」打開了才去找 Docker 並說明缺什麼、銷售資料的範例用 Python 讀附上的檔案、圖畫在聊天區和畫它的那一步、「新對話」刪掉這段對話的檔案；設了 `VLLM_URL` 時，選單列出伺服器提供的模型、側欄和模型分頁寫出伺服器、MTP 由伺服器決定、開了 sampling mask 的伺服器不會從側欄收到 greedy 或不限的 top-k、模型讀伺服器收的媒體，頁面說明它不收哪些和原因，選單不列伺服器上的 LoRA adapter，網頁不支援的家族的模型標上 ⚠、一開始選第一個支援的，選了不支援的會說明原因，並附上用頁面語言填好的 GitHub issue，寫出模型（資料夾只寫名字），不寫伺服器的位址，伺服器跑到一半送來的錯誤用頁面的語言顯示，載入時或之後連不上伺服器，都用頁面的語言說明；沒設 `VLLM_URL` 時，「用什麼跑模型」一開始是 transformers、完全不碰 Docker，選 vLLM (Docker)會放掉本機的模型（沒有東西留著它）再用它啟動伺服器，重跑不會再啟動，換模型就用新模型啟動，換回 transformers 會停掉容器，vLLM 在跑時新開分頁或重新整理會接上它的伺服器和模型，停在 transformers 的分頁會跟著另一個分頁換到 vLLM 再換回來（模型在這裡只載入一次），在 GPU 上 vLLM 預設開 MTP 啟動、切換開關會不開 MTP 重新啟動（按還原建議值會再開），在 CPU 上不開 MTP、開關會說明原因，分頁會跟著另一個分頁選的 MTP，啟動失敗時用頁面的語言說明，只有按「再試一次」才重試，換回 transformers 的切換鈕也還在 | +| `test_simple_harness.py` | 假模型是一個 `Backend`，它的 token 帶的欄位和 `SimpleBackend` 的一樣；harness：事件順序、工具結果接回脈絡、驗證錯誤、重複呼叫、結果截斷、每一種停止原因、允許 / 拒絕 / 等待時按停止；附件變成問題裡的 part 和給 `run_python` 的說明、每一輪的媒體都交給 processor、展開後的脈絡仍然只把工具結果標成新加入、`run_python` 拿到這段對話的資料夾、沒有 `run_python` 的一輪；模型讀到圖在哪裡，圖本身只給頁面 | +| `test_pi_harness.py` | pi 當 harness，用 `tests/fake_pi.py` 代替 pi：兩個 harness 都是 `Harness`，pi 只送介面列出的事件；一輪變成照迴圈順序的網頁事件（依 channel 串流進來的思考、文字和工具呼叫參數、provider 報的 token 數和 cache、呼叫和結果）；pi 在這段對話的資料夾、用它的 session 啟動，帶著側欄的工具、不帶你自己的 extension，送出問題之前先選好模型、關掉重試，完全沒有工具，附件的檔名寫給模型、圖片直接送過去，重複的呼叫有標示，工具失敗，token 上限，下一個問題用同一個 session；透過 extension 先問：允許、拒絕（模型讀到這一輪語言的拒絕原因）、等待時按停止、逾時；回覆到一半按停止和 pi 還沒啟動就按停止，步數上限之後不再畫 step；請求失敗、問題被拒絕、pi 沒有的模型、pi 一啟動就結束和沒有安裝 pi，都用兩種語言說明；pi 有哪些模型；`RUN_PI_TESTS=1` 時，真的 pi 對上 127.0.0.1 上照劇本回答的模型，先問和不先問各一次 | +| `test_render.py` | 一次執行的每一個事件前綴都能用兩種語言畫出來（執行中每 0.5 秒重畫）、HTML 標籤成對、迴圈圖是合法 SVG、模型輸出一律跳脫、特殊 token 的說明浮窗與列表；MTP 的底線、浮窗、統計和草稿模型卡片；一串同樣的特殊 token 畫成一個標籤加個數、截斷的片段不會切開 token、媒體佔了幾個 token、好幾行的程式碼和輸出照行寫出來、這一輪沒提供的工具、一次執行和每一步的圖；模型資料夾卡片：檔案的順序、大小和說明、家族有說明但 checkpoint 沒有的檔案、草稿模型的資料夾、跳脫；backend 沒回報機率、或只回報 log 機率時，token 照它帶的內容畫，不多畫；pi 跑的一輪：照 channel 畫出串流進來的文字、沒有 token 的數字、一律跳脫、合併時不改動事件，迴圈圖、停止卡片和每一步的表格說是 pi 在跑迴圈，pi 的卡片兩種語言都對；英文畫面上沒有留下中文 | +| `test_app.py` | 用 Streamlit `AppTest` 操作整個網頁：Streamlit 不在每次執行完腳本後做垃圾回收，重跑會等正在跑的那一次；送出問題、範例按鈕、側欄設定傳到模型、多輪對話、脈絡與模型分頁、允許 / 拒絕 / 停止、新對話、還原設定、兩個分頁不能同時跑、換模型、運作說明連到 GitHub Pages 上同語言的那一版；MTP 開關預設關閉、打開才載入草稿模型、換到沒有草稿模型的 Qwen 會關掉並停用；英文瀏覽器看到英文、上次選的語言優先於瀏覽器、切換語言時對話和設定都留著、模型讀到的錯誤跟著語言；每種語言的頁面只有那種語言的範例、範例會附上它的檔案，模型讀得懂那一種才出現、輸入框上方選的範例檔案會附在接下來打的問題，模型用得上的才列出來、「讓模型寫程式」打開了才去找 Docker 並說明缺什麼、銷售資料的範例用 Python 讀附上的檔案、圖畫在聊天區和畫它的那一步、「新對話」刪掉這段對話的檔案；設了 `VLLM_URL` 時，選單列出伺服器提供的模型、側欄和模型分頁寫出伺服器、MTP 由伺服器決定、開了 sampling mask 的伺服器不會從側欄收到 greedy 或不限的 top-k、模型讀伺服器收的媒體，頁面說明它不收哪些和原因，選單不列伺服器上的 LoRA adapter，網頁不支援的家族的模型標上 ⚠、一開始選第一個支援的，選了不支援的會說明原因，並附上用頁面語言填好的 GitHub issue，寫出模型（資料夾只寫名字），不寫伺服器的位址，伺服器跑到一半送來的錯誤用頁面的語言顯示，載入時或之後連不上伺服器，都用頁面的語言說明；沒設 `VLLM_URL` 時，「用什麼跑模型」一開始是 transformers、完全不碰 Docker，選 vLLM (Docker)會放掉本機的模型（沒有東西留著它）再用它啟動伺服器，重跑不會再啟動，換模型就用新模型啟動，換回 transformers 會停掉容器，vLLM 在跑時新開分頁或重新整理會接上它的伺服器和模型，停在 transformers 的分頁會跟著另一個分頁換到 vLLM 再換回來（模型在這裡只載入一次），在 GPU 上 vLLM 預設開 MTP 啟動、切換開關會不開 MTP 重新啟動（按還原建議值會再開），在 CPU 上不開 MTP、開關會說明原因，分頁會跟著另一個分頁選的 MTP，啟動失敗時用頁面的語言說明，只有按「再試一次」才重試，換回 transformers 的切換鈕也還在；「用什麼跑迴圈」選 pi 時，側欄是它的模型和設定、沒有解碼設定，問題畫成 pi 的迴圈、在這段對話的資料夾裡跑，側欄的設定傳到 pi，在網頁上按「允許」，停止，「脈絡」分頁說 pi 不給看 prompt、「模型」分頁顯示 pi 的模型和工具，pi 沒有模型或沒有安裝時說明原因並可以「再試一次」，從不找 Docker，換回來時內建 harness 的模型和設定都還在，切換會開新對話，英文畫面沒有中文 | | `test_chat_template.py` | 用每個模型真的 tokenizer 和 chat template（已經下載到 `models/` 才跑）：特殊 token 都是單一 token、每個特殊 token 都有說明、token 浮窗顯示的是它代表的文字、模板寫進歷史的工具呼叫能被 parser 讀回；用真的模板跑 harness，確認 step 卡片標成「新加入」的剛好是工具結果（兩個參數、思考模式、工具呼叫前有文字），用真的 processor、問題附上圖片和影片時也一樣；每個模型（和草稿模型）資料夾裡的每個檔案都有說明，說明提到的欄位在檔案裡都找得到 | | `test_vllm_backend.py` | vLLM backend 對上一個假伺服器，它說的是 vLLM 0.32 的串流格式：`/v1/completions` 收到 render 好的 prompt 的 token id、側欄的每個設定（其他取樣設定都設成不起作用）、停止 token，以及放得進伺服器脈絡的 `max_tokens`；每個 token 的機率和前 5 名，排名只在前 5 名裡才有；加了 `--enable-scale-out` 時走 `/inference/v1/generate`，每個 token 都有排名，雖然選中的 token 排在清單第一個，前 5 名仍照排名；兩條路上同分第一的 token 都是 #1，雖然 vLLM 的排名把同分的也算進去；有 sampling mask 時（vLLM 只在 processed log 機率下送），有剩下的候選和取樣時的機率，沒有模型自己的機率，temperature 0 和 top-k 0 在每條路上都照 vLLM 的做法被拒絕，有 mask 但沒加 `--enable-scale-out` 的伺服器，網頁載入時也會發現，而還沒準備好的伺服器不算這種拒絕；載入時用一個 token 判斷伺服器的 `--logprobs-mode`，沒有 mask 的 processed log 機率在前幾名有被剃掉時看得到剩下幾個，logits 只有排名；greedy 是只剩一個、一定抽中的候選；伺服器收哪幾種媒體、每個檔案跟著佔位 token 送過去、脈絡分頁顯示伺服器的展開、生成時展開得不一樣就是錯誤、不收的種類成為一般檔案；兩條路上都有 MTP 的統計，每一步的統計對得上時才標出猜中的 token，收下的只算到停止 token 為止；停止 token、token 上限、按停止就關掉串流；連不上伺服器、它沒有提供的模型、用它自己的資料夾、或用網頁（或它的 transformers）不認得的家族的模型、或沒有 config.json 的模型啟動的伺服器、沒有或出錯的 `/inference/v1/generate`、它的拒絕和錯誤、伺服器從 logits 算出的排名在列出的機率打平時仍照列出的機率，兩種語言都有，跑到一半的錯誤用頁面的語言顯示；伺服器提供的模型屬於哪個家族，只讀它的 config.json 就知道（這台電腦上的資料夾就地讀，沒有 config.json 的 repo 不屬於任何家族，連不上 Hub 或只在伺服器上的資料夾就說不出來），不支援的模型的錯誤帶著模型 id 和 `model_type`，給網頁的 GitHub issue 用；暖機，以及第一個 token 之前就結束的一步；模型卡片上的規格、伺服器、每種 logprobs 模式下數字代表什麼、sampling mask 和它不收的媒體；一整輪呼叫工具再回答、一輪附上圖片的問題，每個時刻都畫得出來；用真的 tokenizer 和 chat template（在 `models/` 裡時），伺服器拿到的就是脈絡分頁上的 prompt；設了 `VLLM_TEST_URL` 時問真的伺服器，伺服器加了哪些參數就測哪些 | | `test_vllm_docker.py` | 在 Docker 裡啟動 vLLM，Docker 在 `subprocess` 這一層換成假的：先照固定的 commit 下載權重，再用模型、它的 revision 和 `models/` 以 `docker compose up` 跑 `docker-compose/compose.yaml`，Docker 有 nvidia runtime（或 `AGENT_LOOP_VLLM_DEVICE` 指定）時再疊上 `compose.gpu.yaml`；模型已經在跑的伺服器不會再啟動一次，換模型會換掉它；沒有 Docker、Docker 沒在執行、沒有 Compose、`docker compose up` 失敗、容器因為記憶體不夠被停掉、容器結束（列出記錄裡的例外，原因在前）和伺服器一直沒回應，都用兩種語言說明；Compose 還在忙時頁面重跑，也會停掉它；`docker compose down`；程式要求的模型在啟動失敗後還記著，直到停掉為止；結束程式時停掉它啟動的伺服器（沒有的話不碰 Docker）；在 GPU 上 MTP 用 Gemma 照固定 commit 的草稿模型（先下載）或 Qwen 自己的 MTP 層，在 CPU 上或沒要求時都不開，切換時重新啟動伺服器。compose 檔：只聽 127.0.0.1、`models/` 唯讀又離線、預設是固定 commit 的 Qwen3.5、GPU 版是同一個 vLLM、除了只給 CPU 的那個以外參數都一樣，再加上和 CPU 一樣大的 KV cache、偏低的 `--gpu-memory-utilization`，以及手動啟動時不開的推測解碼（有設定時統計才設成 detailed，沒有設定時 vLLM 不收），Docker Compose 也把網頁給的設定原封不動交給 vLLM，Docker Compose 讀得懂兩個檔、掛載的是相對的 `AGENT_LOOP_MODELS` 指的資料夾（有裝的話）；設了 `RUN_VLLM_DOCKER_TESTS=1` 時，Docker 裡真的伺服器透過 `VLLMBackend` 回答，Docker 有 GPU 時先開 MTP、再不開各一次 | @@ -333,6 +358,9 @@ RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real - `vllm_backend.py` 照 vLLM 0.32 原始碼裡的協定寫，測試用的是說這些協定的假伺服器、真的 tokenizer 和真的網頁；`/inference/v1/generate`、它的 sampling mask 和每一步的統計在 vLLM 裡標示為實驗性功能，之後可能會改。對真伺服器的測試（`VLLM_TEST_URL`）只在一種 GPU 上跑過：DGX Spark 的 GB10。 - 在 Mac 上，Docker 裡的 vLLM 是用 CPU 跑，而且慢：Gemma 4 E2B 解碼約 4.6 tok/s，Docker 要有大約 20 GB 以上的記憶體，第一次載入還要花大約 10 分鐘跑它的媒體編碼器，看伺服器收哪些媒體。`compose.gpu.yaml` 只在一種 NVIDIA GPU 上跑過：DGX Spark 的 GB10。CPU 版的 image 修改了 vLLM 0.32.0（`docker-compose/vllm-cpu/patch_vllm.py`）；新版的 vLLM 可能兩個修正都不需要，要修改的程式和寫它時不一樣的話，建 image 時會停下來。 - 用 Ctrl+C 或 SIGTERM 結束程式時，會停掉 vLLM 的容器。用別的方式結束（關掉終端機、`kill -9`）的話，容器會繼續跑、佔著模型，要用 `docker compose -p agent-loop-vllm down` 停掉。 +- 用 pi 時，網頁只看得到 pi 回報的東西：沒有 token、機率和特殊 token，也看不到 pi 送出的 prompt。token 數是 provider 報的，tok/s 是回覆傳回來時量的（含網路），費用是照 pi 目錄裡的價格估的。pi 的模型是 provider 的：要花錢，對話也會送到那個 provider。 +- pi 的工具在這台電腦上執行，沒有沙箱；打開 bash 的話，你做得到的事模型都做得到。打開「執行工具前先問我」，每個呼叫都會先給你看。 +- `pi_harness.py` 照 pi 1.4 的 RPC 協定寫，根據它的文件和 pi 1.4.0 實際送出的紀錄；真 pi 的測試只用 pi 1.4.0 跑過。pi 1.4 需要 Node.js 22.19 以上：Node 太舊的話它啟動不了，網頁會顯示它說了什麼。網頁最多每分鐘問一次 pi 有哪些模型；在 pi 裡登入 provider 之後，按「再試一次」就會列出來。 - `MODEL_ID` 環境變數決定一開始選哪個模型，也可以指定選單裡沒有的 checkpoint，只要它屬於已經支援的家族（Gemma 4 或 Qwen3.5）；其他家族要先照「支援更多模型」加一個 family。 ## 授權 diff --git a/app/assets/style.css b/app/assets/style.css index fa69c62..0f4caeb 100644 --- a/app/assets/style.css +++ b/app/assets/style.css @@ -139,6 +139,9 @@ .tok:hover { outline: 1.5px solid var(--ink); } .tok.ch-thought { font-style: italic; color: var(--muted); } .tok.ch-tool_call { font-family: var(--mono); font-size: 12.5px; color: var(--c-harness); } +/* Text streamed by a harness that sees no tokens (pi): the same channels, without token boundaries. */ +.run.ch-thought { font-style: italic; color: var(--muted); } +.run.ch-tool_call { font-family: var(--mono); font-size: 12.5px; color: var(--c-harness); } .tok.special, .sp { font: 600 11px/1 var(--mono); font-style: normal; diff --git a/app/guide.py b/app/guide.py index c1aea71..1b21920 100644 --- a/app/guide.py +++ b/app/guide.py @@ -10,7 +10,7 @@ switch before, or else their browser's. On every push to main, .github/workflows/pages.yml runs tests/test_guide.py, then builds and publishes them. -The content points at code by text, not by line number: Code("harness/agent.py", "def run(self):") is +The content points at code by text, not by line number: Code("harness/harnesses/simple_harness.py", "def run(self):") is looked up in the file every time the page is built, so the line numbers on the page follow the code when it changes, and each file:line chip and excerpt links to its lines on GitHub at the commit the page was built from. A start text must match exactly one line of its file; tests/test_guide.py @@ -417,7 +417,7 @@ def build(out: Path) -> list[Path]: "zh": "從你按下送出到看到回覆，中間發生的每一件事，照實際執行的順序走一遍。app 的側欄可以選 Gemma 4 E2B 或 Qwen3.5 0.8B，" "兩個模型跑的是同一個迴圈。問題可以附上圖片、聲音或影片（{{media}}），打開「讓模型寫程式」時，" "模型還能在 Docker 沙箱裡執行 Python（{{execute}}）。模型可以在這台電腦上用 transformers 跑，也可以交給一台 vLLM 伺服器（{{backends}}）。" - "兩個模型或兩種跑法不一樣的地方，分成頁籤（tab）各自說明，一個模型或一種 backend 一個；在任何一段選了一個，整頁同一類的頁籤都跟著換。每段都附上做這件事的程式碼；`harness/agent.py:98` 這類標記是檔案的路徑和行號，" + "兩個模型或兩種跑法不一樣的地方，分成頁籤（tab）各自說明，一個模型或一種 backend 一個；在任何一段選了一個，整頁同一類的頁籤都跟著換。每段都附上做這件事的程式碼；`harness/harnesses/simple_harness.py:99` 這類標記是檔案的路徑和行號，" "滑過去會顯示那一行，點旁邊的 GitHub 圖示會在新分頁打開那個檔案、跳到那一行，程式片段標題上的圖示也一樣。" "行號是產生這一頁時從原始碼查出來的，連結也指向那個 commit；每次推上 main，GitHub Actions 都會重新產生，程式改了也會跟著更新。", "en": "Everything that happens between pressing send and seeing the reply, in the order it really runs. The app's sidebar picks " @@ -426,7 +426,7 @@ def build(out: Path) -> list[Path]: "The model runs on this machine with transformers, or on a vLLM server ({{backends}}). Where the two models, or the two ways of running " "them, differ, a section tells each in a tab of its own, one per model or per backend, and picking one in any section switches every tab " "of that kind on the page. " - "Every part comes with the code that does it; a mark like `harness/agent.py:98` is a file's path and a line number, hovering over it shows that line, " + "Every part comes with the code that does it; a mark like `harness/harnesses/simple_harness.py:99` is a file's path and a line number, hovering over it shows that line, " "and the GitHub icon beside it opens the file at that line in a new tab, as the icon on an excerpt's title does for its lines. " "The line numbers are looked up in the source when this page is built, and the links go to that commit; GitHub Actions rebuilds it on every push to main, so they follow the code as it changes.", } @@ -434,8 +434,8 @@ def build(out: Path) -> list[Path]: SECTIONS = [ Section("overview", "ui", {"zh": "總覽：兩條 thread、一張事件列表", "en": "Overview: two threads, one event list"}, - {"zh": "你送出一個問題就是一輪（一個 `Run`）；一輪裡是一個迴圈，每一圈叫一個 step。", - "en": "Each question you send is one turn (one `Run`); inside a turn is a loop, and each time around it is a step."}, ( + {"zh": "你送出一個問題就是一輪（一個 `SimpleHarness`）；一輪裡是一個迴圈，每一圈叫一個 step。", + "en": "Each question you send is one turn (one `SimpleHarness`); inside a turn is a loop, and each time around it is a step."}, ( {"zh": "每個 step 做同一件事：harness 把整段對話 render 成一個字串（脈絡）→ 模型一次推論、一個 token 一個 token 生成 → " "harness 解析輸出。有工具呼叫，就驗證、（需要時問你）、在環境執行，把結果接回對話，進下一圈；" "沒有工具呼叫，就把回覆交給你，這一輪結束。", @@ -448,13 +448,19 @@ def build(out: Path) -> list[Path]: "en": "The app runs on two threads. Streamlit's script thread draws the page; the agent runs on a background thread " "([[app/page.py|self.thread = threading.Thread]]). The two talk through just three things:"}, ( - {"zh": "agent 每走到一個階段，就把一個事件 append 到 `events` 列表（[[harness/agent.py|def emit(self, event_type]]）；畫面只負責把這張列表畫出來。", - "en": "At every stage the agent appends an event to the `events` list ([[harness/agent.py|def emit(self, event_type]]); the page only draws that list."}, - {"zh": "按「停止」時，畫面呼叫 `Run.stop()`（[[harness/agent.py|def stop(self):]]）。", - "en": "Pressing Stop calls `Run.stop()` ([[harness/agent.py|def stop(self):]])."}, - {"zh": "按「允許」或「拒絕」時，畫面呼叫 `Run.decide()`（[[harness/agent.py|def decide(self, call_id]]）。", - "en": "Pressing Allow or Reject calls `Run.decide()` ([[harness/agent.py|def decide(self, call_id]])."}, + {"zh": "agent 每走到一個階段，就把一個事件 append 到 `events` 列表（[[harness/harnesses/simple_harness.py|def emit(self, event_type]]）；畫面只負責把這張列表畫出來。", + "en": "At every stage the agent appends an event to the `events` list ([[harness/harnesses/simple_harness.py|def emit(self, event_type]]); the page only draws that list."}, + {"zh": "按「停止」時，畫面呼叫 `SimpleHarness.stop()`（[[harness/harnesses/simple_harness.py|def stop(self):]]）。", + "en": "Pressing Stop calls `SimpleHarness.stop()` ([[harness/harnesses/simple_harness.py|def stop(self):]])."}, + {"zh": "按「允許」或「拒絕」時，畫面呼叫 `SimpleHarness.decide()`（[[harness/harnesses/simple_harness.py|def decide(self, call_id]]）。", + "en": "Pressing Allow or Reject calls `SimpleHarness.decide()` ([[harness/harnesses/simple_harness.py|def decide(self, call_id]])."}, ), + {"zh": "這三樣就是畫面對「跑迴圈的那一層」的全部要求，`Harness`（[[harness/harnesses/harness.py|class Harness(Protocol):]]）。" + "`SimpleHarness` 是這個專案自己寫的那一個；側欄的「用什麼跑迴圈」選 pi 時，換成 `PiHarness`，把迴圈交給 pi 這個 coding agent（{{pi}}）。" + "接下來各段講的都是 `SimpleHarness`。", + "en": "These three are all the page asks of whatever runs the loop, the `Harness` ([[harness/harnesses/harness.py|class Harness(Protocol):]]). " + "`SimpleHarness` is this project's own; with pi picked under What runs the loop in the sidebar, `PiHarness` takes its place and hands the loop " + "to pi, a coding agent ({{pi}}). The sections that follow are about `SimpleHarness`."}, {"zh": "專案分成兩塊。`harness/` 跑一輪：迴圈、工具、模型，以及跑模型的那一層，它從不 import Streamlit。" "`app/` 是你看到的東西，站在 harness 上面：開始一輪、把 harness 送出的事件畫出來、把你的點擊傳回去。" "根目錄的 `streamlit_app.py` 只負責啟動 `app/page.py`。`harness/` 裡每個檔案的分工：", @@ -462,8 +468,12 @@ def build(out: Path) -> list[Path]: "`app/` is what you see, on top of the harness: it starts a turn, draws the events the harness sends and passes your clicks back. " "`streamlit_app.py` at the top only starts `app/page.py`. What each file in `harness/` does:"}, ( - {"zh": "`harness/agent.py`：harness 的迴圈本體 `Run.run()`（[[harness/agent.py|def run(self):]]）。", - "en": "`harness/agent.py`: the harness's loop itself, `Run.run()` ([[harness/agent.py|def run(self):]])."}, + {"zh": "`harness/harnesses/`：跑迴圈的那一層。`harness.py` 是畫面對它的要求 `Harness`，以及每個 harness 送出的事件；" + "`simple_harness.py` 是這個專案的 harness，迴圈本體 `SimpleHarness.run()`（[[harness/harnesses/simple_harness.py|def run(self):]]）；" + "`pi_harness.py` 把迴圈交給 pi，`pi_approve.ts` 是讓網頁在 pi 執行工具前先問你的 pi extension（{{pi}}）。", + "en": "`harness/harnesses/`: what runs the loop. `harness.py` is what the page asks of it, `Harness`, and the events every harness sends; " + "`simple_harness.py` is this project's harness, with the loop itself, `SimpleHarness.run()` ([[harness/harnesses/simple_harness.py|def run(self):]]); " + "`pi_harness.py` hands the loop to pi, and `pi_approve.ts` is the pi extension through which the page asks you before pi runs a tool ({{pi}})."}, {"zh": "`harness/backends/backend.py`：harness 和畫面對「跑模型的那一層」的全部要求，`Backend`（[[harness/backends/backend.py|class Backend(Protocol):]]）：" "`render()`、`encode()`、`generate()`，以及 token 事件裡哪些欄位一定有、哪些要 backend 知道才有。" "`simple_backend.py` 的 `SimpleBackend` 是其中一種，`vllm_backend.py` 是另一種（{{backends}}）。", @@ -843,14 +853,16 @@ def build(out: Path) -> list[Path]: "`load_llm()` 用 `st.cache_resource(max_entries=1)` 快取，記憶體裡同時只留一個模型。換模型或換「用什麼跑模型」時，" "`release_model()` 先清掉快取、把記憶體還給系統，下一個模型才載入，兩個不會同時佔著記憶體。Streamlit 的 fast reruns 也關掉了（`.streamlit/config.toml`）：重跑會等正在跑的那一次到下一個 Streamlit 呼叫，而不是在新的 thread 裡和它同時跑，所以模型載入到一半時切換的設定，要等載入完才生效，不會在放掉之後才又載入完、留在記憶體裡。" "第一次用到某個模型時，把它建議的 temperature、top-p、top-k 寫進側欄；Qwen3.5 的 checkpoint 沒附 generation_config.json，建議值來自 family" - "（[[harness/backends/simple_backend.py|def _sampling_defaults(self)]]）。側欄的模型選單換了之後會開新對話（[[app/page.py|key=\"model_id\", on_change=switch_model]]）。", + "（[[harness/backends/simple_backend.py|def _sampling_defaults(self)]]）。側欄的模型選單換了之後會開新對話（[[app/page.py|key=\"model_id\", on_change=switch_model]]）。" + "側欄的「用什麼跑迴圈」選了 pi 時，這台電腦不載入模型，改由 `pi_picker()` 問 pi 有哪些模型（{{pi}}）。", "en": "On every rerun, `current_llm()` first gets a `Backend` for the model picked in the sidebar, and the rest of the page takes it for just that ({{backends}}). " "`load_llm()` is cached with `st.cache_resource(max_entries=1)`, so only one model stays in memory. Switching models, or what runs them, first has " "`release_model()` clear that cache and hand the memory back, and only then does the next model load, so the two never hold memory at once. " "Streamlit's fast reruns are off too (`.streamlit/config.toml`): a rerun waits for the run in progress to reach its next Streamlit call rather than running beside it in a new thread, so a switch made while a model loads takes effect once it has loaded, and the model does not finish loading after it was let go. " "The first time a model is used, its recommended temperature, top-p and top-k go into the sidebar; the Qwen3.5 checkpoint ships no " "generation_config.json, so its recommendations come from the family ([[harness/backends/simple_backend.py|def _sampling_defaults(self)]]). " - "Switching models in the sidebar's menu starts a new conversation ([[app/page.py|key=\"model_id\", on_change=switch_model]])."}, + "Switching models in the sidebar's menu starts a new conversation ([[app/page.py|key=\"model_id\", on_change=switch_model]]). " + "With pi picked under What runs the loop, no model loads on this machine; `pi_picker()` asks pi which models it has instead ({{pi}})."}, Code("app/page.py", "@st.cache_resource(max_entries=1", end="def release_model():"), {"zh": "其他的看是用什麼跑模型：", "en": "The rest depends on what runs the model:"}, Tabs({ @@ -906,41 +918,45 @@ def build(out: Path) -> list[Path]: "language: the errors that the harness and the tools send the model are worded in it for the whole turn. `tools` are the tools " "the model is offered: `run_python` only when Let the model write code is on and Docker is ready ({{execute}})."}, Code("app/page.py", "def current_settings("), - {"zh": "`LiveRun` 建立 `Run`，把 `self.events.append` 當成 `emit` 傳進去，連同附件和這段對話的資料夾，然後開背景 thread。" + {"zh": "`harness_for()` 決定這一輪由誰跑：`SimpleHarness` 跑在記憶體裡的模型上，或是 `PiHarness`（{{pi}}）。兩種都帶著這段對話的歷史、附件和資料夾，" + "設定在這時候就固定下來。", + "en": "`harness_for()` decides what runs the turn: a `SimpleHarness` on the model in memory, or a `PiHarness` ({{pi}}). Either gets the conversation so far, " + "the attachments and the conversation's folder, and its settings are fixed now."}, + Code("app/page.py", "def harness_for("), + {"zh": "`LiveRun` 拿 `self.events.append` 當成 `emit` 建出這個 harness，然後開背景 thread。" "thread 跑完 `run.run()` 之後才在 `finally` 裡放開鎖，所以就算 agent 出錯，鎖也一定會還回去。", - "en": "`LiveRun` creates the `Run`, passing `self.events.append` as its `emit`, along with the attachments and the conversation's folder, " - "and starts the background thread. " + "en": "`LiveRun` makes that harness with `self.events.append` as its `emit`, and starts the background thread. " "The thread releases the lock in a `finally` only after `run.run()` returns, so the lock always comes back, even when the agent fails."}, Code("app/page.py", "class LiveRun:", caption={"zh": "`daemon=True`：關掉 server 時不用等 agent 跑完。", "en": "`daemon=True`: stopping the server does not wait for the agent to finish."}), )), Section("prepare", "harness", - {"zh": "Run 準備：訊息、工具、system prompt", "en": "Preparing a run: messages, tools, system prompt"}, + {"zh": "準備一輪：訊息、工具、system prompt", "en": "Preparing a run: messages, tools, system prompt"}, {"zh": "`self.messages` 是 agent 的全部記憶，之後每一步都把它整份重新 render。", "en": "`self.messages` is all the agent remembers; every step renders the whole of it again."}, ( - {"zh": "`Run.__init__` 把之前的對話歷史接上這次的問題，存成 `self.messages`（[[harness/agent.py|self.messages = [*history,]]）。" + {"zh": "`SimpleHarness.__init__` 把之前的對話歷史接上這次的問題，存成 `self.messages`（[[harness/harnesses/simple_harness.py|self.messages = [*history,]]）。" "之後的工具呼叫和工具結果都 append 在後面；模型本身不記得任何事，每一步看到的都是這份 list 重新 render 出來的字串。" "訊息用的是 Hugging Face 的通用格式（`role`、`content`、`tool_calls`），兩個家族都吃得下，格式差異交給各自的 chat template；" "附了檔案時，這則訊息的 `content` 是一串 part（{{media}}）。`self.tools` 是這一輪模型拿得到的工具，" "`self.files` 是這段對話的資料夾。`_stop` 和 `_decided` 是 UI thread 用來通知 agent 的兩個同步工具。", - "en": "`Run.__init__` appends this question to the conversation so far and keeps it as `self.messages` ([[harness/agent.py|self.messages = [*history,]]). " + "en": "`SimpleHarness.__init__` appends this question to the conversation so far and keeps it as `self.messages` ([[harness/harnesses/simple_harness.py|self.messages = [*history,]]). " "Tool calls and tool results are appended after it; the model itself remembers nothing, and what it sees in each step is this " "list rendered into a string again. The messages use Hugging Face's common format (`role`, `content`, `tool_calls`), which " "both families accept; each family's chat template takes care of its own format. With files attached, that message's `content` " "is a list of parts ({{media}}). `self.tools` are the tools the model gets this turn, and `self.files` is the conversation's folder. " "`_stop` and `_decided` are the two synchronization tools the UI thread signals the agent with."}, - Code("harness/agent.py", "def __init__(self, llm: Backend, history"), + Code("harness/harnesses/simple_harness.py", "def __init__(self, llm: Backend, history"), {"zh": "`run()` 一開始拿出這個模型的 family，決定這一輪模型看得到什麼：收集 `self.tools` 裡每個工具的 `schema()` 給 chat template" "（關掉「使用工具」時一個都沒有），" "另外留一份「工具名稱 → 參數 schema」給 parser（Qwen 的參數值是純文字，要靠 schema 才知道型別）；" - "system prompt 也分有工具（[[harness/agent.py|SYSTEM_WITH_TOOLS = ]]）和沒工具兩個版本。" - "第一個事件 `run_start` 帶著模型和 family，畫面就知道這一輪要用哪個模型的格式來畫。", + "system prompt 也分有工具（[[harness/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]]）和沒工具兩個版本。" + "第一個事件 `run_start` 帶著模型和 family，畫面就知道這一輪要用哪個模型的格式來畫；它也說這一輪是哪個 harness 跑的（`simple`，或 pi 的 `pi`）。", "en": "`run()` starts by taking the model's family and deciding what the model gets to see this turn: it collects the `schema()` " "of every tool in `self.tools` for the chat template (none with Use tools off), and keeps a map from tool name to parameters schema for the parser (Qwen's " "argument values are plain text, and only the schema says their types); the system prompt comes in two versions, with tools " - "([[harness/agent.py|SYSTEM_WITH_TOOLS = ]]) and without. The first event, `run_start`, carries the model and the family, so the page " - "knows whose format to draw the turn in."}, - Code("harness/agent.py", "def run(self):", end='stop_reason, final = "max_steps", ""', + "([[harness/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]]) and without. The first event, `run_start`, carries the model and the family, so the page " + "knows whose format to draw the turn in; it also says which harness ran the turn (`simple`, or pi's `pi`)."}, + Code("harness/harnesses/simple_harness.py", "def run(self):", end='stop_reason, final = "max_steps", ""', caption={"zh": "`stop_reason` 的預設值是 `max_steps`：迴圈跑完所有步數都沒有 break，停止原因就是步數上限。", "en": "`stop_reason` starts out as `max_steps`: when the loop uses up its steps without a break, the step limit is why it stopped."}), {"zh": "工具的 schema 就是一份 JSON Schema，包成 Hugging Face chat template 認得的 function 格式：", @@ -953,7 +969,7 @@ def build(out: Path) -> list[Path]: "en": "The chat template joins the system prompt, the tools and the conversation into one long string; it is all the model can see in this step."}, ( {"zh": "每一圈從 `for step in range(1, s.max_steps + 1)` 開始，側欄的「步數上限」就是這個 range 的上界。第一件事是 render：", "en": "Each step starts at `for step in range(1, s.max_steps + 1)`; the sidebar's Max steps is the top of that range. The first thing is to render:"}, - Code("harness/agent.py", "for step in range(1, s.max_steps + 1):", end="roles=_role_counts([system, *self.messages])", + Code("harness/harnesses/simple_harness.py", "for step in range(1, s.max_steps + 1):", end="roles=_role_counts([system, *self.messages])", caption={"zh": "`step_start` 讓畫面開一張新的 step 卡片；`context` 帶著模型實際讀到的完整字串送出。", "en": "`step_start` has the page open a new step card; `context` goes out with the whole string the model really reads."}), {"zh": "`render()` 呼叫 tokenizer 自帶的 chat template，模型在 vLLM 上跑時也是在這裡 render（{{files}}）。格式是模型自己的，harness 不用知道：", @@ -980,19 +996,19 @@ def build(out: Path) -> list[Path]: ), }), Code("harness/backends/simple_backend.py", "def render(self, messages"), - {"zh": "接著 `encode()` 把字串變成模型真正讀的 token（[[harness/agent.py|encoded = self.llm.encode(]]）。只有文字時，就是 tokenizer 切出來的 token；" + {"zh": "接著 `encode()` 把字串變成模型真正讀的 token（[[harness/harnesses/simple_harness.py|encoded = self.llm.encode(]]）。只有文字時，就是 tokenizer 切出來的 token；" "對話裡有圖片、聲音或影片時，chat template 在它們的位置只寫了一個佔位 token，`encode()` 把它展開成那個檔案佔的幾十到幾千個 token" "（{{media}}），所以 `context` 送出的是展開後的字串。`n_tokens` 是它的長度，畫面拿它和脈絡上限比，顯示用掉幾 %。" "`added_from` 是這一步新加入的內容從哪裡開始：先算 render 出來的字串和 `prev_sequence` 共同開頭的長度（{{record}}會說 `prev_sequence` 是什麼），" "再加上媒體展開多出來的長度：媒體都在使用者訊息裡，在新加入的內容前面。第 2 步以後，新加入的正好是工具結果。", - "en": "Next `encode()` turns the string into the tokens the model really reads ([[harness/agent.py|encoded = self.llm.encode(]]). With text " + "en": "Next `encode()` turns the string into the tokens the model really reads ([[harness/harnesses/simple_harness.py|encoded = self.llm.encode(]]). With text " "alone, they are the tokenizer's tokens; when the conversation has images, sound or video, the chat template wrote a single " "placeholder token where each goes, and `encode()` expands it into the tens to thousands of tokens that file takes ({{media}}), " "so `context` goes out with the expanded string. `n_tokens` is its length, which the page compares with the context limit to " "show how much of it is used. `added_from` is where what is new in this step begins: the length of the start that the rendered " "string shares with `prev_sequence` ({{record}} says what `prev_sequence` is), plus however much longer the media made it, since " "the media are all in user messages, before what is new. From step 2 on, what is new is exactly the tool result."}, - Code("harness/agent.py", "def common_prefix(", + Code("harness/harnesses/simple_harness.py", "def common_prefix(", caption={"zh": "逐字元比較，回傳兩個字串共同開頭的長度。", "en": "Compares character by character and returns the length of the start the two strings share."}), {"zh": "畫面上的「脈絡」分頁把 `context` 事件裡的字串原樣印出來（[[app/render.py|def context_html(]]），後面接著模型這一步寫的輸出；" "`added_from` 之後的部分標成「這一步新加入」（第 1 步沒有上一步，從最新一則使用者訊息開始標），family 列出的特殊 token 換成可以滑過看說明的深色標籤；" @@ -1106,7 +1122,7 @@ def build(out: Path) -> list[Path]: "最後的 `done` 留下來，改名成 `model_end` 送出。", "en": "The harness consumes the events that `generate()` yields in a for loop: `prefill` and each `token` are passed on with the step " "number, and the final `done` is kept and sent on as `model_end`."}, - Code("harness/agent.py", "done = None", end='self.emit("model_end"', + Code("harness/harnesses/simple_harness.py", "done = None", end='self.emit("model_end"', caption={"zh": "傳進去的 `self._stop.is_set` 是一個函式：模型每生成一個 token 前都會呼叫它，問「要停了嗎？」。", "en": "The `self._stop.is_set` passed in is a function: the model calls it before every token to ask whether to stop."}), Tabs({ @@ -1389,10 +1405,10 @@ def build(out: Path) -> list[Path]: "The tests check that `SimpleBackend`, `VLLMBackend` and the tests' `FakeLLM` are each a `Backend`."}), {"zh": "畫面只在一個地方決定用哪一個：`load_llm()` 拿到伺服器的網址（`VLLM_URL`，或畫面在 Docker 裡啟動的伺服器）就用 vLLM，沒有就在這台電腦上跑" "（[[app/page.py|llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id)]]）。" - "之後 `current_llm()`、側欄和 `Run`（{{prepare}}）拿到的都只是一個 `Backend`；它放在 `st.cache_resource` 裡，換模型或換「用什麼跑模型」才換掉（{{submit}}）。", + "之後 `current_llm()`、側欄和 `SimpleHarness`（{{prepare}}）拿到的都只是一個 `Backend`；它放在 `st.cache_resource` 裡，換模型或換「用什麼跑模型」才換掉（{{submit}}）。", "en": "The page decides which one in a single place: given a server's URL (`VLLM_URL`, or that of the server the page started in Docker), `load_llm()` " "uses vLLM, and without one runs the model here ([[app/page.py|llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id)]]). " - "From then on `current_llm()`, the sidebar and `Run` ({{prepare}}) get just a `Backend`; it stays in `st.cache_resource` until the model, or what " + "From then on `current_llm()`, the sidebar and `SimpleHarness` ({{prepare}}) get just a `Backend`; it stays in `st.cache_resource` until the model, or what " "runs it, changes ({{submit}})."}, {"zh": "兩種 backend 對模型知道的不一樣多，差別全在 `token` 事件裡。`TOKEN_FIELDS` 每個 backend 都要送：位置、token 的 id 和 piece、它加上的文字、" "是不是特殊 token、正在寫思考、工具呼叫還是回覆。`TOKEN_DETAILS` 則是知道才送：機率、排名、取樣時的機率、剩下的候選、前 5 名，以及 MTP 草稿猜了什麼。", @@ -1480,7 +1496,7 @@ def build(out: Path) -> list[Path]: "en": "The chat template sometimes opens the thinking at the end of the prompt (the family's `thought_prefix`); that part is in the " "prompt, not in `raw`, so it is put back before the output goes to the family's `parse`, with `s.lang` as the language for its " "error messages. With Use tools off, a tool call the model writes anyway counts as none."}, - Code("harness/agent.py", "prefix = family.thought_prefix if", end='self.emit("parsed"'), + Code("harness/harnesses/simple_harness.py", "prefix = family.thought_prefix if", end='self.emit("parsed"'), {"zh": "兩個 parser 回傳一樣的結構：`thinking`、`tool_calls`（每個有 `name`、`arguments`、`error`）、`content`，所以後面的 harness 不用管是哪個模型。", "en": "Both parsers return the same structure, `thinking`, `tool_calls` (each with `name`, `arguments` and `error`) and `content`, " "so the rest of the harness does not care which model it was."}, @@ -1530,14 +1546,14 @@ def build(out: Path) -> list[Path]: "en": "What counts here is whether parsing found a tool call, not which token it stopped at. Gemma stopping at `<|tool_response>` does " "mean it wants a tool, but Qwen stops at `<|im_end|>` both after an answer and for a tool; deciding by the parse lets both " "families share this code."}, - Code("harness/agent.py", 'if done["finish"] == "user_stopped":', end="if not calls:"), + Code("harness/harnesses/simple_harness.py", 'if done["finish"] == "user_stopped":', end="if not calls:"), {"zh": "其他停止條件分散在迴圈各處：", "en": "The other ways to stop are spread around the loop:"}, ( {"zh": "for 迴圈跑完所有步數都沒 break → `max_steps`（{{prepare}}設定的預設值）。", "en": "The for loop uses up its steps without a break → `max_steps` (the default set in {{prepare}})."}, - {"zh": "等待確認時被停止或逾時 → `user_stopped` 或 `approval_timeout`（[[harness/agent.py|stop_reason = \"approval_timeout\" if self._timed_out]]）。", + {"zh": "等待確認時被停止或逾時 → `user_stopped` 或 `approval_timeout`（[[harness/harnesses/simple_harness.py|stop_reason = \"approval_timeout\" if self._timed_out]]）。", "en": "Stopped or timed out while waiting for a confirmation → `user_stopped` or `approval_timeout` " - "([[harness/agent.py|stop_reason = \"approval_timeout\" if self._timed_out]])."}, + "([[harness/harnesses/simple_harness.py|stop_reason = \"approval_timeout\" if self._timed_out]])."}, {"zh": "連續 `MAX_FORMAT_ERRORS`（3）次寫出無法解析的工具呼叫 → `format_errors`。", "en": "`MAX_FORMAT_ERRORS` (3) tool calls in a row that cannot be parsed → `format_errors`."}, {"zh": "backend 或 harness 本身丟出任何例外 → `error`，錯誤訊息會顯示在畫面上，而不是讓 thread 默默死掉；backend 的錯誤（例如 vLLM 伺服器的）兩種語言都有，畫面顯示目前語言的那一個。" @@ -1546,7 +1562,7 @@ def build(out: Path) -> list[Path]: "(a vLLM server's, say) comes in both languages, of which the page shows the one it is in. A tool's exception does not end the turn; it " "becomes an observation ({{execute}})."}, ), - Code("harness/agent.py", "if format_errors >= MAX_FORMAT_ERRORS:", end='self.emit("run_end"', + Code("harness/harnesses/simple_harness.py", "if format_errors >= MAX_FORMAT_ERRORS:", end='self.emit("run_end"', caption={"zh": "不管怎麼停，最後一定送出 `run_end`，帶著停止原因、最終回覆和整份訊息列表。", "en": "However it stops, `run_end` always goes out last, with the stop reason, the final reply and the whole message list."}), )), @@ -1580,7 +1596,7 @@ def build(out: Path) -> list[Path]: "`prev_sequence`; `generation_prompt=False`, since it is not the model's turn to write yet. Compared with it, what the next " "step's prompt has in addition is the tool result. Comparing with the last prompt plus the model's output would be off, " "because the template rewrites the model's part when it renders it again (it sorts the arguments and replaces the thinking markup)."}, - Code("harness/agent.py", 'ids = [f"call_{step}_{i}"', end="prev_sequence = self.llm.render("), + Code("harness/harnesses/simple_harness.py", 'ids = [f"call_{step}_{i}"', end="prev_sequence = self.llm.render("), )), Section("check", "harness", {"zh": "驗證與權限：模型從不直接碰環境", "en": "Validation and permission: the model never touches the environment"}, @@ -1590,7 +1606,7 @@ def build(out: Path) -> list[Path]: "參數不符合 schema，都不會執行。", "en": "Each call goes to `_handle_call()`. The first check is validation: a call that failed to parse, names no tool offered this turn " "(`run_python` without Let the model write code, say) or has arguments that do not fit the schema never runs."}, - Code("harness/agent.py", "def _handle_call(", end="args, errors = tool.validate(args, lang)"), + Code("harness/harnesses/simple_harness.py", "def _handle_call(", end="args, errors = tool.validate(args, lang)"), {"zh": "`Tool.validate()` 檢查必要參數有沒有給、有沒有多出不認得的參數、型別和 enum 對不對，然後補上預設值。" "錯誤訊息是完整的句子，用這一輪開始時頁面的語言寫，因為它會原封不動變成模型下一步讀到的觀察。", "en": "`Tool.validate()` checks that the required arguments are there, that there are no unknown ones, and that the types and enums " @@ -1602,18 +1618,18 @@ def build(out: Path) -> list[Path]: "en": "The second check records repeats: the tool name plus the arguments as sorted JSON is the key, compared with the earlier calls " "in this turn; an exact repeat is flagged in the step card as the loop maybe going in circles. It is only a warning and blocks " "nothing. Then the `tool_call` event goes out; with errors, the errors go straight back as the observation, without reaching the environment."}, - Code("harness/agent.py", 'key = (call["name"], json.dumps', end='{"error": "; ".join(errors)}'), + Code("harness/harnesses/simple_harness.py", 'key = (call["name"], json.dumps', end='{"error": "; ".join(errors)}'), {"zh": "第三關是你。開著「執行工具前先問我」時，harness 送出 `approval_request`，然後停在 `_wait_for_decision()` 等。" "拒絕、停止、逾時都會變成一則錯誤觀察，模型下一步只看得到這段文字。", "en": "The third check is you. With Ask me before running a tool on, the harness sends `approval_request` and waits in " "`_wait_for_decision()`. A rejection, a stop or a timeout each becomes an error observation, and that text is all the model sees in the next step."}, - Code("harness/agent.py", "if self.settings.approve_tools:", end='return self._observe(step, cid, call["name"], {"error": error}'), + Code("harness/harnesses/simple_harness.py", "if self.settings.approve_tools:", end='return self._observe(step, cid, call["name"], {"error": error}'), {"zh": "等待用的是 `threading.Condition`：每 0.5 秒醒來一次，直到你按下按鈕（`decide()` 會 `notify_all`）、按了停止，" - "或超過 `APPROVAL_TIMEOUT`（[[harness/agent.py|APPROVAL_TIMEOUT = ]]，5 分鐘）。逾時就當作拒絕並停止整輪，免得一個關掉的分頁把模型佔住。", + "或超過 `APPROVAL_TIMEOUT`（[[harness/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]]，5 分鐘）。逾時就當作拒絕並停止整輪，免得一個關掉的分頁把模型佔住。", "en": "The wait uses a `threading.Condition`: it wakes every 0.5 s until you press a button (`decide()` calls `notify_all`), press stop, " - "or `APPROVAL_TIMEOUT` passes ([[harness/agent.py|APPROVAL_TIMEOUT = ]], 5 minutes). A timeout counts as a rejection and stops the " + "or `APPROVAL_TIMEOUT` passes ([[harness/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]], 5 minutes). A timeout counts as a rejection and stops the " "whole turn, so a closed tab cannot hold on to the model."}, - Code("harness/agent.py", "def _wait_for_decision("), + Code("harness/harnesses/simple_harness.py", "def _wait_for_decision("), {"zh": "畫面這邊，等待中的呼叫會在聊天區顯示「允許／拒絕」按鈕，按下去就呼叫 `run.run.decide()`；" "像 `run_python` 的程式碼這種好幾行的參數，另外用程式碼區塊顯示，看得清楚再決定。" "「先問我」寫在 harness 的程式裡，而不是寫在 prompt 裡請模型遵守，所以模型沒有辦法繞過。", @@ -1633,7 +1649,7 @@ def build(out: Path) -> list[Path]: "conversation's folder. A `ToolError` from the tool is an expected failure (a mistyped timezone, a page that does not exist, " "code that fails), and its message goes to the model as it is, in the turn's language; any other exception is caught too and " "becomes `{\"error\": …}`."}, - Code("harness/agent.py", 'self.emit("tool_start"', end="return self._observe(step, cid, call[\"name\"], result, ok=ok"), + Code("harness/harnesses/simple_harness.py", 'self.emit("tool_start"', end="return self._observe(step, cid, call[\"name\"], result, ok=ok"), {"zh": "五個工具各自連到不同的環境：", "en": "The five tools reach different environments:"}, ( {"zh": "`get_current_time`：讀系統時鐘，用 IANA 時區名稱換算（[[harness/tools.py|def get_current_time(]]）。", @@ -1693,11 +1709,11 @@ def build(out: Path) -> list[Path]: "en": "In `_observe()` the tool result (or error) becomes a JSON string (a `ToolResult` with charts is split first: the model reads only " "its data, and the images go only into the event, for the page), cut at `MAX_RESULT_CHARS` (2000 characters) with " "`…(truncated)` added, so one long web page cannot flood the context; the `tool_result` event goes out, and the string is returned."}, - Code("harness/agent.py", "def _observe("), + Code("harness/harnesses/simple_harness.py", "def _observe("), {"zh": "回傳的字串變成一則 `role: tool` 的訊息 append 到 `self.messages`。同時更新連續格式錯誤的次數：這次是解析錯誤就加一，正常就歸零。", "en": "The returned string is appended to `self.messages` as a `role: tool` message. The count of format errors in a row is updated at " "the same time: one more for a parse error, back to zero otherwise."}, - Code("harness/agent.py", "for cid, call in zip(ids, calls):", end="if format_errors >= MAX_FORMAT_ERRORS:"), + Code("harness/harnesses/simple_harness.py", "for cid, call in zip(ids, calls):", end="if format_errors >= MAX_FORMAT_ERRORS:"), {"zh": "然後回到 for 迴圈開頭，下一圈 render 時 chat template 把工具結果寫進脈絡，這就是 step 卡片上「上一步的觀察接回脈絡」那一段：", "en": "Then it is back to the top of the for loop, and when the next step renders, the chat template writes the tool result into the " "context; this is what the step card shows as the last step's observation, back in the context:"}, @@ -1732,14 +1748,14 @@ def build(out: Path) -> list[Path]: Code("app/page.py", "@st.fragment(run_every=", end="viz_panel(llm)"), {"zh": "每次重畫，`LiveRun.data()` 複製一份目前的事件列表交給 `render.build()`，按順序折成一個個 step。" "`run_start` 告訴它這一輪是哪個模型、哪個 family，之後畫特殊 token、停止 token 和使用者訊息的位置都問這個 family；" - "`step_start` 開一個新的 step，`context`、`prefill`、`token`、`parsed` 填進去，`model_end` 帶來的推測解碼結果標出哪些 token 是猜中的" + "`step_start` 開一個新的 step，`context`、`prefill`、`token`、`parsed` 填進去（pi 跑的一輪沒有 family，也沒有前三個，換成串流進來的文字 `delta`，{{pi}}），`model_end` 帶來的推測解碼結果標出哪些 token 是猜中的" "（伺服器到這時才說，{{mtp}}），工具相關的事件依 `call_id` 歸到各自的呼叫。" "之後所有 HTML 都是這個結構（和頁面語言）的純函式，所以執行到一半的任何時刻都畫得出來；" "切換語言也只是用另一種語言重畫同一份事件，跑完的對話也會跟著換。", "en": "On every redraw, `LiveRun.data()` copies the event list so far and hands it to `render.build()`, which folds it into steps in " "order. `run_start` says which model and family the turn uses, and from then on the special tokens, the stop tokens and where " "the user's message starts are all asked of that family; `step_start` opens a new step, `context`, `prefill`, `token` and " - "`parsed` fill it in, `model_end` marks the tokens that were guesses when a server says so only then ({{mtp}}), and the tool events go to their calls by `call_id`. All the HTML after that is a pure function of this " + "`parsed` fill it in (a turn pi ran has no family and none of the first three, but `delta`, the text as it streams, {{pi}}), `model_end` marks the tokens that were guesses when a server says so only then ({{mtp}}), and the tool events go to their calls by `call_id`. All the HTML after that is a pure function of this " "structure (and of the page's language), so it can be drawn at any moment of a run; switching the language just redraws the " "same events in the other one, finished turns included."}, Code("app/render.py", "def build(events: list[dict])"), @@ -1751,23 +1767,73 @@ def build(out: Path) -> list[Path]: "en": "When a turn ends, `absorb_finished()` keeps the full message list from `run_end` as `history`. The next question is appended " "to it, so the agent remembers what was said before, earlier tool results included."}, Code("app/page.py", "def absorb_finished("), - {"zh": "停止按鈕呼叫 `Run.stop()`：設定 `_stop` 並叫醒正在等確認的 thread。解碼迴圈每個 token 前都會檢查 `_stop`，" + {"zh": "停止按鈕呼叫 `SimpleHarness.stop()`：設定 `_stop` 並叫醒正在等確認的 thread。解碼迴圈每個 token 前都會檢查 `_stop`，" "所以按下去之後最多再生成一個 token 就停。", - "en": "The stop button calls `Run.stop()`: it sets `_stop` and wakes a thread that is waiting for a confirmation. The decoding loop " + "en": "The stop button calls `SimpleHarness.stop()`: it sets `_stop` and wakes a thread that is waiting for a confirmation. The decoding loop " "checks `_stop` before every token, so after the press at most one more token is generated."}, - Code("harness/agent.py", "def stop(self):", end="def decide(self, call_id"), + Code("harness/harnesses/simple_harness.py", "def stop(self):", end="def decide(self, call_id"), + )), + Section("pi", "harness", + {"zh": "換一個 harness：pi", "en": "Another harness: pi"}, + {"zh": "同一張圖、同一套事件，迴圈換成 pi 跑：看得到它每一步做了什麼，看不到 token。", + "en": "The same diagram and the same events, with pi running the loop: what it does at each step shows, its tokens do not."}, ( + {"zh": "pi 是一個 coding agent，有它自己的迴圈、system prompt 和工具（read、bash、edit、write…），問的是它在 provider 那邊登入過的模型（Claude、GPT…）。" + "側欄的「用什麼跑迴圈」選 pi 時，每個問題都由 `PiHarness` 用 RPC 模式啟動一個 pi 子行程：指令從 stdin 送進去，紀錄從 stdout 讀出來，一行一個 JSON。" + "它先選好模型和思考等級、關掉 pi 自己的自動重試，再把問題當成 prompt 送出；每一個指令都等到 pi 回應才送下一個，因為 pi 是同時處理指令的。", + "en": "pi is a coding agent with its own loop, system prompt and tools (read, bash, edit, write…), and it asks a model it is logged in to at a provider (Claude, GPT…). " + "With pi picked under What runs the loop, `PiHarness` starts pi in RPC mode as a child process for each question: commands go in on stdin, " + "records come out on stdout, one JSON object per line. It picks the model and the thinking level and turns pi's own retries off, then sends the question " + "as a prompt, each command only after pi has answered the last, since pi handles commands concurrently."}, + Code("harness/harnesses/pi_harness.py", "self._pi = Process(self._args()", end="self._loop()"), + {"zh": "pi 的一個 turn（向模型送一次請求，加上回覆裡要執行的工具）就是頁面上的一個 step。`_on()` 把 pi 的紀錄翻成頁面的事件：", + "en": "One of pi's turns (one request to the model, and the tools its reply calls) is a step on the page. `_on()` turns pi's records into the page's events:"}, + ( + {"zh": "`turn_start` → `step_start`；pi 自己沒有步數上限，跑滿側欄的步數，`PiHarness` 就請 pi 中止，不再畫下一步。", + "en": "`turn_start` → `step_start`; pi has no step limit of its own, so once the sidebar's limit is reached `PiHarness` asks pi to abort and draws no further step."}, + {"zh": "`message_update` 裡一段段的文字、思考和工具呼叫的參數 → `delta`，帶著它是哪一種（channel）。", + "en": "the pieces of text, thinking and tool-call arguments in `message_update` → `delta`, with which of them it is (its channel)."}, + {"zh": "模型那則訊息的 `message_end` → `model_end`（provider 報的 token 數和 pi 估的費用）和 `parsed`（思考、回覆、工具呼叫）。", + "en": "the `message_end` of the model's message → `model_end` (the token counts the provider reported, and the cost by pi's prices) and `parsed` (the thinking, the reply, the tool calls)."}, + {"zh": "`tool_execution_start` → `tool_call` 和 `tool_start`，`tool_execution_end` → `tool_result`；`agent_settled` → `run_end`。", + "en": "`tool_execution_start` → `tool_call` and `tool_start`, `tool_execution_end` → `tool_result`; `agent_settled` → `run_end`."}, + ), + Code("harness/harnesses/pi_harness.py", "def _on(self, rec: dict) -> bool:"), + {"zh": "pi 問模型用的是 provider 的 API，拿回來的是文字，不是 token：沒有 token id、機率和候選，也不知道模型實際讀到什麼字串；pi 也不說它送出的 prompt。" + "所以 pi 的 step 沒有脈絡段，模型段畫的是一段段傳回來的文字，加上 provider 報的數字（[[app/render.py|def streamed_html(]]），" + "就像 backend 沒回報的 token 細節，畫面寧可少畫，也不拿別的數字湊（{{backends}}）。", + "en": "pi asks the model through its provider's API, which sends back text, not tokens: no token ids, probabilities or candidates, and nothing of the " + "string the model really read; nor does pi say which prompt it sends. So a step of pi's has no context section, and its model section is the text " + "as it came back, with the numbers the provider reported ([[app/render.py|def streamed_html(]]); as with the token details a backend does not report, " + "the page would rather show less than fill in a number of its own ({{backends}})."}, + {"zh": "pi 執行工具前不會問人。打開「執行工具前先問我」時，`PiHarness` 讓 pi 多載入一個很小的 extension：pi 每次要執行工具，它就開一個確認對話框。" + "在 RPC 模式下，這個對話框是 stdout 上的一筆 `extension_ui_request`，`PiHarness` 把它變成 `approval_request`，你按的「允許」或「拒絕」再從 stdin 送回去；" + "被拒絕的呼叫不執行，模型讀到的結果就是拒絕的原因，用這一輪開始時頁面的語言寫。", + "en": "pi does not ask anyone before it runs a tool. With Ask me before running a tool on, `PiHarness` has pi load one small extension, which opens a confirm " + "dialog every time pi is about to run a tool. In RPC mode the dialog is an `extension_ui_request` on stdout, which `PiHarness` turns into an " + "`approval_request`, and your Allow or Reject goes back on stdin; a rejected call does not run, and the result the model reads is the refusal, in the " + "language the page was in when the turn started."}, + Code("harness/harnesses/pi_approve.ts", 'pi.on("tool_call"', end="return undefined;"), + {"zh": "pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：不像 `run_python` 在沒有網路、唯讀的 Docker 容器裡（{{execute}}），bash 碰得到你碰得到的一切。" + "網頁啟動 pi 時不載入你自己的 extension、skill、prompt 範本和 AGENTS.md，畫出來的就是 pi 本身的迴圈；模型和登入資訊還是你的。" + "pi 把對話存在這段對話資料夾裡的 session 檔，下一個問題用同一個 session 啟動 pi，模型就讀得到前面的全部。" + "另一個會把自己的迴圈報成事件的 agent，例如 Claude Agent SDK 或 Codex，也可以照這樣再寫一個 harness。", + "en": "pi's tools run on this computer, in the conversation's folder, without a sandbox: unlike `run_python` in a Docker container with no network and " + "read-only files ({{execute}}), bash reaches whatever you can. The page starts pi without your own extensions, skills, prompt templates and AGENTS.md, " + "so what it draws is pi's own loop; the models and the credentials are still yours. pi keeps the conversation in a session file in the " + "conversation's folder, and the next question starts pi on the same session, so the model reads all of it. Another agent that reports its loop as " + "events, such as the Claude Agent SDK or Codex, could be one more harness written the same way."}, )), ] # (event, file, text on the line that sends it, what it carries, what the page does with it) EVENTS = [ - ("run_start", "harness/agent.py", '"run_start", run_id=', - {"zh": "模型和 family、這一輪的設定、可用的工具", "en": "the model and family, the turn's settings, the available tools"}, + ("run_start", "harness/harnesses/simple_harness.py", '"run_start", run_id=', + {"zh": "哪個 harness 跑的、模型和 family（pi 沒有）、這一輪的設定、可用的工具", "en": "which harness ran it, the model and family (none from pi), the turn's settings, the available tools"}, {"zh": "用哪個模型的格式畫這一輪；步數上限、解碼說明", "en": "whose format to draw the turn in; the step limit, the decoding notes"}), - ("step_start", "harness/agent.py", 'self.emit("step_start"', + ("step_start", "harness/harnesses/simple_harness.py", 'self.emit("step_start"', {"zh": "step 編號", "en": "the step number"}, {"zh": "開一張新的 step 卡片；迴圈圖亮起「脈絡」", "en": "opens a new step card; Context lights up in the loop diagram"}), - ("context", "harness/agent.py", '"context",', + ("context", "harness/harnesses/simple_harness.py", '"context",', {"zh": "模型讀到的完整 prompt（媒體已展開）、token 數、`added_from`、每種媒體佔的 token 數、脈絡上限、各角色訊息數", "en": "the whole prompt as the model reads it (media expanded), its token count, `added_from`, the tokens each kind of media takes, " "the context limit, the number of messages per role"}, @@ -1781,32 +1847,36 @@ def build(out: Path) -> list[Path]: "en": "id, piece, text, channel, whether it is special, probability, rank, the top 5 candidates; with MTP, also `draft` (what the drafter guessed and whether it was kept). " "The fields from the probability on are there only when the backend knows them: `simple_backend.py` knows them all, a vLLM server the probability and the top 5, and more as its flags allow"}, {"zh": "token 的熱度顏色、底線與浮窗、左邊逐字出現的回覆", "en": "the token's heat color, underline and popover; the reply appearing bit by bit on the left"}), - ("model_end", "harness/agent.py", 'self.emit("model_end"', + ("delta", "harness/harnesses/pi_harness.py", 'self.emit("delta"', + {"zh": "模型輸出的一段文字和它的 channel（思考、回覆、工具呼叫）；只有看不到 token 的 harness 送，也就是 pi", + "en": "a piece of the model's output and its channel (thinking, reply, tool call); only from a harness that sees no tokens, which is pi"}, + {"zh": "step 卡片模型段一段段出現的文字、左邊逐字出現的回覆", "en": "the text appearing in the step card's model section, the reply appearing bit by bit on the left"}), + ("model_end", "harness/harnesses/simple_harness.py", 'self.emit("model_end"', {"zh": "停止原因、停止 token、token 數、tok/s、MTP 統計（vLLM 伺服器還可能說哪些 token 是猜中的）、原始輸出", "en": "why it stopped, the stop token, the token count, tok/s, the MTP counts (from a vLLM server, perhaps which tokens were guesses), the raw output"}, {"zh": "「停在 …」、tok/s、MTP 收下幾個、模型分頁的表格", "en": "“stopped at …”, tok/s, how many MTP guesses were kept, the table in the Model tab"}), - ("parsed", "harness/agent.py", 'self.emit("parsed"', + ("parsed", "harness/harnesses/simple_harness.py", 'self.emit("parsed"', {"zh": "思考、回覆、工具呼叫", "en": "the thinking, the reply, the tool calls"}, {"zh": "Harness 段開始出現", "en": "the Harness section appears"}), - ("tool_call", "harness/agent.py", '"tool_call",', + ("tool_call", "harness/harnesses/simple_harness.py", '"tool_call",', {"zh": "名稱、驗證後的參數、錯誤、是否重複、環境", "en": "the name, the validated arguments, errors, whether it repeats, the environment"}, {"zh": "Harness 段的呼叫、驗證結果、重複警告", "en": "the call, the validation result and the repeat warning in the Harness section"}), - ("approval_request", "harness/agent.py", 'self.emit("approval_request"', + ("approval_request", "harness/harnesses/simple_harness.py", 'self.emit("approval_request"', {"zh": "等待確認的呼叫", "en": "the call waiting for a confirmation"}, {"zh": "聊天區的「允許／拒絕」按鈕；迴圈圖閃爍", "en": "the Allow and Reject buttons in the chat; the loop diagram pulses"}), - ("approval", "harness/agent.py", 'self.emit("approval",', + ("approval", "harness/harnesses/simple_harness.py", 'self.emit("approval",', {"zh": "允許、拒絕、逾時或被停止", "en": "allowed, rejected, timed out or stopped"}, {"zh": "Harness 段顯示你的決定", "en": "the Harness section shows your decision"}), - ("tool_start", "harness/agent.py", 'self.emit("tool_start"', + ("tool_start", "harness/harnesses/simple_harness.py", 'self.emit("tool_start"', {"zh": "工具名稱、環境", "en": "the tool name, the environment"}, {"zh": "迴圈圖亮起「環境」", "en": "Environment lights up in the loop diagram"}), - ("tool_result", "harness/agent.py", 'self.emit("tool_result"', + ("tool_result", "harness/harnesses/simple_harness.py", 'self.emit("tool_result"', {"zh": "結果字串、成功與否、是否截斷、耗時、給你看的圖", "en": "the result string, whether it succeeded, whether it was cut, the time it took, the images for you"}, {"zh": "環境段的結果、聊天區的工具標籤；圖畫在兩邊", "en": "the result in the Environment section, the tool chip in the chat; the charts in both"}), - ("error", "harness/agent.py", 'self.emit("error"', + ("error", "harness/harnesses/simple_harness.py", 'self.emit("error"', {"zh": "例外的類型和訊息；backend 的錯誤兩種語言都有（`text`）", "en": "the exception's type and message, and both languages of a backend's error (`text`)"}, {"zh": "聊天區和停止卡片的錯誤訊息", "en": "the error message in the chat and on the stop card"}), - ("run_end", "harness/agent.py", 'self.emit("run_end"', + ("run_end", "harness/harnesses/simple_harness.py", 'self.emit("run_end"', {"zh": "停止原因、最終回覆、整份訊息列表", "en": "the stop reason, the final reply, the whole message list"}, {"zh": "停止卡片；訊息列表成為下一輪的 `history`", "en": "the stop card; the message list becomes the next turn's `history`"}), ] diff --git a/app/page.py b/app/page.py index 2337edd..4386eed 100644 --- a/app/page.py +++ b/app/page.py @@ -7,11 +7,15 @@ its events (render.py) and passes on the user's clicks. The settings read from the environment below are read once, when the module is first imported. -The agent runs in a background thread (harness.agent.Run) and appends one event per loop +The agent runs in a background thread (harness.harnesses.simple_harness.SimpleHarness) and appends one event per loop stage to a list. While a run is live, a fragment redraws the page from that list every half second. Clicking stop or approve reruns the script but not the thread, so the loop keeps going. +The sidebar can also hand the loop to pi (harness/harnesses/pi_harness.py), a coding agent that runs its own and asks a +model it has credentials for (Claude, GPT…); the page then draws what pi says it does, with no tokens and no prompt, +and no model loads here. + With VLLM_URL set, a vLLM server runs the model instead of this machine (harness/backends/vllm_backend.py), and the model menu lists what that server serves, marking those the page does not support; picking one of those says why and offers a GitHub issue asking for it, already filled in. Without it, the sidebar can still pick vLLM: the page then starts a @@ -37,17 +41,21 @@ import threading import uuid import weakref +from dataclasses import dataclass, field from pathlib import Path +from typing import Callable from urllib.parse import quote, urlencode import streamlit as st from app import render from harness import families, media, tools -from harness.agent import Run, RunSettings from harness.backends import vllm_backend, vllm_docker from harness.backends.backend import Backend, Sampling from harness.backends.simple_backend import SimpleBackend, empty_cache +from harness.harnesses import pi_harness +from harness.harnesses.harness import Harness +from harness.harnesses.simple_harness import RunSettings, SimpleHarness from harness.i18n import LANGS, from_locale, pick from harness.tools import TOOLS @@ -237,6 +245,62 @@ def current_llm() -> Backend: return llm +# ---------- pi ---------- + +@dataclass +class PiAgent: + """pi and the model it asks, where the page would otherwise have a Backend: what the chat needs to know of it.""" + model_id: str # "provider/id", as pi's own model menu names it + model: dict # pi's model object for it (pi_harness.probe) + version: str + info: dict = field(default_factory=dict) # no server and no model folder: pi's model runs at its provider + + @property + def modalities(self) -> tuple[str, ...]: + return ("image",) if "image" in self.model.get("input", ()) else () + + +def pi_picked() -> bool: + return st.session_state.get("harness") == "pi" + + +@st.cache_data(ttl=60, show_spinner=False) +def pi_found() -> dict: + """What pi can run (pi_harness.probe), asked again at most every minute; a PiError is not kept, and is asked again.""" + return pi_harness.probe() + + +def retry_pi(): + pi_found.clear() + + +def pi_settings(who: PiAgent) -> pi_harness.PiSettings: + s = st.session_state + return pi_harness.PiSettings(who.model["provider"], who.model["id"], thinking=s.pi_thinking, tools=tuple(s.pi_tools), + approve_tools=s.approve_tools, max_steps=s.max_steps, lang=s.lang) + + +def pi_picker(busy: bool) -> PiAgent: + """The models pi has credentials for, in the sidebar; pi is started once to ask (and again after a minute).""" + with st.spinner(tr("問 pi 它能用哪些模型…", "Asking pi which models it can use…")): + found = pi_found() + models = {f'{m["provider"]}/{m["id"]}': m for m in found["models"]} + if not models: + raise pi_harness.PiError( + "pi 還沒有可以用的模型。在終端機執行 `pi`，用 `/login` 登入一個 provider；或在啟動這個 app 之前設好 API key" + "（例如 `ANTHROPIC_API_KEY`、`OPENAI_API_KEY`），或在 pi 的 `models.json` 加上一個相容的端點。然後按「再試一次」。", + "pi has no model it can use yet. Run `pi` in a terminal and log in to a provider with `/login`, or set an API key " + "(such as `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`) before starting this app, or add a compatible endpoint to pi's `models.json`. Then press Try again.", + ) + if st.session_state.get("pi_model") not in models: + st.session_state.pi_model = found["default"] if found["default"] in models else next(iter(models)) + with st.sidebar: + st.selectbox(tr("模型", "Model"), list(models), key="pi_model", on_change=new_chat, disabled=busy, + help=tr("pi 有登入資訊的模型（provider/模型 id）。換模型會開始新的對話。", + "The models pi has credentials for (provider/model id). Switching models starts a new conversation.")) + return PiAgent(st.session_state.pi_model, models[st.session_state.pi_model], found["version"]) + + class Slot: """One model, one run at a time, across every browser tab. The run is kept here as well as in session state so another tab can see it and stop it, e.g. after the tab that started it was reloaded.""" @@ -252,15 +316,15 @@ def generation_slot() -> Slot: class LiveRun: - """One user turn: the harness thread and the events it has emitted so far.""" + """One user turn: the harness thread and the events it has emitted so far. `make` makes the harness (simple_harness.SimpleHarness + or pi_harness.PiHarness) given the function it sends its events to.""" - def __init__(self, llm: Backend, history: list[dict], text: str, attachments: list[media.Attachment], files: Path, - settings: RunSettings, slot: Slot): + def __init__(self, make: Callable[[Callable[[dict], None]], Harness], text: str, attachments: list[media.Attachment], slot: Slot): self.text = text self.attachments = attachments self.events: list[dict] = [] self.absorbed = False - self.run = Run(llm, history, text, settings, self.events.append, attachments, files) + self.run = make(self.events.append) self._slot = slot slot.run = self self.thread = threading.Thread(target=self._work, daemon=True) @@ -287,6 +351,14 @@ def data(self) -> dict: def init_state(): for key, value in DEFAULTS.items(): st.session_state.setdefault(key, value) + st.session_state.setdefault("harness", "simple") # what runs the loop: this project's harness, or pi + st.session_state.setdefault("pi_tools", list(pi_harness.DEFAULT_TOOLS)) + st.session_state.setdefault("pi_thinking", "off") + # The sidebar draws only one harness's settings; Streamlit would forget the other's, whose widgets are not drawn, + # unless they are set again here. + for key in [*DEFAULTS, "model_id", "pi_model", "pi_tools", "pi_thinking"]: + if key in st.session_state: + st.session_state[key] = st.session_state[key] st.session_state.setdefault("runs", []) st.session_state.setdefault("history", []) st.session_state.setdefault("view", None) @@ -336,12 +408,15 @@ def sandbox_problem() -> dict | None: def code_allowed() -> bool: - """Whether the sidebar lets the model write code; Docker is looked for only then.""" + """Whether the sidebar lets the model write code: run_python, for which Docker is looked for only then, or with + pi its bash tool.""" + if pi_picked(): + return "bash" in st.session_state.pi_tools return st.session_state.use_tools and st.session_state.python def code_ready() -> bool: - return code_allowed() and sandbox_problem() is None + return code_allowed() and (pi_picked() or sandbox_problem() is None) def offered_tools() -> tuple[str, ...]: @@ -349,15 +424,15 @@ def offered_tools() -> tuple[str, ...]: return tuple(name for name in TOOLS if name != "run_python" or code_ready()) -def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: +def start_run(who: "Backend | PiAgent", text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: """Ask the question in a new run, with `uploads` (file name, contents) saved in the conversation's folder first.""" slot = generation_slot() if not slot.lock.acquire(blocking=False): st.toast(tr("另一個 agent 正在執行（可能在別的分頁），請稍等。", "Another agent is running (maybe in another tab). Please wait.")) return False try: - attachments = [media.save(chat_folder(), name, data, llm.modalities) for name, data in uploads] - run = LiveRun(llm, st.session_state.history, text, attachments, chat_folder(), current_settings(llm), slot) + attachments = [media.save(chat_folder(), name, data, who.modalities) for name, data in uploads] + run = LiveRun(harness_for(who, text, attachments), text, attachments, slot) except Exception: slot.lock.release() raise @@ -366,6 +441,16 @@ def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> return True +def harness_for(who: "Backend | PiAgent", text: str, attachments: list[media.Attachment]) -> Callable[[Callable[[dict], None]], Harness]: + """What runs the turn: pi, or this project's harness on the model in memory, with the sidebar's settings now.""" + history, folder = st.session_state.history, chat_folder() + if isinstance(who, PiAgent): + settings, session = pi_settings(who), st.session_state.chat_id + return lambda emit: pi_harness.PiHarness(settings, history, text, emit, attachments, folder, session) + settings = current_settings(who) + return lambda emit: SimpleHarness(who, history, text, settings, emit, attachments, folder) + + def other_tab_run() -> LiveRun | None: """A run that is still going but was started by another tab, or by this tab before a reload.""" run = generation_slot().run @@ -438,11 +523,28 @@ def show_run(run_index: int): # ---------- sidebar ---------- +def harness_picker(busy: bool): + """The very top of the sidebar: what runs the loop. It is drawn before anything can fail, so that the other can + always be picked.""" + with st.sidebar: + st.markdown(tr("### Agent Loop 可視化", "### Agent Loop Visualizer")) + names = {"simple": tr("內建 harness", "Built-in harness"), "pi": "pi"} + st.segmented_control( + tr("用什麼跑迴圈", "What runs the loop"), list(names), format_func=names.get, key="harness", required=True, + on_change=new_chat, disabled=busy, + help=tr("內建 harness：這個專案自己寫的迴圈（`harness/harnesses/simple_harness.py`），模型在這台電腦或 vLLM 上跑，每個 token 都看得到。" + "pi：把迴圈交給 pi 這個 coding agent，它問你在 pi 裡設定好的模型（Claude、GPT…），網頁畫出它每一步做了什麼；" + "看不到 token 和 prompt。pi 的工具在這台電腦上執行，沒有沙箱。換過去會開始新的對話。", + "Built-in harness: this project's own loop (`harness/harnesses/simple_harness.py`), with the model on this computer or on vLLM, and every token in view. " + "pi: hands the loop to pi, a coding agent, which asks a model you set up in pi (Claude, GPT…), and the page draws what it does " + "at each step; the tokens and the prompt stay out of view. pi's tools run on this computer, without a sandbox. Switching starts a new conversation."), + ) + + def model_picker(menu: dict[str, str], busy: bool): """The top of the sidebar: the model, and, without VLLM_URL, what runs it. It is drawn before the model loads, so that while vLLM starts, or after it failed to, another model or transformers can still be picked.""" with st.sidebar: - st.markdown(tr("### Agent Loop 可視化", "### Agent Loop Visualizer")) about = tr("換模型會開始新的對話，解碼設定也換成新模型的建議值。harness、工具和畫面都不變。", "Switching models starts a new conversation and resets the decoding settings to what the new model recommends. " "The harness, the tools and the page stay the same.") @@ -520,6 +622,31 @@ def sidebar(llm: Backend, live: bool, busy: bool): st.caption(tr("設定會套用到下一個問題。", "Settings apply to the next question.")) +def pi_sidebar(who: PiAgent, live: bool, busy: bool): + """pi's settings: its tools, how much its model thinks, asking first, and the step limit the page keeps for it.""" + with st.sidebar: + st.caption(f"pi {who.version} · `{who.model_id}` · " + tr("在 provider 那邊跑", "runs at the provider")) + st.markdown(tr("##### 迴圈", "##### Loop")) + st.pills(tr("pi 的工具", "pi's tools"), list(pi_harness.TOOLS), selection_mode="multi", key="pi_tools", + help=tr("模型看得到哪些 pi 的工具。全部取消就只是聊天機器人。", "Which of pi's tools the model sees. With none, it is just a chatbot.")) + st.caption(tr("pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：bash 碰得到你碰得到的一切。", + "pi's tools run on this computer, in this conversation's folder, without a sandbox: bash reaches whatever you can.")) + levels = pi_harness.THINKING if who.model.get("reasoning") else ("off",) + if st.session_state.pi_thinking not in levels: + st.session_state.pi_thinking = "off" + st.segmented_control(tr("思考", "Thinking"), list(levels), key="pi_thinking", required=True, + help=tr("pi 的思考等級：越高，模型回覆前想得越多。模型不會思考時只有 off。", + "pi's thinking level: the higher, the more the model thinks before it replies. A model that does not think has only off.")) + st.toggle(tr("執行工具前先問我", "Ask me before running a tool"), key="approve_tools", + help=tr("pi 自己不問；網頁載入一個小 extension（`pi_approve.ts`），每次執行工具前都先等你確認。", + "pi does not ask by itself; the page loads a small extension (`pi_approve.ts`) that waits for you before every tool call.")) + st.markdown(tr("##### 上限", "##### Limits")) + st.slider(tr("步數上限", "Max steps"), 1, 100, key="max_steps", + help=tr("pi 自己沒有步數上限；迴圈跑滿這麼多步，網頁就請 pi 中止。", "pi has no step limit of its own; once the loop has gone around this many times, the page asks pi to abort.")) + st.button(tr("新對話", "New chat"), on_click=new_chat, disabled=live, width="stretch") + st.caption(tr("設定會套用到下一個問題。", "Settings apply to the next question.")) + + def can_mtp(llm: Backend) -> bool: """Whether the MTP switch does anything: with transformers, for a model with a drafter; on vLLM in Docker, on a GPU and for a model vLLM drafts for. A server at VLLM_URL decides for itself.""" @@ -569,7 +696,7 @@ def mtp_help(llm: Backend) -> str: # ---------- left: chat ---------- -def chat_panel(llm: Backend, live_run: LiveRun | None, other: LiveRun | None): +def chat_panel(llm: "Backend | PiAgent", live_run: LiveRun | None, other: LiveRun | None): runs = st.session_state.runs busy = live_run is not None or other is not None box = st.container(height=BOX_HEIGHT, border=False, key="chat_box") @@ -629,7 +756,7 @@ def chat_panel(llm: Backend, live_run: LiveRun | None, other: LiveRun | None): sample_picker(sample_row, samples, llm, busy) -def sample_picker(row, samples: list[str], llm: Backend, busy: bool): +def sample_picker(row, samples: list[str], llm: "Backend | PiAgent", busy: bool): """The sample files from assets/examples that this model can use, to ask a question of one's own about: a file picked here goes with the next question typed in the input box, as one attached with the + would.""" readable = llm.modalities # not the model: Streamlit keeps format_func after the run, and with it the model @@ -648,9 +775,10 @@ def uploaded(sent) -> list[tuple[str, bytes]]: return files + ([("recording.wav", audio.getvalue())] if (audio := sent.get("audio")) else []) -def examples(llm: Backend, busy: bool): +def examples(llm: "Backend | PiAgent", busy: bool): """The page language's examples that this model can try: one that attaches a file shows only if the model reads - that kind, and one that needs Python only once the model may write code, disabled until Docker is ready.""" + that kind, and one that needs Python only once the model may write code, disabled until Docker is ready (with pi, + once it has bash).""" lang, hidden = st.session_state.lang, False for i, (question, needs) in enumerate(EXAMPLES[lang]): sample = needs if needs not in (None, "python") else None @@ -660,12 +788,14 @@ def examples(llm: Backend, busy: bool): if kind in ("python", "file") and not code_allowed(): hidden = True continue - problem = sandbox_problem() if kind in ("python", "file") else None + problem = sandbox_problem() if kind in ("python", "file") and not pi_picked() else None note = problem[lang] if problem else tr(f"會附上範例檔案 {sample}", f"Attaches the sample file {sample}") if sample else None if st.button(question, key=f"example-{i}", icon=EXAMPLE_ICONS.get(kind), help=note, disabled=busy or problem is not None): if start_run(llm, question, [(sample, (SAMPLES / sample).read_bytes())] if sample else []): st.rerun() - if hidden: + if hidden and pi_picked(): + st.caption(tr("在側欄給 pi 它的 bash 工具，還有要寫 Python 的範例。", "Give pi its bash tool in the sidebar for examples that need Python.")) + elif hidden: st.caption(tr("打開側欄的「讓模型寫程式」，還有要寫 Python 的範例。", "Turn on Let the model write code in the sidebar for examples that need Python.")) @@ -730,7 +860,7 @@ def assistant_message(i: int, run: LiveRun): # ---------- right: the loop ---------- -def viz_panel(llm: Backend): +def viz_panel(llm: "Backend | PiAgent"): runs = st.session_state.runs index = st.session_state.view run = runs[index] if runs and index is not None else None @@ -747,6 +877,8 @@ def viz_panel(llm: Backend): with st.container(height=BOX_HEIGHT, border=False, key=f"{view}_view"): if view == "context": context_panel(index, data) + elif view == "model" and isinstance(llm, PiAgent): + pi_model_panel(llm, data) elif view == "model": model_panel(llm, data) else: @@ -754,10 +886,10 @@ def viz_panel(llm: Backend): def loop_panel(index: int | None, run: LiveRun | None, data: dict | None, model_id: str): - lang = st.session_state.lang - st.html(render.diagram_html(data, st.context.theme.type == "dark", lang, model=model_id)) + lang, harness = st.session_state.lang, st.session_state.harness + st.html(render.diagram_html(data, st.context.theme.type == "dark", lang, model=model_id, harness=harness)) if data is None: - st.html(render.INTRO[lang]) + st.html((render.PI_INTRO if harness == "pi" else render.INTRO)[lang]) return st.caption(render.run_stats(data, lang)) # A fixed height makes it scroll; style.css shrinks it to the space left under the diagram. @@ -779,6 +911,12 @@ def loop_panel(index: int | None, run: LiveRun | None, data: dict | None, model_ def context_panel(index: int | None, data: dict | None): lang = st.session_state.lang steps = [s for s in data["steps"] if s["context"]] if data else [] + if not steps and pi_picked(): + st.info(tr("pi 不說它送給模型 API 的 prompt 是什麼，所以這裡沒有東西可以顯示；每一步做了什麼在「迴圈」分頁。" + "用內建 harness 的話，這裡會顯示每一步模型實際讀到的完整字串。", + "pi does not say what prompt it sends to the model's API, so there is nothing to show here; what each step did is in the Loop tab. " + "With the built-in harness, this shows the whole string the model read in each step.")) + return if not steps: st.info(tr("送出問題後，這裡會顯示每一步模型實際讀到的完整字串。", "Once you send a question, this shows the whole string the model read in each step.")) return @@ -809,6 +947,18 @@ def model_panel(llm: Backend, data: dict | None): st.html(render.folder_html(llm.info, lang)) +def pi_model_panel(who: PiAgent, data: dict | None): + lang = st.session_state.lang + st.html(render.pi_model_html(who.model, who.version, lang)) + settings = data["settings"] if data else {"thinking": st.session_state.pi_thinking, "approve_tools": st.session_state.approve_tools, + "max_steps": st.session_state.max_steps, "tools": st.session_state.pi_tools} + st.html(render.pi_settings_html(settings, lang)) + if data and data["steps"]: + st.markdown(tr("##### 每一步的請求", "##### The request in each step")) + st.dataframe(render.step_rows(data, lang), hide_index=True, width="stretch") + st.html(render.pi_tools_html(pi_harness.TOOLS, settings["tools"], lang)) + + def issue_url(error: vllm_backend.UnsupportedModel) -> str: """A new GitHub issue asking the page to support a model, its title and description filled in in the page's language. It names the model and its model_type, never VLLM_URL, which may be a private address, and a checkpoint @@ -853,20 +1003,30 @@ def agent_page(): init_state() own_live = any(run.live for run in st.session_state.runs) polling = own_live or other_tab_run() is not None - model_picker(model_menu(), polling) - llm = current_llm() - except (vllm_backend.ServerError, vllm_docker.DockerError) as e: # the vLLM server is down, refused or did not start + harness_picker(polling) + if pi_picked(): + llm = pi_picker(polling) + else: + model_picker(model_menu(), polling) + llm = current_llm() + except (vllm_backend.ServerError, vllm_docker.DockerError, pi_harness.PiError) as e: # vLLM is down, refused or did not start, or pi unsupported = isinstance(e, vllm_backend.UnsupportedModel) st.error(e.text[st.session_state.lang], icon=":material/extension_off:" if unsupported else ":material/cloud_off:") if isinstance(e, vllm_docker.DockerError): # the sidebar still has the menu and the switch back to transformers st.button(tr("再試一次", "Try again"), icon=":material/refresh:", on_click=retry_vllm) + if isinstance(e, pi_harness.PiError): # and the switch back to the built-in harness + st.button(tr("再試一次", "Try again"), icon=":material/refresh:", on_click=retry_pi) if unsupported: # and the menu has the models the page does support ask_for_model(e) st.stop() - sidebar(llm, own_live, polling) - # Streamlit keeps the fragment after the run, to run it again on its own; held weakly, the model can still be let go - # (release_model()) the moment another is picked, rather than once the next run has replaced the fragment. - model = weakref.ref(llm) + if isinstance(llm, PiAgent): + pi_sidebar(llm, own_live, polling) + model = lambda: llm # nothing in memory here to let go + else: + sidebar(llm, own_live, polling) + # Streamlit keeps the fragment after the run, to run it again on its own; held weakly, the model can still be let go + # (release_model()) the moment another is picked, rather than once the next run has replaced the fragment. + model = weakref.ref(llm) @st.fragment(run_every=POLL_SECONDS if polling else None) def workspace(): diff --git a/app/render.py b/app/render.py index 5b51223..989c23e 100644 --- a/app/render.py +++ b/app/render.py @@ -7,6 +7,8 @@ A token shows as much of the model's distribution as the backend reported (backend.TOKEN_DETAILS): simple_backend.SimpleBackend reports all of it, and what a backend leaves out is left out here too, never filled in. +So does a step: a harness that does not run the model itself (pi, harnesses/pi_harness.py) sends no context and no tokens, +only the text the model streams (delta) and the token counts its provider reported, and its steps are drawn from those. """ import base64 @@ -37,6 +39,21 @@ "error": ("error", "Something went wrong in the code; the message is below."), }, } +# Where pi runs the loop (harnesses/pi_harness.py), the page stops it from outside, so these stops are said otherwise. +PI_STOP_REASONS = { + "zh": { + "max_steps": ("達到步數上限", "pi 自己沒有步數上限：迴圈跑滿設定的步數，網頁就請 pi 中止，不管模型想不想繼續。"), + "user_stopped": ("使用者中斷", "你按了停止，網頁請 pi 中止：模型正在寫的回覆和正在執行的工具都會停下。"), + "approval_timeout": ("等待確認逾時", "一直沒有人按「允許」或「拒絕」（例如分頁已經關掉或重新整理），網頁當作拒絕並請 pi 中止，讓其他分頁可以使用。"), + "error": ("執行錯誤", "pi 或模型的 API 回報了錯誤，訊息在下面。"), + }, + "en": { + "max_steps": ("step limit reached", "pi has no step limit of its own: once the loop has gone around the set number of times, the page asks pi to abort, whether or not the model wants to go on."), + "user_stopped": ("stopped by the user", "You pressed stop, and the page asked pi to abort: the reply the model was writing and the tool that was running both stop."), + "approval_timeout": ("confirmation timed out", "No one pressed Allow or Reject (the tab was closed or reloaded, say), so the page took it as a refusal and asked pi to abort, freeing it for other tabs."), + "error": ("error", "pi or the model's API reported an error; the message is below."), + }, +} LAYER_TYPES = { "zh": {"sliding_attention": "滑動視窗", "full_attention": "full attention", "linear_attention": "線性 attention"}, "en": {"sliding_attention": "sliding window", "full_attention": "full attention", "linear_attention": "linear attention"}, @@ -149,6 +166,34 @@ """, } +# The same cycle, as pi runs it (harnesses/pi_harness.py). +PI_INTRO = { + "zh": """ +
+

送出問題後，這裡會一步一步畫出 pi 的 agent 迴圈。迴圈由 pi 跑，網頁只畫它說它做了什麼：

+
+
脈絡
pi 把它的 system prompt、工具和對話組成一個請求，送到模型的 API。請求裡寫了什麼，pi 不說，所以「脈絡」分頁是空的。
+
模型
一次 API 請求。回覆是一段一段傳回來的文字，不是 token，所以沒有機率；token 數是 provider 報的。
+
Harness
pi 讀出回覆裡的工具呼叫、檢查參數；打開「執行工具前先問我」時，先問你。
+
環境
pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱。結果接回對話，開始下一步。
+
停止
模型不再呼叫工具、達到步數上限，或是你按下停止。
+
+
+""", + "en": """ +
+

Send a question, and pi's agent loop is drawn here step by step. pi runs the loop; the page draws what pi says it does:

+
+
Context
pi joins its system prompt, its tools and the conversation into one request to the model's API. pi does not say what the request holds, so the Context tab stays empty.
+
Model
One request to the API. The reply streams back as text, not tokens, so there are no probabilities; the token counts are the provider's.
+
Harness
pi reads the tool calls in the reply and checks their arguments; with Ask me before running a tool on, it asks you first.
+
Environment
pi's tools run on this computer, in this conversation's folder, without a sandbox. Their results go back into the conversation, and the next step begins.
+
Stop
The model calls no more tools, the step limit is reached, or you press stop.
+
+
+""", +} + CONTEXT_LEGEND = { "zh": """

@@ -304,17 +349,18 @@ def media_line(counts: dict, lang: str) -> str: # ---------- events -> steps ---------- def build(events: list[dict]) -> dict: - data = {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + data = {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} for ev in events: kind = ev["type"] data["last"], data["t"] = kind, ev["t"] if kind == "run_start": - data["model"], data["family"] = ev["model"], families.get(ev["family"]) - data["settings"] = ev["settings"] + # A harness that does not know how the model writes its turns (pi) sends no family. + data["model"], data["family"] = ev["model"], families.get(ev["family"]) if ev.get("family") else None + data["harness"], data["settings"] = ev.get("harness", "simple"), ev["settings"] elif kind == "step_start": data["steps"].append({ "n": ev["step"], "t0": ev["t"], "t1": ev["t"], "context": None, "prefill": None, - "tokens": [], "done": None, "parsed": None, "calls": {}, + "tokens": [], "text": [], "done": None, "parsed": None, "calls": {}, }) elif kind == "run_end": data["end"] = ev @@ -325,6 +371,11 @@ def build(events: list[dict]) -> dict: step["t1"] = ev["t"] if kind == "token": step["tokens"].append(ev) + elif kind == "delta": # kept as runs of one channel: [{"channel", "text"}] + if step["text"] and step["text"][-1]["channel"] == ev["channel"]: + step["text"][-1] = {**step["text"][-1], "text": step["text"][-1]["text"] + ev["text"]} + else: + step["text"].append({"channel": ev["channel"], "text": ev["text"]}) elif kind in ("context", "prefill", "parsed"): step[kind] = ev elif kind == "model_end": @@ -375,10 +426,23 @@ def pending_approvals(data: dict) -> list[dict]: def draft(data: dict) -> str: - """The reply as it streams: content tokens of the step in progress.""" + """The reply as it streams: content tokens of the step in progress, or the content its harness streamed.""" if not data["steps"]: return "" - return "".join(t["text"] for t in data["steps"][-1]["tokens"] if t["channel"] == "content") + step = data["steps"][-1] + return "".join(t["text"] for t in step["tokens"] + step["text"] if t["channel"] == "content") + + +def generated(step: dict) -> int: + """How many tokens the model wrote in a step: those drawn, or the count its provider reported when none are.""" + return len(step["tokens"]) or (step["done"] or {}).get("n_tokens") or 0 + + +def context_tokens(step: dict) -> int | None: + """How many tokens the model read in a step: the context's, or the prompt its provider reported reading.""" + if step["context"]: + return step["context"]["n_tokens"] + return ((step["done"] or {}).get("usage") or {}).get("prompt") def stop_title(stop_reason: str, lang: str) -> str: @@ -395,8 +459,8 @@ def status(data: dict, lang: str) -> str: if last == "prefill": read = step["prefill"]["n_tokens"] return f"Step {n} · " + pick(lang, f"模型讀完 {read:,} 個 token，開始生成", f"the model has read {read:,} tokens and starts generating") - if last == "token": - channel = step["tokens"][-1]["channel"] + if last in ("token", "delta"): + channel = (step["tokens"] or step["text"])[-1]["channel"] if channel == "thought": return f"Step {n} · " + pick(lang, "模型思考中…", "the model is thinking…") if channel == "tool_call": @@ -421,8 +485,8 @@ def footer(data: dict, lang: str) -> str: def run_stats(data: dict, lang: str) -> str: s = data["settings"] steps = data["steps"] - ctx = next((st["context"]["n_tokens"] for st in reversed(steps) if st["context"]), 0) - gen = sum(len(st["tokens"]) for st in steps) + ctx = next((n for st in reversed(steps) if (n := context_tokens(st)) is not None), 0) + gen = sum(generated(st) for st in steps) return f"Step {len(steps)} / {s.get('max_steps', '?')} · " + pick( lang, f"脈絡 {ctx:,} tokens · 共生成 {gen:,} tokens", f"context {ctx:,} tokens · {gen:,} tokens generated" ) + f" · {fmt_ms(data['t'])}" @@ -459,6 +523,13 @@ def trace_html(data: dict, lang: str) -> str: ("env-context", "M102 150 V84", 92, 120, "end", {"zh": "觀察", "en": "observation"}), ("harness-stop", "M462 184 H554", 509, 174, "middle", {"zh": "沒有工具呼叫", "en": "no tool call"}), ] +# What the diagram says otherwise when pi runs the loop: its own harness, its tools on this computer, and text, not +# tokens, from the model's API. +PI_DIAGRAM = { + "harness": {"zh": "pi · 驗證 · 權限", "en": "pi · check · ask"}, + "env": {"zh": "這台電腦", "en": "this computer"}, + "model-harness": {"zh": "輸出文字", "en": "text out"}, +} def loop_state(data: dict | None, lang: str) -> dict: @@ -470,7 +541,7 @@ def loop_state(data: dict | None, lang: str) -> dict: caps, steps = state["captions"], data["steps"] for s in steps: state["visited"].add("context") - if s["prefill"]: + if s["prefill"] or s["text"]: state["visited"].add("model") if s["done"]: state["visited"].add("harness") @@ -479,14 +550,17 @@ def loop_state(data: dict | None, lang: str) -> dict: step = steps[-1] if steps else None if step: - if step["context"]: - caps["context"] = f"{step['context']['n_tokens']:,} tokens" + if (read := context_tokens(step)) is not None: + caps["context"] = f"{read:,} tokens" if step["prefill"]: caps["model"] = f"prefill {fmt_ms(step['prefill']['ms'])}" if step["tokens"]: caps["model"] = pick(lang, f"已生成 {len(step['tokens'])} tokens", f"{len(step['tokens'])} tokens so far") - if step["done"]: - caps["model"] = f"{step['done']['n_tokens']} tok · {step['done']['tok_per_s']} tok/s" + elif step["text"]: + chars = sum(len(r["text"]) for r in step["text"]) + caps["model"] = pick(lang, f"已收到 {chars:,} 字元", f"{chars:,} characters so far") + if done := step["done"]: + caps["model"] = f"{done['n_tokens']} tok" + (f" · {done['tok_per_s']} tok/s" if done.get("tok_per_s") else "") calls = list(step["calls"].values()) if step["parsed"]: caps["harness"] = pick(lang, "、", ", ").join(c["name"] for c in calls) or pick(lang, "沒有工具呼叫", "no tool call") @@ -504,10 +578,12 @@ def loop_state(data: dict | None, lang: str) -> dict: last = data["last"] started = bool(step and step["calls"] and list(step["calls"].values())[-1]["started"]) state["active"], state["edge"] = { - "step_start": ("context", "env-context" if step and step["n"] > 1 else None), + # pi sends the request as its turn starts, and says nothing more until the model's output comes back. + "step_start": ("context", "context-model" if data["harness"] == "pi" else "env-context" if step and step["n"] > 1 else None), "context": ("context", "context-model"), "prefill": ("model", "context-model"), "token": ("model", None), + "delta": ("model", None), "model_end": ("harness", "model-harness"), "parsed": ("harness", "model-harness"), "tool_call": ("harness", None), @@ -557,15 +633,17 @@ def loop_state(data: dict | None, lang: str) -> dict: """ -def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = None, model: str | None = None) -> str: +def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = None, model: str | None = None, + harness: str | None = None) -> str: """The loop diagram as an : Streamlit's HTML sanitizer removes inline . - `model` names the model when no run says which one it is.""" + `model` and `harness` name the model and what runs the loop when no run says which they are.""" st = loop_state(data, lang) if counter: st["counter"] = counter model = (data or {}).get("model") or model if model: st["captions"].setdefault("model", model.split("/")[-1]) + words_for = PI_DIAGRAM if ((data or {}).get("harness") or harness) == "pi" else {} pal = PALETTES["dark" if dark else "light"] css = DIAGRAM_CSS for key, value in pal.items(): @@ -573,10 +651,12 @@ def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = edges = [] for key, path, x, y, anchor, label in EDGES: cls = "edge on" if key == st["edge"] else "edge" + label = words_for.get(key, label) edges.append(f'{label[lang]}') nodes = [] for key, x, y, words in NODES: title, caption = words[lang] + caption = words_for[key][lang] if key in words_for else caption cls = ["node"] if key in st["visited"]: cls.append("visited") @@ -845,6 +925,8 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: f'

{meta}

{token_legend(s, live, lang, [t for st in data["steps"] for t in st["tokens"]])}' f'{tokens_html(step["tokens"], not live, family, lang)}' ) + elif step["text"] or step["done"]: # from a harness that sees the model's text, not its tokens + parts.append(streamed_html(step, lang)) if step["parsed"]: body = [] @@ -860,6 +942,9 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: if c["errors"]: errors = esc(pick(lang, "；", "; ").join(c["errors"])) body.append(f'

✗ {errors} → ' + pick(lang, "把錯誤當作觀察送回模型", "the error goes back to the model as an observation") + "

") + elif family is None: # pi: the API sends the call as JSON already, and pi checks it as it runs the tool + body.append('

' + pick(lang, "API 傳回的已經是 JSON；pi 執行前再用工具的 schema 檢查參數", + "the API returned it as JSON; pi checks the arguments against the tool's schema before running it") + "

") else: name = esc(family.name) body.append('

✓ ' + pick(lang, f"從 {name} 格式解析成 JSON，符合工具 schema", @@ -879,8 +964,9 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: "the confirmation timed out; the harness took it as a refusal and stopped the loop") + "

") elif c["approval"] == "stopped": body.append('

⏹ ' + pick(lang, "執行被停止，這個呼叫沒有執行", "the run was stopped, so this call did not run") + "

") + what = PI_DIAGRAM["harness"][lang] if data["harness"] == "pi" else pick(lang, "解析 · 驗證 · 權限", "parse · check · ask") parts.append( - f'

Harness {pick(lang, "解析 · 驗證 · 權限", "parse · check · ask")}

' + f'

Harness {what}

' f'{"".join(body)}
' ) @@ -906,6 +992,49 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: return "".join(parts) +def text_html(runs: list[dict]) -> str: + """The text a harness streamed (delta), by channel: thinking in italics, tool calls in monospace; no token + boundaries, since it saw none.""" + return '
' + "".join(f'{esc(r["text"])}' for r in runs) + "
" + + +def streamed_html(step: dict, lang: str) -> str: + """The model section of a step whose harness asks a model's API (pi): the text as it streamed, and what the + provider reported once the reply was done. No number here was guessed: an API sends no probabilities.""" + sep, done, meta = '|', step["done"], [] + if done: + meta.append(pick(lang, f'生成 {done["n_tokens"]:,} tokens', f'wrote {done["n_tokens"]:,} tokens') + + (f' · {done["tok_per_s"]} tok/s' if done.get("tok_per_s") else "")) + if done.get("first_ms") is not None: + meta.append(pick(lang, f"第一段輸出在 {fmt_ms(done['first_ms'])} 後回來", f"first output after {fmt_ms(done['first_ms'])}")) + usage = done.get("usage") or {} + if usage.get("prompt"): + cached = pick(lang, f"，其中 {usage['cached']:,} 來自 cache", f", {usage['cached']:,} from the cache") if usage.get("cached") else "" + meta.append(pick(lang, f"讀了 {usage['prompt']:,} tokens{cached}", f"read {usage['prompt']:,} tokens{cached}")) + if usage.get("cost"): + meta.append(pick(lang, f"照 pi 的價目約 ${usage['cost']:.4f}", f"about ${usage['cost']:.4f} at pi's prices")) + if done["finish"] == "max_new_tokens": + meta.append(pick(lang, "到達 token 上限", "hit the token limit")) + elif done["finish"] == "user_stopped": + meta.append(pick(lang, "使用者中斷", "stopped by the user")) + elif done["finish"] == "error": + meta.append(pick(lang, "請求失敗", "the request failed")) + else: + chars = sum(len(r["text"]) for r in step["text"]) + meta.append(pick(lang, f"已收到 {chars:,} 字元…", f"{chars:,} characters so far…")) + legend = ( + '

' + pick(lang, "API 傳回來的是文字，不是 token，所以沒有機率可畫；token 數是 provider 報的。", + "The API sends back text, not tokens, so there are no probabilities to draw; the token counts are the provider's.") + + "" + pick(lang, '斜體 = 思考等寬 = 工具呼叫', + 'italic = thinkingmonospace = tool call') + "

" + ) + return ( + f'

{pick(lang, "模型", "Model")} ' + f'{pick(lang, "一次 API 請求，回覆一段一段傳回來", "one request to the API; the reply streams back")}

' + f'

{sep.join(meta)}

{legend}{text_html(step["text"])}
' + ) + + def call_html(name: str, arguments: dict | None) -> str: """A tool call as name(arguments JSON). An argument that runs over several lines, such as code, is shown as … in the JSON and written out below it.""" @@ -924,13 +1053,14 @@ def error_text(data: dict, lang: str) -> str | None: def stop_html(data: dict, lang: str) -> str: end = data["end"] - title, desc = STOP_REASONS[lang].get(end["stop_reason"], (end["stop_reason"], "")) + reasons = {**STOP_REASONS[lang], **(PI_STOP_REASONS[lang] if data["harness"] == "pi" else {})} + title, desc = reasons.get(end["stop_reason"], (end["stop_reason"], "")) last = data["steps"][-1] if data["steps"] else None piece = last["done"]["stop_piece"] if last and last["done"] else None if end["stop_reason"] == "no_tool_call" and piece: stop_chip, meaning = chip(piece, data["family"], lang), stop_meaning(piece, data["family"], lang) desc += pick(lang, f" 最後停在 {stop_chip}（{meaning}）。", f" It stopped at {stop_chip} ({meaning}).") - gen = sum(len(s["tokens"]) for s in data["steps"]) + gen = sum(generated(s) for s in data["steps"]) error = f'

{esc(error_text(data, lang))}

' if data["error"] else "" cls = "stop-card is-error" if end["stop_reason"] == "error" else "stop-card" totals = pick(lang, f'共 {len(data["steps"])} 步 · 生成 {gen:,} tokens', f'{n_steps(len(data["steps"]), lang)} · {gen:,} tokens generated') @@ -1106,15 +1236,15 @@ def knobs_html(s: dict, defaults: dict, lang: str) -> str: def step_rows(data: dict, lang: str) -> list[dict]: rows = [] for s in data["steps"]: - if not s["context"]: + if not s["context"] and not s["done"]: continue probs = [t["p"] for t in s["tokens"] if not t["special"] and known(t, "p")] done = s["done"] or {} rows.append({ "step": s["n"], - pick(lang, "脈絡 tokens", "context tokens"): s["context"]["n_tokens"], + pick(lang, "脈絡 tokens", "context tokens"): context_tokens(s), "prefill ms": s["prefill"]["ms"] if s["prefill"] else None, - pick(lang, "生成 tokens", "generated tokens"): len(s["tokens"]), + pick(lang, "生成 tokens", "generated tokens"): generated(s), "tok/s": done.get("tok_per_s"), pick(lang, "平均機率", "mean probability"): round(sum(probs) / len(probs), 3) if probs else None, pick(lang, "停止", "stop"): done.get("stop_piece") or done.get("finish"), @@ -1280,3 +1410,76 @@ def folder_html(info: dict, lang: str) -> str: " Below is each file's size and purpose, and what it holds in this family's checkpoints; the values of the fields are read from the files.") + f'

{files_html(folder, family.file_notes, family, lang, served="server" in info)}{extra}' ) + + +# ---------- pi ---------- + +def pi_model_html(model: dict, version: str, lang: str) -> str: + """The model pi asks, from pi's own model object: it runs at its provider, and the page sees only what the API + sends back.""" + name, where = model.get("name") or model["id"], f'{model["provider"]}/{model["id"]}' + cost = model.get("cost") or {} + specs = [ + ("provider", model["provider"]), + ("API", model.get("api")), + (pick(lang, "脈絡上限", "context limit"), f'{model["contextWindow"]:,} tokens' if model.get("contextWindow") else None), + (pick(lang, "輸出上限", "output limit"), f'{model["maxTokens"]:,} tokens' if model.get("maxTokens") else None), + (pick(lang, "讀得懂", "reads"), " · ".join(MODALITIES[lang].get(k, k) for k in model.get("input") or ("text",))), + (pick(lang, "會思考", "thinks"), pick(lang, "會", "yes") if model.get("reasoning") else pick(lang, "不會", "no")), + (pick(lang, "每百萬 token", "per million tokens"), + pick(lang, f'讀 ${cost["input"]} · 寫 ${cost["output"]}', f'${cost["input"]} in · ${cost["output"]} out') if cost.get("input") is not None and cost.get("output") is not None else None), + ] + spec_html = "".join(f"
{esc(k)}
{esc(v)}
" for k, v in specs if v) + about = pick( + lang, + f"pi {version} 跑迴圈，透過 {model['provider']} 的 API 問這個模型；模型在 provider 的伺服器上跑，不在這台電腦上。" + "模型清單和登入資訊是 pi 自己的（`pi` 再輸入 `/login`，或環境裡的 API key）。" + "API 傳回來的是文字，不是 token，所以這裡沒有 token 的機率、特殊 token 和模型資料夾；pi 也不說它送出的 prompt 是什麼。", + f"pi {version} runs the loop and asks this model through {model['provider']}'s API; the model runs on the provider's servers, " + "not on this computer. The models and the credentials are pi's own (`pi`, then `/login`, or an API key in the environment). " + "An API sends back text, not tokens, so there are no token probabilities, special tokens or model folder here; nor does pi say which prompt it sends.", + ) + return ( + f'

{esc(name)} {esc(where)}

{prose_html(about)}

' + f'
{spec_html}
' + ) + + +def pi_settings_html(s: dict, lang: str) -> str: + """What the sidebar gave pi for the turn.""" + rows = [ + (pick(lang, "思考", "Thinking"), s.get("thinking", "off"), + pick(lang, "pi 的思考等級：off 到 high，越高模型在回覆前想得越多（也越貴）。模型不會思考時只有 off。", + "pi's thinking level, from off to high: the higher, the more the model thinks before it replies (and the more it costs). A model that does not think has only off.")), + (pick(lang, "執行工具前先問我", "Ask me before running a tool"), pick(lang, "開", "on") if s.get("approve_tools") else pick(lang, "關", "off"), + pick(lang, "pi 自己不問；網頁載入一個小 extension（pi_approve.ts），在每次執行工具前問你。", + "pi does not ask by itself; the page loads a small extension (pi_approve.ts) that asks you before every tool call.")), + (pick(lang, "步數上限", "Max steps"), s.get("max_steps"), + pick(lang, "pi 沒有步數上限；跑滿設定的步數，網頁就請 pi 中止。", "pi has no step limit; once the loop has gone around this many times, the page asks pi to abort.")), + ] + body = "".join(f"
{k} {esc(v)}
{d}
" for k, v, d in rows) + return ( + f'

{pick(lang, "pi 的設定", "pi settings")}

' + + pick(lang, "解碼的設定（temperature、top-k、top-p）在 provider 那邊，pi 不讓人改；側欄只給得了這些。", + "The decoding settings (temperature, top-k, top-p) are the provider's, and pi does not let them be changed; the sidebar gives only these.") + + f'

{body}
' + ) + + +def pi_tools_html(tools: dict, offered: list[str], lang: str) -> str: + """pi's built-in tools, each with what it does and where it runs; one not in `offered` is marked as kept from the model.""" + items = [] + for name, (what, environment) in tools.items(): + off = "" if name in offered else '

' + pick(lang, "這一輪沒有提供給模型，見側欄的設定", "not offered to the model in this turn; see the sidebar") + "

" + items.append( + f'

  • {esc(name)}

    {esc(what[lang])}

    ' + f'

    {pick(lang, "環境：", "environment: ")}{esc(environment[lang])}

    {off}
    • ' + ) + return ( + f'

    {pick(lang, "動作空間：pi 的工具", "Action space: the tools pi has")}

    ' + + pick(lang, "pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：bash 碰得到你碰得到的一切。想先看過每個呼叫，就打開「執行工具前先問我」。" + "工具的 schema 和說明是 pi 寫給模型的，網頁看不到。", + "pi's tools run on this computer, in this conversation's folder, without a sandbox: bash reaches whatever you can. To see each call first, " + "turn on Ask me before running a tool. The tools' schemas and descriptions are pi's, for the model, and the page does not see them.") + + f'

      {"".join(items)}
    ' + ) diff --git a/harness/__init__.py b/harness/__init__.py index 8232a2f..435014f 100644 --- a/harness/__init__.py +++ b/harness/__init__.py @@ -1,9 +1,11 @@ """The harness: everything that runs a turn, and nothing a person sees. -agent.py is the loop itself (Run): context, model, parse, tool, observation, stop, with one event per stage for the -app to draw. tools.py has the tools it runs, with run_python's Docker sandbox built from sandbox/. media.py turns the -files attached to a question into what the model reads. families/ is everything model-specific, and backends/ is what -runs the model, behind one interface. i18n.py has the page's two languages, which the errors the model reads use too. +harnesses/ is what runs the loop, behind one interface: simple_harness.py is this project's own loop (SimpleHarness): +context, model, parse, tool, observation, stop, with one event per stage for the app to draw; pi_harness.py hands the +loop to pi, a coding agent, and turns what pi reports into the same events. tools.py has the tools SimpleHarness runs, +with run_python's Docker sandbox built from sandbox/. media.py turns the files attached to a question into what the +model reads. families/ is everything model-specific, and backends/ is what runs the model, behind one interface. +i18n.py has the page's two languages, which the errors the model reads use too. Nothing here imports Streamlit, and importing the package loads nothing heavy: the app's guide imports families and i18n with pygments alone. diff --git a/harness/backends/backend.py b/harness/backends/backend.py index 168f9eb..452dda2 100644 --- a/harness/backends/backend.py +++ b/harness/backends/backend.py @@ -1,8 +1,8 @@ """The interface between the harness and whatever runs the model. -The harness (agent.Run) and the page (app/page.py) reach the model only through a Backend. It renders the -conversation into a prompt with the model's chat template, encodes the prompt into the tokens the model reads, -and generates, yielding the events render.py draws. The app runs one of two: simple_backend.SimpleBackend, transformers +This project's harness (harnesses/simple_harness.SimpleHarness) and the page (app/page.py) reach the model only through +a Backend. It renders the conversation into a prompt with the model's chat template, encodes the prompt into the tokens +the model reads, and generates, yielding the events render.py draws. The app runs one of two: simple_backend.SimpleBackend, transformers on this machine, with a hand-written decoding loop that sees the whole distribution at every token, or vllm_backend.VLLMBackend, a vLLM server. tests/fakes.FakeLLM replays scripted replies. A backend that asks another serving engine, such as SGLang or Ollama, would be one more; this module says what it has to provide. diff --git a/harness/families/__init__.py b/harness/families/__init__.py index 9806195..c5bf4ed 100644 --- a/harness/families/__init__.py +++ b/harness/families/__init__.py @@ -1,6 +1,6 @@ """Model families: everything that changes when the model changes. -The harness (agent.py), the decoding loop (simple_backend.py) and the page (render.py) are written once. +This project's harness (harnesses/simple_harness.py), the decoding loop (simple_backend.py) and the page (render.py) are written once. What differs between Gemma 4 and Qwen3.5 is how the model writes its turns, its thinking and its tool calls, which tokens stop it, and what its control tokens mean. A family module describes all of that in one Family, and everything else asks the family. diff --git a/harness/harnesses/__init__.py b/harness/harnesses/__init__.py new file mode 100644 index 0000000..db7db5c --- /dev/null +++ b/harness/harnesses/__init__.py @@ -0,0 +1,7 @@ +"""What runs the agent loop, behind the one interface the app knows. + +harness.py is that interface (Harness) and the events every harness sends. simple_harness.py is this project's own +harness (SimpleHarness), which runs the model through a Backend (backends/); pi_harness.py hands the loop to pi, a +coding agent that runs its own (PiHarness), and pi_approve.ts is the pi extension through which the page asks before +pi runs a tool. +""" diff --git a/harness/harnesses/harness.py b/harness/harnesses/harness.py new file mode 100644 index 0000000..70863e5 --- /dev/null +++ b/harness/harnesses/harness.py @@ -0,0 +1,55 @@ +"""The interface between the page and whatever runs the agent loop. + +The page (app/page.py) runs one turn at a time and reaches the loop only through a Harness: it makes one with a +function that appends each event to the turn's list, calls run() in a background thread, and passes the user's +clicks on with stop() and decide(). render.py draws the list. This project's own harness is +simple_harness.SimpleHarness, which runs the model through a Backend (backends/backend.py), so it knows every token; +pi_harness.PiHarness hands the loop to pi, a coding agent that runs its own and says what it does as JSON. Another +agent that reports its loop as events, such as the Claude Agent SDK or Codex, would be one more; this module says +what it has to send. + +A harness that runs the model through a Backend knows the prompt the model reads and each token it writes; one that +asks a model's API sees only the text that comes back. So, like a token event's details (backend.TOKEN_DETAILS), most +events are sent only by a harness that knows what they say, and render.py draws a step from whichever it has: the page +never shows a prompt or a number the harness did not see. + +Nothing here imports anything, so a harness can be written, and the interface read, without the rest. +""" + +from typing import Protocol, runtime_checkable + +# The events every harness sends: run_start first, a step_start each time around the loop, and run_end last (after +# an error event when something failed). run_start carries which harness ran the turn (`harness`: "simple", "pi"), the +# model, its family (a key of families.FAMILIES, or None when the harness does not know how the model writes its turns), +# the turn's settings and the tools the model gets; run_end the stop reason, the final reply and the conversation. +ALWAYS = ("run_start", "step_start", "run_end") +# The events a harness sends in a step when it knows what they say, in the order they come: +# context the prompt as the model reads it, from a harness that renders it (simple_harness.SimpleHarness) +# prefill, token the prompt read, then each token (backend.TOKEN_FIELDS), from a harness that runs the model itself +# delta a piece of the model's output as it streams, with its channel, from one that sees text, not tokens (pi) +# model_end the model's turn ended: why, and how many tokens, with their speed when it was measured +# parsed what the harness read in it: the thinking, the reply, the tool calls +# tool_call a call it will run (or reports as failed), with its environment +# approval_request, approval the call held for the user, and the user's answer +# tool_start, tool_result the call running in its environment, and what came back +# error what went wrong, before run_end +SOMETIMES = ("context", "prefill", "token", "delta", "model_end", "parsed", "tool_call", "approval_request", "approval", + "tool_start", "tool_result", "error") + + +@runtime_checkable +class Harness(Protocol): + """What runs one turn. The page makes one per question (app.page.LiveRun) and runs one at a time.""" + + id: str + + def run(self) -> None: + """Run the turn to its end, sending its events. It returns after run_end and never raises: a failure is an + error event followed by run_end.""" + + def stop(self) -> None: + """Stop the turn as soon as it can: the user pressed Stop. It is called from another thread, and run() still + ends with run_end.""" + + def decide(self, call_id: str, approved: bool) -> None: + """The user's answer to the approval_request for the call `call_id`. It is called from another thread.""" diff --git a/harness/harnesses/pi_approve.ts b/harness/harnesses/pi_approve.ts new file mode 100644 index 0000000..ff85af4 --- /dev/null +++ b/harness/harnesses/pi_approve.ts @@ -0,0 +1,22 @@ +/** + * Ask me before running a tool, for pi run by the Agent Loop Visualizer (harness/harnesses/pi_harness.py). + * + * Before pi runs a tool call, this asks through a confirm dialog. In RPC mode the dialog is an extension_ui_request on + * stdout, which pi_harness.py turns into the page's Allow and Reject buttons; the answer comes back on stdin as an + * extension_ui_response. A call the user rejects does not run, and the model reads the refusal as its result, in the + * language the turn started in (AGENT_LOOP_REFUSAL, which pi_harness.py sets). + */ + +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +const TITLE = "agent-loop:approve"; // pi_harness.APPROVE_TITLE, how pi_harness.py tells this dialog from any other + +export default function (pi: ExtensionAPI) { + pi.on("tool_call", async (event, ctx) => { + const call = JSON.stringify({ toolCallId: event.toolCallId, toolName: event.toolName, input: event.input }); + if (!(await ctx.ui.confirm(TITLE, call))) { + return { block: true, reason: process.env.AGENT_LOOP_REFUSAL || "The user refused to run this tool call" }; + } + return undefined; + }); +} diff --git a/harness/harnesses/pi_harness.py b/harness/harnesses/pi_harness.py new file mode 100644 index 0000000..020313a --- /dev/null +++ b/harness/harnesses/pi_harness.py @@ -0,0 +1,468 @@ +"""pi as the harness: the page hands the turn to pi (https://pi.dev), a coding agent that runs a loop of its own, +and draws what pi says it does. + +pi runs as a child process in RPC mode (`pi --mode rpc`): commands go in on its stdin and records come out on its +stdout, one JSON object per line. For each question PiHarness starts pi in the conversation's folder, picks the model and +the thinking level, sends the question as a prompt and turns pi's events into the page's (harness.py): one of pi's +turns (turn_start to turn_end: one request to the model and the tools its reply calls) is a step, the text the model +streams is delta, the finished assistant message is model_end and parsed, and each tool execution is tool_call, +tool_start and tool_result. pi keeps the conversation in a session file in the conversation's folder, so the next +question starts pi on the same session and the model reads all of it. + +pi asks a model's API (Anthropic, OpenAI, Google, ...) or any endpoint in its models.json, with the credentials it +keeps itself: `pi` then /login, or an API key in the environment. An API sends back text, not tokens, so a step shows +the text as it streams and the token counts the provider reported, and no probabilities; nor does pi say what prompt +it sends, so a step has no context section either. + +pi's tools (read, bash, edit, write, grep, find, ls) run on this machine, in the conversation's folder, without a +sandbox: bash reaches whatever the user can. Ask me before running a tool goes through pi_approve.ts, a pi extension +that asks before every call. In RPC mode its question is a dialog on stdout (extension_ui_request), which becomes the +page's approval_request; the user's answer goes back on stdin. + +The page starts pi without the user's own extensions, skills, prompt templates and context files (AGENTS.md), so +that what it draws is pi's own loop, the same on every machine; the models and the credentials are still the user's. +""" + +import base64 +import collections +import json +import mimetypes +import os +import queue +import shlex +import subprocess +import tempfile +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +from harness import media +from harness.harnesses.simple_harness import APPROVAL_TIMEOUT +from harness.i18n import pick + +# How to start pi: the `pi` on the PATH, unless PI_COMMAND says otherwise (a path, or "npx -y @earendil-works/pi-coding-agent"). +PI_COMMAND = shlex.split(os.environ.get("PI_COMMAND", "pi")) +INSTALL = "npm install -g @earendil-works/pi-coding-agent" +FLAGS = ["--mode", "rpc", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files"] +FILES = {"zh": "這台電腦的檔案", "en": "files on this computer"} +# pi's built-in tools: what each does and where it runs, as the page says it. +TOOLS = { + "read": ({"zh": "讀一個檔案（文字或圖片）", "en": "reads a file (text or an image)"}, FILES), + "bash": ({"zh": "執行 shell 指令", "en": "runs a shell command"}, {"zh": "這台電腦的 shell", "en": "shell on this computer"}), + "edit": ({"zh": "把檔案裡的一段文字換成另一段", "en": "replaces exact text in a file"}, FILES), + "write": ({"zh": "新建或覆寫一個檔案", "en": "creates or overwrites a file"}, FILES), + "grep": ({"zh": "搜尋檔案的內容", "en": "searches the contents of files"}, FILES), + "find": ({"zh": "用 glob 找檔案的路徑", "en": "finds paths by a glob pattern"}, FILES), + "ls": ({"zh": "列出一個資料夾", "en": "lists a folder"}, FILES), +} +OTHER_TOOL = {"zh": "pi 的工具", "en": "a pi tool"} # where a tool runs that pi has and the page does not know +DEFAULT_TOOLS = ("read", "bash", "edit", "write") # what pi offers the model unless told otherwise +THINKING = ("off", "minimal", "low", "medium", "high") # pi's thinking levels for a model that thinks +APPROVE = Path(__file__).with_name("pi_approve.ts") +APPROVE_TITLE = "agent-loop:approve" # the title pi_approve.ts gives its dialog +SESSION_DIR = ".pi-session" # in the conversation's folder +START_TIMEOUT = 30 # seconds pi may take to start and to answer a command +STOP_GRACE = 10 # seconds pi has to settle after an abort, or to exit after its stdin closes, before it is killed +DIALOGS = ("select", "confirm", "input", "editor") # the extension_ui_request methods that wait for an answer + + +class PiError(Exception): + """pi could not be started, refused a command or exited; `text` says so in both of the page's languages.""" + + def __init__(self, zh: str, en: str): + super().__init__(en) + self.text = {"zh": zh, "en": en} + + +def environment_of(tool: str) -> dict[str, str]: + return TOOLS[tool][1] if tool in TOOLS else OTHER_TOOL + + +def not_found() -> PiError: + command = " ".join(PI_COMMAND) + return PiError( + f"找不到 pi（`{command}`）。用 `{INSTALL}` 安裝它（需要 Node.js 22.19 以上），或用環境變數 `PI_COMMAND` 指定怎麼啟動它。", + f"pi (`{command}`) was not found. Install it with `{INSTALL}` (it needs Node.js 22.19 or later), or say how to start it with `PI_COMMAND`.", + ) + + +class Process: + """One pi in RPC mode: commands are written to its stdin, and a thread reads its stdout into `inbox`.""" + + def __init__(self, args: list[str], cwd: Path, inbox: queue.Queue, env: dict | None = None): + try: + self.proc = subprocess.Popen( + [*PI_COMMAND, *FLAGS, *args], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, env={**os.environ, "PI_SKIP_VERSION_CHECK": "1", **(env or {})}, + ) + except (FileNotFoundError, PermissionError): + raise not_found() from None + self.inbox = inbox + self.backlog: collections.deque = collections.deque() # what came while request() waited for its response + self.stderr: collections.deque = collections.deque(maxlen=20) + self._lock = threading.Lock() + threading.Thread(target=self._read, daemon=True).start() + threading.Thread(target=self._read_stderr, daemon=True).start() + + def _read(self): + # A binary pipe splits lines on LF alone, as RPC's framing asks: a JSON string may hold U+2028. + for line in self.proc.stdout: + try: + self.inbox.put(("record", json.loads(line))) + except json.JSONDecodeError: + continue + self.inbox.put(("exit", None)) + + def _read_stderr(self): + for line in self.proc.stderr: + self.stderr.append(line.decode(errors="replace").rstrip()) + + def send(self, command: dict): + with self._lock: + try: + self.proc.stdin.write(json.dumps(command, ensure_ascii=False).encode() + b"\n") + self.proc.stdin.flush() + except (OSError, ValueError): # pi has exited, which its stdout says too + pass + + def request(self, command: dict, timeout: float = START_TIMEOUT) -> dict: + """Send a command and wait for its response; return the response's data. What comes meanwhile is kept, in + order, in `backlog`.""" + rid = uuid.uuid4().hex[:8] + self.send({**command, "id": rid}) + deadline = time.monotonic() + timeout + while True: + try: + kind, value = self.inbox.get(timeout=max(deadline - time.monotonic(), 0.01)) + except queue.Empty: + raise PiError(f"pi 在 {timeout} 秒內沒有回應 `{command['type']}`。", f"pi did not answer `{command['type']}` within {timeout} s.") from None + if kind == "exit": + raise self.exited() + if kind == "record" and value.get("type") == "response" and value.get("id") == rid: + if not value.get("success"): + error = value.get("error") or "" + raise PiError(f"pi 拒絕了 `{command['type']}`：{error}", f"pi refused `{command['type']}`: {error}") + return value.get("data") or {} + self.backlog.append((kind, value)) + + def exited(self) -> PiError: + said = "\n".join(self.stderr) + return PiError("pi 意外結束了。" + (f"它最後說：\n{said}" if said else ""), "pi exited unexpectedly." + (f" Its last words:\n{said}" if said else "")) + + def close(self, grace: float = STOP_GRACE): + """Close stdin, which asks pi to shut down, and kill it if it has not within `grace` seconds.""" + with self._lock: + try: + self.proc.stdin.close() + except OSError: + pass + try: + self.proc.wait(grace) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + +def probe() -> dict: + """What pi can run: its version, the models it has credentials for (pi's own model objects: provider, id, name, + input, reasoning, contextWindow, maxTokens, cost) and the one it starts on by itself ("provider/id", or None). + Raises PiError when pi does not start or answer.""" + try: + done = subprocess.run([*PI_COMMAND, "--version"], capture_output=True, text=True, timeout=START_TIMEOUT) + except (FileNotFoundError, PermissionError): + raise not_found() from None + except subprocess.TimeoutExpired: + raise PiError(f"`{' '.join(PI_COMMAND)} --version` 沒有結束。", f"`{' '.join(PI_COMMAND)} --version` did not finish.") from None + if done.returncode: + said = (done.stderr or done.stdout).strip()[-1000:] + raise PiError(f"pi 沒有啟動：\n{said}", f"pi did not start:\n{said}") + with tempfile.TemporaryDirectory() as folder: # somewhere with no project settings of its own + process = Process(["--no-session", "--no-tools"], Path(folder), queue.Queue()) + try: + models = process.request({"type": "get_available_models"})["models"] + current = process.request({"type": "get_state"}).get("model") + finally: + process.close() + names = [f"{m['provider']}/{m['id']}" for m in models] + start = f"{current['provider']}/{current['id']}" if current else None + return {"version": done.stdout.strip(), "models": models, "default": start if start in names else None} + + +@dataclass +class PiSettings: + provider: str # the model, as pi names it: its provider and its id there + model: str + thinking: str = "off" # one of THINKING + tools: tuple[str, ...] = DEFAULT_TOOLS + approve_tools: bool = False + max_steps: int = 6 + lang: str = "zh" # what pi_approve.ts tells the model when the user refuses a call is worded in this language + + +class PiHarness: + """One turn, run by pi (harness.Harness).""" + + def __init__(self, settings: PiSettings, history: list[dict], text: str, emit: Callable[[dict], None], + attachments: list[media.Attachment] = (), folder: Path | None = None, session: str = "chat"): + self.id = uuid.uuid4().hex[:12] + self.settings = settings + self.history = history # only for run_end: pi keeps the conversation in its session + self.text = text + self.attachments = list(attachments) + self.folder = Path(folder or tempfile.mkdtemp()) # the conversation's: pi works in it and keeps its session there + self.session = session # the session's id, the same for every question of the conversation + self._emit = emit + self._start = time.perf_counter() + self._inbox: queue.Queue = queue.Queue() # pi's records, and the user's clicks, for run() to handle in order + self._pi: Process | None = None + self._step, self._open = 0, False # the page's step, and whether pi's current turn is one (not after an abort) + self._turn_t = self._stream_t = 0.0 + self._wrote = False # whether the model has streamed anything yet in this message + self._calls: dict[str, dict] = {} # call id -> name, arguments and step of each call drawn + self._pending: dict[str, tuple[str, float]] = {} # call id -> the dialog's id and when it times out + self._seen: list[tuple[str, str]] = [] + self._last: dict | None = None # the last assistant message of a step + self._error: str | None = None + self._aborting: str | None = None # why the turn is being stopped: user_stopped, max_steps, approval_timeout + self._kill_at: float | None = None # after an abort: when to stop waiting for pi to settle + self._hung = False # pi did not settle in time, so it is killed rather than asked to exit + + # --- controls called from the web server --- + + def stop(self): + self._inbox.put(("stop", None)) + + def decide(self, call_id: str, approved: bool): + self._inbox.put(("decide", (call_id, approved))) + + # --- the turn --- + + def emit(self, event_type: str, **data): + self._emit({"type": event_type, "t": round((time.perf_counter() - self._start) * 1000), **data}) + + def run(self): + s = self.settings + self.emit( + "run_start", run_id=self.id, model=f"{s.provider}/{s.model}", family=None, harness="pi", + settings={"model": f"{s.provider}/{s.model}", "thinking": s.thinking, "use_tools": bool(s.tools), "tools": list(s.tools), + "approve_tools": s.approve_tools, "max_steps": s.max_steps, "lang": s.lang}, + tools=list(s.tools), + ) + stop_reason = "error" + try: + self.folder.mkdir(parents=True, exist_ok=True) + self._pi = Process(self._args(), self.folder, self._inbox, { + "AGENT_LOOP_REFUSAL": pick(s.lang, "使用者拒絕執行這個工具呼叫", "The user refused to run this tool call"), + }) + # Each waits for its answer: pi handles commands concurrently, and a prompt must not go to the model it starts on. + self._pi.request({"type": "set_model", "provider": s.provider, "modelId": s.model}) + self._pi.request({"type": "set_thinking_level", "level": s.thinking}) + # A failed request is retried by pi as a new turn, which would be a step of its own; the page shows the error instead. + self._pi.request({"type": "set_auto_retry", "enabled": False}) + message, images = self._message() + if not self._stopped_meanwhile(): + self._pi.request({"type": "prompt", "message": message, **({"images": images} if images else {})}) + self._loop() + stop_reason = self._stop_reason() + if stop_reason == "error": + self.emit("error", message=f"pi: {self._error}") + except PiError as e: # pi did not start, refused, or exited + self.emit("error", message=f"PiError: {e}", text={lang: f"PiError: {t}" for lang, t in e.text.items()}) + except Exception as e: # a crash here is shown, not a silent hang + self.emit("error", message=f"{type(e).__name__}: {e}") + finally: + if self._pi: + self._pi.close(0 if self._hung else STOP_GRACE) + final = self._text(self._last) if self._last and stop_reason != "error" else "" + self.emit("run_end", stop_reason=stop_reason, final=final, + messages=[*self.history, {"role": "user", "content": self.text}, {"role": "assistant", "content": final}]) + + def _args(self) -> list[str]: + s = self.settings + args = ["--session-dir", str(self.folder / SESSION_DIR), "--session-id", self.session] + args += ["--tools", ",".join(s.tools)] if s.tools else ["--no-tools"] + return args + (["--extension", str(APPROVE)] if s.approve_tools else []) + + def _message(self) -> tuple[str, list[dict]]: + """The question, with a line for the model on the files attached (in English, as all the model reads from the + page is), and the images it reads inline.""" + images = [{"type": "image", "data": base64.b64encode(Path(a.path).read_bytes()).decode(), "mimeType": mimetypes.guess_type(a.name)[0] or "image/png"} + for a in self.attachments if a.kind == "image"] + files = ", ".join(f"{a.name} ({media.fmt_size(a.size)})" for a in self.attachments) + note = f"Attached, in the current folder: {files}" if files else "" + return "\n\n".join(t for t in (self.text, note) if t), images + + def _stopped_meanwhile(self) -> bool: + """Whether Stop was pressed while pi started, before the question went to it.""" + if any(kind == "stop" for kind, _ in self._pi.backlog): + self._aborting = "user_stopped" + return self._aborting is not None + + def _loop(self): + while True: + if self._pi.backlog: + kind, value = self._pi.backlog.popleft() + else: + try: + kind, value = self._inbox.get(timeout=0.5) + except queue.Empty: + kind, value = "tick", None + now = time.monotonic() + if kind == "exit": + if not self._error and not self._aborting: + raise self._pi.exited() + return + if kind == "stop": + self._abort("user_stopped") + elif kind == "decide": + self._decided(*value) + elif kind == "record" and self._on(value): + return + if any(deadline < now for _, deadline in self._pending.values()): + self._abort("approval_timeout") + if self._kill_at and now > self._kill_at: # pi did not settle after the abort + self._hung = True + return + + def _abort(self, reason: str): + if self._aborting: + return + self._aborting = reason + for call_id, (dialog, _) in list(self._pending.items()): + timed_out = reason == "approval_timeout" + self.emit("approval", step=self._calls[call_id]["step"], call_id=call_id, approved=False, timed_out=timed_out, stopped=not timed_out) + self._pi.send({"type": "extension_ui_response", "id": dialog, "cancelled": True}) + self._pending.clear() + self._pi.send({"type": "abort"}) + self._kill_at = time.monotonic() + STOP_GRACE + + def _stop_reason(self) -> str: + if self._aborting: + return self._aborting + if self._error: + return "error" + if self._last and self._last.get("stopReason") == "length": + return "max_new_tokens" + return "no_tool_call" + + # --- pi's records --- + + def _on(self, rec: dict) -> bool: + """Turn one of pi's records into the page's events; True once pi has settled.""" + kind = rec.get("type") + if kind == "turn_start": + self._turn_start() + elif kind == "message_start" and self._open and rec["message"].get("role") == "assistant": + self._stream_t, self._wrote = time.perf_counter(), False + elif kind == "message_update" and self._open: + self._update(rec["assistantMessageEvent"]) + elif kind == "message_end" and self._open and rec["message"].get("role") == "assistant": + self._message_end(rec["message"]) + elif kind == "tool_execution_start" and self._open and not rec.get("parentToolCallId"): + self._tool_start(rec["toolCallId"], rec["toolName"], rec.get("args") or {}) + elif kind == "tool_execution_end" and rec.get("toolCallId") in self._calls and not rec.get("parentToolCallId"): + self._tool_end(rec["toolCallId"], rec.get("result") or {}, rec.get("isError", False), rec.get("durationMs")) + elif kind == "extension_ui_request": + self._dialog(rec) + return kind == "agent_settled" + + def _turn_start(self): + if self._aborting: # pi goes on to the turn its tool results start before the abort reaches it + self._open = False + return + if self._step >= self.settings.max_steps: # pi has no step limit of its own; the page's is kept by stopping it + self._abort("max_steps") + self._open = False + return + self._step += 1 + self._open = True + self._turn_t = self._stream_t = time.perf_counter() + self.emit("step_start", step=self._step) + + def _update(self, e: dict): + kind = e.get("type") + if kind == "text_delta": + self._delta("content", e.get("delta", "")) + elif kind == "thinking_delta": + self._delta("thought", e.get("delta", "")) + elif kind == "toolcall_start": + self._delta("tool_call", ("\n" if self._wrote else "") + e.get("toolName", "") + " ") + elif kind == "toolcall_delta": + self._delta("tool_call", e.get("delta", "")) + + def _delta(self, channel: str, text: str): + if text: + self._wrote = True + self.emit("delta", step=self._step, channel=channel, text=text) + + def _message_end(self, message: dict): + now = time.perf_counter() + usage = message.get("usage") or {} + reason = message.get("stopReason") + if reason == "error" and not self._aborting: + self._error = message.get("errorMessage") or reason + finish = "user_stopped" if self._aborting else {"length": "max_new_tokens", "aborted": "user_stopped", "error": "error"}.get(reason, "end_turn") + n, seconds = usage.get("output") or 0, now - self._stream_t + self.emit( + "model_end", step=self._step, finish=finish, stop_piece=None, n_tokens=n, + first_ms=round((self._stream_t - self._turn_t) * 1000), decode_ms=round(seconds * 1000), + # tokens over the time they took to arrive, network included; nothing when they came all at once + tok_per_s=round(n / seconds, 1) if n and seconds > 0.05 else None, mtp=None, + usage={"prompt": (usage.get("input") or 0) + (usage.get("cacheRead") or 0) + (usage.get("cacheWrite") or 0), + "cached": usage.get("cacheRead") or 0, "output": n, "cost": (usage.get("cost") or {}).get("total")}, + ) + blocks = message.get("content") or [] + calls = [{"name": b.get("name", ""), "arguments": b.get("arguments") or {}, "raw_args": json.dumps(b.get("arguments") or {}, ensure_ascii=False), + "error": None} for b in blocks if b.get("type") == "toolCall"] + thinking = "\n\n".join(b["thinking"] for b in blocks if b.get("type") == "thinking" and b.get("thinking")) + self.emit("parsed", step=self._step, thinking=thinking, content=self._text(message), tool_calls=calls) + self._last = message + + @staticmethod + def _text(message: dict) -> str: + return "".join(b.get("text", "") for b in message.get("content") or [] if b.get("type") == "text") + + def _tool_start(self, call_id: str, name: str, args: dict): + key = (name, json.dumps(args, sort_keys=True, ensure_ascii=False)) + repeated = key in self._seen + self._seen.append(key) + self._calls[call_id] = {"name": name, "arguments": args, "step": self._step} + environment = environment_of(name) + self.emit("tool_call", step=self._step, call_id=call_id, name=name, arguments=args, raw_args=json.dumps(args, ensure_ascii=False), + errors=[], repeated=repeated, environment=environment) + if not self.settings.approve_tools: # otherwise it starts once the user allows it (_decided) + self.emit("tool_start", step=self._step, call_id=call_id, name=name, environment=environment) + + def _tool_end(self, call_id: str, result: dict, is_error: bool, ms: float | None): + call = self._calls[call_id] + parts = result.get("content") or [] + text = "\n".join(p.get("text", "") if p.get("type") == "text" else f"[{p.get('type')}]" for p in parts) + truncated = bool((result.get("structuredContent") or {}).get("truncated") or (result.get("details") or {}).get("truncation")) + self.emit("tool_result", step=call["step"], call_id=call_id, name=call["name"], ok=not is_error, result=text, + truncated=truncated, ms=round(ms or 0, 1), images=[]) + + def _dialog(self, rec: dict): + method = rec.get("method") + if method == "confirm" and rec.get("title") == APPROVE_TITLE: + call_id = json.loads(rec.get("message") or "{}").get("toolCallId") + if self._aborting or call_id not in self._calls: + self._pi.send({"type": "extension_ui_response", "id": rec["id"], "cancelled": True}) + return + self._pending[call_id] = (rec["id"], time.monotonic() + APPROVAL_TIMEOUT) + call = self._calls[call_id] + self.emit("approval_request", step=call["step"], call_id=call_id, name=call["name"], arguments=call["arguments"]) + elif method in DIALOGS: # no other extension is loaded, so nothing else should ask; an answer keeps pi from waiting + self._pi.send({"type": "extension_ui_response", "id": rec["id"], "cancelled": True}) + + def _decided(self, call_id: str, approved: bool): + if call_id not in self._pending: + return + dialog, _ = self._pending.pop(call_id) + call = self._calls[call_id] + self.emit("approval", step=call["step"], call_id=call_id, approved=approved, timed_out=False, stopped=False) + if approved: + self.emit("tool_start", step=call["step"], call_id=call_id, name=call["name"], environment=environment_of(call["name"])) + self._pi.send({"type": "extension_ui_response", "id": dialog, "confirmed": approved}) diff --git a/harness/agent.py b/harness/harnesses/simple_harness.py similarity index 97% rename from harness/agent.py rename to harness/harnesses/simple_harness.py index 2021aa6..532b9e7 100644 --- a/harness/agent.py +++ b/harness/harnesses/simple_harness.py @@ -1,4 +1,4 @@ -"""The harness: the program around the model. +"""SimpleHarness: this project's own harness, the program around the model. Each step renders the conversation into one prompt, lets the model generate, and reads what came out. A tool call is parsed, validated, optionally approved by the @@ -6,7 +6,8 @@ A reply without a tool call ends the run; so do the step limit, the per-step token limit, repeated malformed calls and the stop button. Every stage is emitted as an event so the browser can draw it. The model is whatever Backend (backend.py) the page -hands it, so the loop is the same whichever runs the model. +hands it, so the loop is the same whichever runs the model. It is one Harness +(harness.py); the page can hand a turn to pi instead (pi_harness.py). A run keeps the page's language from when it started: the errors and refusals the model reads come back in it. The system prompt is the same in both languages. @@ -62,7 +63,7 @@ def common_prefix(a: str, b: str) -> int: return i -class Run: +class SimpleHarness: def __init__(self, llm: Backend, history: list[dict], user_text: str, settings: RunSettings, emit: Callable[[dict], None], attachments: list[media.Attachment] = (), files: Path | None = None): self.id = uuid.uuid4().hex[:12] @@ -102,7 +103,7 @@ def run(self): schemas = {t.name: t.parameters for t in TOOLS.values()} system = {"role": "system", "content": SYSTEM_WITH_TOOLS if tools else SYSTEM_NO_TOOLS} self.emit( - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, settings=_settings_dict(s), tools=list(self.tools), ) diff --git a/tests/fake_pi.py b/tests/fake_pi.py new file mode 100644 index 0000000..38e44aa --- /dev/null +++ b/tests/fake_pi.py @@ -0,0 +1,243 @@ +"""A scripted stand-in for pi in RPC mode (harness/harnesses/pi_harness.py), run as a child process the way pi is. + +It speaks pi's protocol as pi 1.4 does (docs/rpc.md, docs/json.md): commands on stdin, responses and events on +stdout, one JSON object per line. What its model says comes from the JSON file in FAKE_PI_SCRIPT: + + {"models": [pi model objects], "turns": [turn, ...], "delay": seconds between streamed pieces} + +and each turn is {"stop": pi's stopReason, "error": its errorMessage, "usage": {...}, "say": [item, ...]}, where an item +is {"thinking": text}, {"text": text} or {"tool": name, "args": {...}, "result": text, "fails": bool}. A turn with +"forever": true streams until it is aborted, and with "deaf": true even then. Loaded with --extension (pi_approve.ts), it asks before each tool, as the +extension does. What it was started with and every command it read are appended to FAKE_PI_LOG, one JSON per line. +FakePi, at the end, puts it in pi's place for a test. +""" + +import json +import os +import queue +import sys +import threading +import time + +ARGS = sys.argv[1:] +LOG = os.environ.get("FAKE_PI_LOG") +SCRIPT = json.loads(open(os.environ["FAKE_PI_SCRIPT"]).read()) if os.environ.get("FAKE_PI_SCRIPT") else {"turns": []} +DELAY = SCRIPT.get("delay", 0.005) +MODELS = SCRIPT.get("models", [{"id": "fake-model", "name": "Fake Model", "provider": "fake", "api": "openai-completions", + "reasoning": False, "input": ["text", "image"], "contextWindow": 32000, "maxTokens": 2048, + "cost": {"input": 1, "output": 2, "cacheRead": 0, "cacheWrite": 0}}]) +inbox: queue.Queue = queue.Queue() +model = MODELS[0] if MODELS else None + + +def log(entry: dict): + if LOG: + with open(LOG, "a") as f: + f.write(json.dumps(entry) + "\n") + + +def out(record: dict): + sys.stdout.write(json.dumps(record) + "\n") + sys.stdout.flush() + + +def read_stdin(): + for line in sys.stdin: + command = json.loads(line) + log({"command": command}) + inbox.put(command) + inbox.put(None) + + +class Aborted(Exception): + pass + + +def check(abortable=True): + """Handle the commands that came meanwhile; raise Aborted on an abort.""" + while True: + try: + command = inbox.get_nowait() + except queue.Empty: + return + if command is None: + sys.exit(0) + if command.get("type") == "abort": + out({"id": command.get("id"), "type": "response", "command": "abort", "success": True}) + if abortable and not SCRIPT.get("deaf"): + raise Aborted + else: + answer(command) + + +def answer(command: dict): + global model + kind, rid = command.get("type"), command.get("id") + ok = {"id": rid, "type": "response", "command": kind, "success": True} + if kind == "get_available_models": + out({**ok, "data": {"models": MODELS}}) + elif kind == "get_state": + out({**ok, "data": {"model": model, "thinkingLevel": "off", "isStreaming": False}}) + elif kind == "set_model": + found = next((m for m in MODELS if m["provider"] == command["provider"] and m["id"] == command["modelId"]), None) + if found: + model = found + out({**ok, "data": found}) + else: + out({**ok, "success": False, "error": f"Model not found: {command['provider']}/{command['modelId']}"}) + elif kind == "prompt": + if SCRIPT.get("refuse"): + out({**ok, "success": False, "error": SCRIPT["refuse"]}) + return + out({**ok, "data": {"disposition": "started"}}) + run(command) + elif kind == "extension_ui_response": + pass # an answer to a dialog nobody is waiting for any more + else: + out(ok) + + +def wait_for_answer(dialog: str) -> bool: + """Block until the client answers the dialog, as pi's ctx.ui.confirm() does; an abort cancels it.""" + while True: + command = inbox.get() + if command is None: + sys.exit(0) + if command.get("type") == "extension_ui_response" and command.get("id") == dialog: + return bool(command.get("confirmed")) + if command.get("type") == "abort": + out({"id": command.get("id"), "type": "response", "command": "abort", "success": True}) + raise Aborted + answer(command) + + +def assistant(content, stop="pending", usage=None, error=None): + message = {"role": "assistant", "content": content, "api": "openai-completions", "provider": model["provider"], "model": model["id"], + "usage": usage or {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 0, "cost": {"total": 0}}, + "stopReason": stop} + return {**message, "errorMessage": error} if error else message + + +def stream(kind: str, index: int, text: str): + for i in range(0, len(text), 4): + check() + out({"type": "message_update", "assistantMessageEvent": {"type": kind, "contentIndex": index, "delta": text[i:i + 4]}}) + time.sleep(DELAY) + + +def run(prompt: dict): + approve = "--extension" in ARGS + out({"type": "agent_start"}) + out({"type": "turn_start"}) + out({"type": "message_start", "message": {"role": "user", "content": [{"type": "text", "text": prompt["message"]}]}}) + out({"type": "message_end", "message": {"role": "user", "content": [{"type": "text", "text": prompt["message"]}]}}) + aborted, calls = False, 0 + try: + for n, turn in enumerate(SCRIPT["turns"]): + if n: + check() + out({"type": "turn_start"}) + time.sleep(DELAY) + check() + content: list = [] + out({"type": "message_start", "message": assistant([])}) + if turn.get("forever"): + while True: + stream("text_delta", 0, "more ") + for i, item in enumerate(turn.get("say", [])): + if "thinking" in item: + stream("thinking_delta", i, item["thinking"]) + content.append({"type": "thinking", "thinking": item["thinking"]}) + elif "text" in item: + stream("text_delta", i, item["text"]) + content.append({"type": "text", "text": item["text"]}) + else: + calls += 1 + call = {"type": "toolCall", "id": f"call_{calls}", "name": item["tool"], "arguments": item.get("args", {})} + out({"type": "message_update", "assistantMessageEvent": {"type": "toolcall_start", "contentIndex": i, "id": call["id"], "toolName": item["tool"]}}) + stream("toolcall_delta", i, json.dumps(call["arguments"])) + content.append(call) + usage = {"input": 1000, "output": 20, "cacheRead": 200, "cacheWrite": 0, "totalTokens": 1220, "cost": {"total": 0.0021}, **turn.get("usage", {})} + calls_here = [b for b in content if b["type"] == "toolCall"] + message = assistant(content, turn.get("stop", "toolUse" if calls_here else "stop"), usage, turn.get("error")) + out({"type": "message_end", "message": message}) + results = [] + for call in calls_here: + item = next(i for i in turn["say"] if i.get("tool") == call["name"]) + out({"type": "tool_execution_start", "toolCallId": call["id"], "toolName": call["name"], "args": call["arguments"]}) + allowed = True + if approve: + dialog = f"dialog-{call['id']}" + out({"type": "extension_ui_request", "id": dialog, "method": "confirm", "title": "agent-loop:approve", + "message": json.dumps({"toolCallId": call["id"], "toolName": call["name"], "input": call["arguments"]})}) + allowed = wait_for_answer(dialog) + if allowed: + time.sleep(DELAY) + result = {"content": [{"type": "text", "text": item.get("result", "")}], "details": {}} + out({"type": "tool_execution_end", "toolCallId": call["id"], "toolName": call["name"], "result": result, + "isError": bool(item.get("fails")), "durationMs": 3}) + else: + refusal = os.environ.get("AGENT_LOOP_REFUSAL", "") + out({"type": "tool_execution_end", "toolCallId": call["id"], "toolName": call["name"], + "result": {"content": [{"type": "text", "text": refusal}], "details": {}}, "isError": True}) + results.append(call["id"]) + out({"type": "turn_end", "message": message, "toolResults": results}) + if message["stopReason"] in ("stop", "length", "error"): + break + except Aborted: + aborted = True + message = assistant([], "aborted") + out({"type": "message_end", "message": message}) + out({"type": "turn_end", "message": message, "toolResults": []}) + out({"type": "agent_end", "messages": [], "willRetry": False}) + out({"type": "agent_settled", "aborted": aborted}) + + +def main(): + log({"args": ARGS, "cwd": os.getcwd(), "refusal": os.environ.get("AGENT_LOOP_REFUSAL")}) + if ARGS == ["--version"]: + print(SCRIPT.get("version", "1.4.0")) + return + if SCRIPT.get("crash"): + sys.stderr.write(SCRIPT["crash"] + "\n") + sys.exit(1) + threading.Thread(target=read_stdin, daemon=True).start() + while True: + command = inbox.get() + if command is None: + return + try: + answer(command) + except Aborted: + pass + + +class FakePi: + """For a test: this file in pi's place (pi_harness.PI_COMMAND). say() writes what its model says next; log() reads back + what each pi started was given and was sent.""" + + def __init__(self, monkeypatch, folder): + from harness.harnesses import pi_harness # only in the test's process + + self.script, self.log_file = folder / "fake_pi_script.json", folder / "fake_pi.log" + monkeypatch.setattr(pi_harness, "PI_COMMAND", [sys.executable, __file__]) + monkeypatch.setenv("FAKE_PI_SCRIPT", str(self.script)) + monkeypatch.setenv("FAKE_PI_LOG", str(self.log_file)) + self.say() + + def say(self, *turns, **more): + self.script.write_text(json.dumps({"turns": list(turns), **more})) + + def log(self) -> list[dict]: + return [json.loads(line) for line in self.log_file.read_text().splitlines()] if self.log_file.exists() else [] + + def commands(self) -> list[dict]: + return [e["command"] for e in self.log() if "command" in e] + + def started(self) -> dict: + """What the last pi that ran a turn was started with (not a `pi --version`).""" + return [e for e in self.log() if "args" in e and e["args"] != ["--version"]][-1] + + +if __name__ == "__main__": + main() diff --git a/tests/fakes.py b/tests/fakes.py index d9c1511..bcb884a 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -331,7 +331,7 @@ def qwen_tool_call(name: str, **args) -> str: def record_run(outputs: list, user: str = "台北現在幾點？", history: list | None = None, sampling=None, family=None, attachments=(), files=None, **settings): """Run the harness to the end on the calling thread; return the run, its events and the fake model.""" - from harness.agent import Run, RunSettings + from harness.harnesses.simple_harness import RunSettings, SimpleHarness from harness.backends.backend import Sampling fake = FakeLLM(list(outputs), family=family) @@ -339,6 +339,6 @@ def record_run(outputs: list, user: str = "台北現在幾點？", history: list fake.drafter_id = fake.info["drafter_id"] = DRAFTER["model_id"] fake.load_drafter() events: list[dict] = [] - run = Run(fake, history or [], user, RunSettings(sampling=sampling or Sampling(), **settings), events.append, attachments, files) + run = SimpleHarness(fake, history or [], user, RunSettings(sampling=sampling or Sampling(), **settings), events.append, attachments, files) run.run() return run, events, fake diff --git a/tests/guide_excerpts.txt b/tests/guide_excerpts.txt index fb79d22..a333281 100644 --- a/tests/guide_excerpts.txt +++ b/tests/guide_excerpts.txt @@ -10,16 +10,19 @@ [[app/page.py|self.thread = threading.Thread]] self.thread = threading.Thread(target=self._work, daemon=True) -[[harness/agent.py|def emit(self, event_type]] +[[harness/harnesses/simple_harness.py|def emit(self, event_type]] def emit(self, event_type: str, **data): -[[harness/agent.py|def stop(self):]] +[[harness/harnesses/simple_harness.py|def stop(self):]] def stop(self): -[[harness/agent.py|def decide(self, call_id]] +[[harness/harnesses/simple_harness.py|def decide(self, call_id]] def decide(self, call_id: str, approved: bool): -[[harness/agent.py|def run(self):]] +[[harness/harnesses/harness.py|class Harness(Protocol):]] + class Harness(Protocol): + +[[harness/harnesses/simple_harness.py|def run(self):]] def run(self): [[harness/backends/backend.py|class Backend(Protocol):]] @@ -350,15 +353,15 @@ app/page.py: def current_llm() -> Backend: if not slot.lock.acquire(blocking=False): app/page.py: def start_run( - def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: + def start_run(who: "Backend | PiAgent", text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: """Ask the question in a new run, with `uploads` (file name, contents) saved in the conversation's folder first.""" slot = generation_slot() if not slot.lock.acquire(blocking=False): st.toast(tr("另一個 agent 正在執行（可能在別的分頁），請稍等。", "Another agent is running (maybe in another tab). Please wait.")) return False try: - attachments = [media.save(chat_folder(), name, data, llm.modalities) for name, data in uploads] - run = LiveRun(llm, st.session_state.history, text, attachments, chat_folder(), current_settings(llm), slot) + attachments = [media.save(chat_folder(), name, data, who.modalities) for name, data in uploads] + run = LiveRun(harness_for(who, text, attachments), text, attachments, slot) except Exception: slot.lock.release() raise @@ -379,17 +382,27 @@ app/page.py: def current_settings( lang=s.lang, ) +app/page.py: def harness_for( + def harness_for(who: "Backend | PiAgent", text: str, attachments: list[media.Attachment]) -> Callable[[Callable[[dict], None]], Harness]: + """What runs the turn: pi, or this project's harness on the model in memory, with the sidebar's settings now.""" + history, folder = st.session_state.history, chat_folder() + if isinstance(who, PiAgent): + settings, session = pi_settings(who), st.session_state.chat_id + return lambda emit: pi_harness.PiHarness(settings, history, text, emit, attachments, folder, session) + settings = current_settings(who) + return lambda emit: SimpleHarness(who, history, text, settings, emit, attachments, folder) + app/page.py: class LiveRun: class LiveRun: - """One user turn: the harness thread and the events it has emitted so far.""" + """One user turn: the harness thread and the events it has emitted so far. `make` makes the harness (simple_harness.SimpleHarness + or pi_harness.PiHarness) given the function it sends its events to.""" - def __init__(self, llm: Backend, history: list[dict], text: str, attachments: list[media.Attachment], files: Path, - settings: RunSettings, slot: Slot): + def __init__(self, make: Callable[[Callable[[dict], None]], Harness], text: str, attachments: list[media.Attachment], slot: Slot): self.text = text self.attachments = attachments self.events: list[dict] = [] self.absorbed = False - self.run = Run(llm, history, text, settings, self.events.append, attachments, files) + self.run = make(self.events.append) self._slot = slot slot.run = self self.thread = threading.Thread(target=self._work, daemon=True) @@ -412,10 +425,10 @@ app/page.py: class LiveRun: == prepare == -[[harness/agent.py|self.messages = [*history,]] +[[harness/harnesses/simple_harness.py|self.messages = [*history,]] self.messages = [*history, {"role": "user", "content": media.user_content(user_text, list(attachments))}] -harness/agent.py: def __init__(self, llm: Backend, history +harness/harnesses/simple_harness.py: def __init__(self, llm: Backend, history def __init__(self, llm: Backend, history: list[dict], user_text: str, settings: RunSettings, emit: Callable[[dict], None], attachments: list[media.Attachment] = (), files: Path | None = None): self.id = uuid.uuid4().hex[:12] @@ -431,10 +444,10 @@ harness/agent.py: def __init__(self, llm: Backend, history self._decisions: dict[str, bool] = {} self._decided = threading.Condition() -[[harness/agent.py|SYSTEM_WITH_TOOLS = ]] +[[harness/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]] SYSTEM_WITH_TOOLS = """You are a helpful assistant running inside an agent loop. -harness/agent.py: def run(self): … stop_reason, final = "max_steps", "" +harness/harnesses/simple_harness.py: def run(self): … stop_reason, final = "max_steps", "" def run(self): s = self.settings family = self.llm.family # how this model writes its turns, thinking and tool calls @@ -442,7 +455,7 @@ harness/agent.py: def run(self): … stop_reason, final = "max_steps", "" schemas = {t.name: t.parameters for t in TOOLS.values()} system = {"role": "system", "content": SYSTEM_WITH_TOOLS if tools else SYSTEM_NO_TOOLS} self.emit( - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, settings=_settings_dict(s), tools=list(self.tools), ) @@ -460,7 +473,7 @@ harness/tools.py: def schema(self) -> dict: == context == -harness/agent.py: for step in range(1, s.max_steps + 1): … roles=_role_counts([system, *self.messages]) +harness/harnesses/simple_harness.py: for step in range(1, s.max_steps + 1): … roles=_role_counts([system, *self.messages]) for step in range(1, s.max_steps + 1): self.emit("step_start", step=step) prompt = self.llm.render([system, *self.messages], tools, s.thinking) @@ -487,10 +500,10 @@ harness/backends/simple_backend.py: def render(self, messages messages, tools=tools or None, tokenize=False, add_generation_prompt=generation_prompt, enable_thinking=thinking ) -[[harness/agent.py|encoded = self.llm.encode(]] +[[harness/harnesses/simple_harness.py|encoded = self.llm.encode(]] encoded = self.llm.encode(prompt, media.parts(self.messages)) -harness/agent.py: def common_prefix( +harness/harnesses/simple_harness.py: def common_prefix( def common_prefix(a: str, b: str) -> int: n = min(len(a), len(b)) i = 0 @@ -640,7 +653,7 @@ harness/backends/vllm_backend.py: def encode(self, prompt == model == -harness/agent.py: done = None … self.emit("model_end" +harness/harnesses/simple_harness.py: done = None … self.emit("model_end" done = None for ev in self.llm.generate(encoded, s.sampling, self._stop.is_set): kind = ev.pop("type") @@ -1116,7 +1129,7 @@ harness/backends/backend.py: TOKEN_FIELDS = ( … TOKEN_DETAILS = ( == parse == -harness/agent.py: prefix = family.thought_prefix if … self.emit("parsed" +harness/harnesses/simple_harness.py: prefix = family.thought_prefix if … self.emit("parsed" prefix = family.thought_prefix if prompt.endswith(family.thought_prefix) else "" parsed = family.parse(prefix + raw, schemas, s.lang) calls = parsed["tool_calls"] if tools else [] @@ -1217,7 +1230,7 @@ harness/families/qwen3_5.py: def convert(value: str, kind == stop == -harness/agent.py: if done["finish"] == "user_stopped": … if not calls: +harness/harnesses/simple_harness.py: if done["finish"] == "user_stopped": … if not calls: if done["finish"] == "user_stopped": stop_reason, final = "user_stopped", parsed["content"] self.messages.append({"role": "assistant", "content": final}) @@ -1228,10 +1241,10 @@ harness/agent.py: if done["finish"] == "user_stopped": … if not calls: self.messages.append({"role": "assistant", "content": final}) break -[[harness/agent.py|stop_reason = "approval_timeout" if self._timed_out]] +[[harness/harnesses/simple_harness.py|stop_reason = "approval_timeout" if self._timed_out]] stop_reason = "approval_timeout" if self._timed_out else "user_stopped" -harness/agent.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" +harness/harnesses/simple_harness.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" if format_errors >= MAX_FORMAT_ERRORS: stop_reason = "format_errors" break @@ -1247,7 +1260,7 @@ harness/agent.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" == record == -harness/agent.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( +harness/harnesses/simple_harness.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( ids = [f"call_{step}_{i}" for i in range(len(calls))] assistant = { "role": "assistant", @@ -1267,7 +1280,7 @@ harness/agent.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( == check == -harness/agent.py: def _handle_call( … args, errors = tool.validate(args, lang) +harness/harnesses/simple_harness.py: def _handle_call( … args, errors = tool.validate(args, lang) def _handle_call(self, step: int, cid: str, call: dict, seen: list) -> str: """Validate, approve and execute one tool call. Returns the observation text for the model.""" lang = self.settings.lang @@ -1299,7 +1312,7 @@ harness/tools.py: def validate(self, args: dict filled.update(args) return filled, errors -harness/agent.py: key = (call["name"], json.dumps … {"error": "; ".join(errors)} +harness/harnesses/simple_harness.py: key = (call["name"], json.dumps … {"error": "; ".join(errors)} key = (call["name"], json.dumps(args, sort_keys=True, ensure_ascii=False)) repeated = key in seen seen.append(key) @@ -1317,7 +1330,7 @@ harness/agent.py: key = (call["name"], json.dumps … {"error": "; ".join(errors if errors: return self._observe(step, cid, call["name"], {"error": "; ".join(errors)}, ok=False, ms=0) -harness/agent.py: if self.settings.approve_tools: … return self._observe(step, cid, call["name"], {"error": error} +harness/harnesses/simple_harness.py: if self.settings.approve_tools: … return self._observe(step, cid, call["name"], {"error": error} if self.settings.approve_tools: self.emit("approval_request", step=step, call_id=cid, name=call["name"], arguments=args) approved = self._wait_for_decision(cid) @@ -1333,10 +1346,10 @@ harness/agent.py: if self.settings.approve_tools: … return self._observe(step, error = pick(lang, "使用者拒絕執行這個工具呼叫", "The user refused to run this tool call") return self._observe(step, cid, call["name"], {"error": error}, ok=False, ms=0) -[[harness/agent.py|APPROVAL_TIMEOUT = ]] +[[harness/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]] APPROVAL_TIMEOUT = 300 # seconds; an unanswered request (say, from a closed tab) must not hold the model forever -harness/agent.py: def _wait_for_decision( +harness/harnesses/simple_harness.py: def _wait_for_decision( def _wait_for_decision(self, cid: str) -> bool: deadline = time.monotonic() + APPROVAL_TIMEOUT with self._decided: @@ -1366,7 +1379,7 @@ app/page.py: for call in render.pending_approvals(data): == execute == -harness/agent.py: self.emit("tool_start" … return self._observe(step, cid, call["name"], result, ok=ok +harness/harnesses/simple_harness.py: self.emit("tool_start" … return self._observe(step, cid, call["name"], result, ok=ok self.emit("tool_start", step=step, call_id=cid, name=call["name"], environment=tool.environment) start = time.perf_counter() try: @@ -1439,7 +1452,7 @@ harness/tools.py: def sandbox_command( == observe == -harness/agent.py: def _observe( +harness/harnesses/simple_harness.py: def _observe( def _observe(self, step: int, cid: str, name: str, result: dict | ToolResult, ok: bool, ms: float) -> str: # The model reads a ToolResult's data; its images (run_python's charts) are only for the page to show. result, images = (result.data, result.images) if isinstance(result, ToolResult) else (result, []) @@ -1450,7 +1463,7 @@ harness/agent.py: def _observe( self.emit("tool_result", step=step, call_id=cid, name=name, ok=ok, result=text, truncated=truncated, ms=round(ms, 1), images=images) return text -harness/agent.py: for cid, call in zip(ids, calls): … if format_errors >= MAX_FORMAT_ERRORS: +harness/harnesses/simple_harness.py: for cid, call in zip(ids, calls): … if format_errors >= MAX_FORMAT_ERRORS: for cid, call in zip(ids, calls): result = self._handle_call(step, cid, call, seen) if call["error"]: @@ -1489,17 +1502,18 @@ app/page.py: @st.fragment(run_every= … viz_panel(llm) app/render.py: def build(events: list[dict]) def build(events: list[dict]) -> dict: - data = {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + data = {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} for ev in events: kind = ev["type"] data["last"], data["t"] = kind, ev["t"] if kind == "run_start": - data["model"], data["family"] = ev["model"], families.get(ev["family"]) - data["settings"] = ev["settings"] + # A harness that does not know how the model writes its turns (pi) sends no family. + data["model"], data["family"] = ev["model"], families.get(ev["family"]) if ev.get("family") else None + data["harness"], data["settings"] = ev.get("harness", "simple"), ev["settings"] elif kind == "step_start": data["steps"].append({ "n": ev["step"], "t0": ev["t"], "t1": ev["t"], "context": None, "prefill": None, - "tokens": [], "done": None, "parsed": None, "calls": {}, + "tokens": [], "text": [], "done": None, "parsed": None, "calls": {}, }) elif kind == "run_end": data["end"] = ev @@ -1510,6 +1524,11 @@ app/render.py: def build(events: list[dict]) step["t1"] = ev["t"] if kind == "token": step["tokens"].append(ev) + elif kind == "delta": # kept as runs of one channel: [{"channel", "text"}] + if step["text"] and step["text"][-1]["channel"] == ev["channel"]: + step["text"][-1] = {**step["text"][-1], "text": step["text"][-1]["text"] + ev["text"]} + else: + step["text"].append({"channel": ev["channel"], "text": ev["text"]}) elif kind in ("context", "prefill", "parsed"): step[kind] = ev elif kind == "model_end": @@ -1537,10 +1556,12 @@ app/render.py: def build(events: list[dict]) app/render.py: state["active"], state["edge"] = { state["active"], state["edge"] = { - "step_start": ("context", "env-context" if step and step["n"] > 1 else None), + # pi sends the request as its turn starts, and says nothing more until the model's output comes back. + "step_start": ("context", "context-model" if data["harness"] == "pi" else "env-context" if step and step["n"] > 1 else None), "context": ("context", "context-model"), "prefill": ("model", "context-model"), "token": ("model", None), + "delta": ("model", None), "model_end": ("harness", "model-harness"), "parsed": ("harness", "model-harness"), "tool_call": ("harness", None), @@ -1562,7 +1583,7 @@ app/page.py: def absorb_finished( st.session_state.history = end["messages"] run.absorbed = True -harness/agent.py: def stop(self): … def decide(self, call_id +harness/harnesses/simple_harness.py: def stop(self): … def decide(self, call_id def stop(self): self._stop.set() with self._decided: @@ -1573,15 +1594,62 @@ harness/agent.py: def stop(self): … def decide(self, call_id self._decisions[call_id] = approved self._decided.notify_all() +== pi == + +harness/harnesses/pi_harness.py: self._pi = Process(self._args() … self._loop() + self._pi = Process(self._args(), self.folder, self._inbox, { + "AGENT_LOOP_REFUSAL": pick(s.lang, "使用者拒絕執行這個工具呼叫", "The user refused to run this tool call"), + }) + # Each waits for its answer: pi handles commands concurrently, and a prompt must not go to the model it starts on. + self._pi.request({"type": "set_model", "provider": s.provider, "modelId": s.model}) + self._pi.request({"type": "set_thinking_level", "level": s.thinking}) + # A failed request is retried by pi as a new turn, which would be a step of its own; the page shows the error instead. + self._pi.request({"type": "set_auto_retry", "enabled": False}) + message, images = self._message() + if not self._stopped_meanwhile(): + self._pi.request({"type": "prompt", "message": message, **({"images": images} if images else {})}) + self._loop() + +harness/harnesses/pi_harness.py: def _on(self, rec: dict) -> bool: + def _on(self, rec: dict) -> bool: + """Turn one of pi's records into the page's events; True once pi has settled.""" + kind = rec.get("type") + if kind == "turn_start": + self._turn_start() + elif kind == "message_start" and self._open and rec["message"].get("role") == "assistant": + self._stream_t, self._wrote = time.perf_counter(), False + elif kind == "message_update" and self._open: + self._update(rec["assistantMessageEvent"]) + elif kind == "message_end" and self._open and rec["message"].get("role") == "assistant": + self._message_end(rec["message"]) + elif kind == "tool_execution_start" and self._open and not rec.get("parentToolCallId"): + self._tool_start(rec["toolCallId"], rec["toolName"], rec.get("args") or {}) + elif kind == "tool_execution_end" and rec.get("toolCallId") in self._calls and not rec.get("parentToolCallId"): + self._tool_end(rec["toolCallId"], rec.get("result") or {}, rec.get("isError", False), rec.get("durationMs")) + elif kind == "extension_ui_request": + self._dialog(rec) + return kind == "agent_settled" + +[[app/render.py|def streamed_html(]] + def streamed_html(step: dict, lang: str) -> str: + +harness/harnesses/pi_approve.ts: pi.on("tool_call" … return undefined; + pi.on("tool_call", async (event, ctx) => { + const call = JSON.stringify({ toolCallId: event.toolCallId, toolName: event.toolName, input: event.input }); + if (!(await ctx.ui.confirm(TITLE, call))) { + return { block: true, reason: process.env.AGENT_LOOP_REFUSAL || "The user refused to run this tool call" }; + } + return undefined; + == events == -[[harness/agent.py|"run_start", run_id=]] - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, +[[harness/harnesses/simple_harness.py|"run_start", run_id=]] + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, -[[harness/agent.py|self.emit("step_start"]] +[[harness/harnesses/simple_harness.py|self.emit("step_start"]] self.emit("step_start", step=step) -[[harness/agent.py|"context",]] +[[harness/harnesses/simple_harness.py|"context",]] "context", [[harness/backends/simple_backend.py|yield {"type": "prefill"]] @@ -1590,29 +1658,32 @@ harness/agent.py: def stop(self): … def decide(self, call_id [[harness/backends/simple_backend.py|"type": "token",]] "type": "token", -[[harness/agent.py|self.emit("model_end"]] +[[harness/harnesses/pi_harness.py|self.emit("delta"]] + self.emit("delta", step=self._step, channel=channel, text=text) + +[[harness/harnesses/simple_harness.py|self.emit("model_end"]] self.emit("model_end", step=step, raw=raw, **done) -[[harness/agent.py|self.emit("parsed"]] +[[harness/harnesses/simple_harness.py|self.emit("parsed"]] self.emit("parsed", step=step, thinking=parsed["thinking"], content=parsed["content"], tool_calls=calls) -[[harness/agent.py|"tool_call",]] +[[harness/harnesses/simple_harness.py|"tool_call",]] "tool_call", -[[harness/agent.py|self.emit("approval_request"]] +[[harness/harnesses/simple_harness.py|self.emit("approval_request"]] self.emit("approval_request", step=step, call_id=cid, name=call["name"], arguments=args) -[[harness/agent.py|self.emit("approval",]] +[[harness/harnesses/simple_harness.py|self.emit("approval",]] self.emit("approval", step=step, call_id=cid, approved=approved, timed_out=self._timed_out, stopped=stopped) -[[harness/agent.py|self.emit("tool_start"]] +[[harness/harnesses/simple_harness.py|self.emit("tool_start"]] self.emit("tool_start", step=step, call_id=cid, name=call["name"], environment=tool.environment) -[[harness/agent.py|self.emit("tool_result"]] +[[harness/harnesses/simple_harness.py|self.emit("tool_result"]] self.emit("tool_result", step=step, call_id=cid, name=name, ok=ok, result=text, truncated=truncated, ms=round(ms, 1), images=images) -[[harness/agent.py|self.emit("error"]] +[[harness/harnesses/simple_harness.py|self.emit("error"]] self.emit("error", message=f"{type(e).__name__}: {e}", **texts) -[[harness/agent.py|self.emit("run_end"]] +[[harness/harnesses/simple_harness.py|self.emit("run_end"]] self.emit("run_end", stop_reason=stop_reason, final=final, messages=self.messages) diff --git a/tests/test_app.py b/tests/test_app.py index 14ff64f..ae5f643 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -21,11 +21,13 @@ from streamlit.testing.v1 import AppTest from app import page, render +from fake_pi import FakePi from fakes import MEDIA_TOKENS, FakeLLM, tool_call -from harness import agent, tools +from harness import tools from harness.backends import vllm_backend, vllm_docker from harness.backends.backend import Sampling from harness.families import GEMMA4, QWEN3_5 +from harness.harnesses import pi_harness, simple_harness APP = Path(__file__).resolve().parent.parent / "streamlit_app.py" ANSWER = "答案是 42。" @@ -925,7 +927,7 @@ def test_another_tab_can_stop_a_leftover_run(at, new_tab, outputs): def test_unanswered_approval_frees_the_model(at, new_tab, outputs, monkeypatch): - monkeypatch.setattr(agent, "APPROVAL_TIMEOUT", 0.2) + monkeypatch.setattr(simple_harness, "APPROVAL_TIMEOUT", 0.2) start_waiting_for_approval(at, outputs) settle(at) assert any(c.endswith("停止：等待確認逾時") for c in captions(at)) @@ -1201,3 +1203,150 @@ def test_a_new_chat_removes_the_conversations_files(at, outputs): assert folder.exists() click(at, "新對話") assert not folder.exists() and at.session_state["runs"] == [] + + +# ---------- pi ---------- + +TIME = {"tool": "bash", "args": {"command": "date"}, "result": "Mon Oct 26 14:00:00 CST 2026\n"} + + +@pytest.fixture +def fake_pi(monkeypatch, tmp_path): + """tests/fake_pi.py in pi's place, with one model; `say` scripts what its model says.""" + return FakePi(monkeypatch, tmp_path) + + +def harness(at): + return next(b for b in at.button_group if b.key == "harness") + + +def pick_pi(at): + harness(at).set_value("pi").run() + assert not at.exception, at.exception + return at + + +def test_the_built_in_harness_is_picked_at_first_and_pi_can_be(at): + assert harness(at).value == "simple" and harness(at).options == ["內建 harness", "pi"] + + +def test_with_pi_the_sidebar_has_its_models_and_settings_and_no_decoding(at, fake_pi, models): + pick_pi(at) + assert [s.value for s in at.selectbox if s.key == "pi_model"] == ["fake/fake-model"] + assert {t.label for t in at.toggle} == {"執行工具前先問我"} + assert {s.label for s in at.slider} == {"步數上限"} + assert next(b for b in at.button_group if b.key == "pi_tools").value == list(pi_harness.DEFAULT_TOOLS) + assert any("pi 1.4.0 · `fake/fake-model`" in c for c in captions(at)) + assert 'class="intro"' in page_html(at) and "pi 的 agent 迴圈" in page_html(at) + assert len(models) == 1 # the model loaded before is not loaded again, nor another + + +def test_a_question_to_pi_draws_its_loop(at, fake_pi): + fake_pi.say({"say": [{"thinking": "Use date."}, TIME]}, {"say": [{"text": "下午兩點。"}]}) + pick_pi(at) + ask(at, "現在幾點？") + assert "下午兩點。" in [m.value for m in at.chat_message[1].markdown] + html = page_html(at) + assert "Step 1 · bash" in html and "STEP 2" in html + assert "API 傳回的已經是 JSON" in html and "這台電腦的 shell" in html and "Mon Oct 26" in html + assert "讀了 1,200 tokens，其中 200 來自 cache" in html + assert "停止：模型沒有再呼叫工具" in html and "看這一步的完整脈絡 →" not in [b.label for b in at.button] + assert Path(fake_pi.started()["cwd"]).name == at.session_state["chat_id"] + + +def test_pi_gets_the_sidebars_settings(at, fake_pi): + fake_pi.say({"say": [{"text": "好。"}]}) + pick_pi(at) + next(b for b in at.button_group if b.key == "pi_tools").set_value(["read", "ls"]).run() + at.slider[0].set_value(3).run() + ask(at, "嗨") + args = fake_pi.started()["args"] + assert args[args.index("--tools") + 1] == "read,ls" + assert at.session_state["runs"][0].events[0]["settings"]["max_steps"] == 3 + + +def test_pi_asks_before_running_a_tool_through_the_page(at, fake_pi): + fake_pi.say({"say": [TIME]}, {"say": [{"text": "下午兩點。"}]}) + pick_pi(at) + toggle(at, "執行工具前先問我").set_value(True) + at.run() + submit(at, "現在幾點？") + wait_for_event(at, "approval_request") + assert "**Agent 想執行這個工具，要允許嗎？** \n`bash({\"command\": \"date\"})`" in [m.value for m in at.markdown] + click(at, "允許") + settle(at) + assert "你允許執行" in page_html(at) and "下午兩點。" in [m.value for m in at.markdown] + + +def test_pi_can_be_stopped(at, fake_pi): + fake_pi.say({"forever": True}) + pick_pi(at) + submit(at, "說個沒完") + wait_for_event(at, "delta") + click(at, "停止這次執行") + settle(at) + assert "停止：使用者中斷" in page_html(at) and "網頁請 pi 中止" in page_html(at) + + +def test_with_pi_the_context_tab_says_pi_does_not_show_its_prompt_and_the_model_tab_shows_pis_model(at, fake_pi): + fake_pi.say({"say": [{"text": "好。"}]}) + pick_pi(at) + ask(at, "嗨") + tab(at).set_value("context").run() + assert any("pi 不說它送給模型 API 的 prompt" in i.value for i in at.info) + tab(at).set_value("model").run() + html = page_html(at) + assert "Fake Model" in html and "fake/fake-model" in html and "32,000 tokens" in html + assert "動作空間：pi 的工具" in html and "沒有沙箱" in html and "特殊 token（" not in html + assert len(at.dataframe) == 1 + + +def test_pi_without_a_model_says_how_to_set_one_up_and_tries_again_when_asked(at, fake_pi): + fake_pi.say(models=[]) + pick_pi(at) + assert any("pi 還沒有可以用的模型" in e.value for e in at.error) + assert harness(at).value == "pi" # and the switch back is still there + fake_pi.say() + click(at, "再試一次") + assert not at.error and [s.value for s in at.selectbox if s.key == "pi_model"] == ["fake/fake-model"] + + +def test_pi_that_is_not_installed_says_how_to_install_it(at, monkeypatch): + monkeypatch.setattr(pi_harness, "PI_COMMAND", ["no-such-pi-anywhere"]) + pick_pi(at) + assert any(pi_harness.INSTALL in e.value for e in at.error) + + +def test_with_pi_docker_is_never_looked_for(at, fake_pi, sandbox): + pick_pi(at) + fake_pi.say({"say": [{"text": "好。"}]}) + ask(at, "嗨") + assert sandbox["looked"] == 0 + assert ".csv" in at.chat_input[0].proto.file_type # pi has bash, so a data file is of use to it + + +def test_back_from_pi_the_built_in_harness_keeps_its_model_and_settings(at, fake_pi, models): + next(s for s in at.slider if s.label == "temperature").set_value(0.3).run() + pick_pi(at) + harness(at).set_value("simple").run() + assert next(s for s in at.slider if s.label == "temperature").value == 0.3 + assert at.session_state["model_id"] == "google/gemma-4-E2B-it" and len(models) == 1 + + +def test_switching_what_runs_the_loop_starts_a_new_conversation(at, fake_pi, outputs): + outputs.append(ANSWER) + ask(at) + pick_pi(at) + assert at.session_state["runs"] == [] and at.session_state["history"] == [] + + +def test_an_english_page_with_pi_has_no_chinese_left(browser, new_tab, fake_pi): + browser["locale"] = "en-US" + at = new_tab() + fake_pi.say({"say": [{"thinking": "Use date."}, TIME]}, {"say": [{"text": "It is 2 pm."}]}) + pick_pi(at) + ask(at, "What time is it?") + for view in ("loop", "context", "model"): + tab(at).set_value(view).run() + text = drawn_text(at) + assert not CJK.search(text), CJK.search(text).string[max(CJK.search(text).start() - 60, 0):CJK.search(text).end() + 20] diff --git a/tests/test_chat_template.py b/tests/test_chat_template.py index 69b0724..a15e7c1 100644 --- a/tests/test_chat_template.py +++ b/tests/test_chat_template.py @@ -13,9 +13,9 @@ from app import render from fakes import FakeLLM, gemma_call, qwen_call from harness import families, media -from harness.agent import SYSTEM_NO_TOOLS, SYSTEM_WITH_TOOLS, Run, RunSettings from harness.backends import simple_backend from harness.backends.simple_backend import MODELS_DIR, SimpleBackend +from harness.harnesses.simple_harness import SYSTEM_NO_TOOLS, SYSTEM_WITH_TOOLS, RunSettings, SimpleHarness from harness.tools import TOOLS from html_check import assert_balanced @@ -78,7 +78,7 @@ def call_text(family, name, args): def after_call(name, args, result='{"ok": true}'): - """The messages the harness has after one tool call, built the way agent.Run builds them.""" + """The messages the harness has after one tool call, built the way simple_harness.SimpleHarness builds them.""" return [ system(), {"role": "user", "content": "問題"}, @@ -239,7 +239,7 @@ def test_step_card_marks_only_the_observation_as_new(model, monkeypatch, before, end = "好。" + next(iter(family.stop_tokens)) events = [] llm = TemplateLLM(model, [scripted(family, before, args, thinking), end]) - Run(llm, [], "ReAct 是誰提出的？", RunSettings(thinking=thinking), events.append).run() + SimpleHarness(llm, [], "ReAct 是誰提出的？", RunSettings(thinking=thinking), events.append).run() [parsed, _] = [e for e in events if e["type"] == "parsed"] assert [c["arguments"] for c in parsed["tool_calls"]] == [args] first, second = (e for e in events if e["type"] == "context") @@ -266,7 +266,7 @@ def test_with_media_the_step_card_still_marks_only_the_observation_as_new(with_p (("receipt.png", "image"), ("ball.mp4", "video"))] events = [] llm = MediaTemplateLLM(with_processor, [scripted(family, "", {"query": "ReAct"}, False), "好。" + next(iter(family.stop_tokens))]) - Run(llm, [], "這是什麼？", RunSettings(), events.append, files).run() + SimpleHarness(llm, [], "這是什麼？", RunSettings(), events.append, files).run() first, second = (e for e in events if e["type"] == "context") assert first["media_tokens"]["image"] > 100 and first["media_tokens"]["video"] > 100 assert first["n_tokens"] == len(with_processor.tokenizer(first["prompt"], add_special_tokens=False).input_ids) diff --git a/tests/test_guide.py b/tests/test_guide.py index 429c668..7a8a04b 100644 --- a/tests/test_guide.py +++ b/tests/test_guide.py @@ -142,10 +142,10 @@ def test_the_snapshot_follows_the_quoted_code_not_its_line_numbers(tmp_path, mon shutil.copy(guide.ROOT / file, tmp_path / file) monkeypatch.setattr(guide, "ROOT", tmp_path) before = quoted_sections() - agent_py = tmp_path / "harness/agent.py" + agent_py = tmp_path / "harness/harnesses/simple_harness.py" agent_py.write_text("# a line that moves every other one down\n" + agent_py.read_text(encoding="utf-8"), encoding="utf-8") assert quoted_sections() == before - code = next(code for code in CODES if code.file == "harness/agent.py") + code = next(code for code in CODES if code.file == "harness/harnesses/simple_harness.py") lines = agent_py.read_text(encoding="utf-8").splitlines() lines[span(code)[0] - 1] += " # edited" agent_py.write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -169,7 +169,7 @@ def test_event_table_points_at_the_line_that_sends_it(kind, file, text, _carries def test_event_table_covers_every_event_the_harness_sends(): events = set() - for file in ("harness/agent.py", "harness/backends/simple_backend.py"): + for file in ("harness/harnesses/simple_harness.py", "harness/backends/simple_backend.py", "harness/harnesses/pi_harness.py"): for node in ast.walk(guide.source(file)[1]): if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "emit" and isinstance(node.args[0], ast.Constant): events.add(node.args[0].value) @@ -330,20 +330,20 @@ def test_the_root_page_sends_a_reader_to_their_language(tmp_path): def test_prose_markup_is_escaped(): - html = guide.prose_html("a `x` **粗** [[harness/agent.py|def run(self):]] {{stop}}", "zh", "0123456789abcdef") + html = guide.prose_html("a `x` **粗** [[harness/harnesses/simple_harness.py|def run(self):]] {{stop}}", "zh", "0123456789abcdef") assert html.startswith("a <b> x<y> 粗 ") - line = guide.find_line("harness/agent.py", "def run(self):") - assert f'>harness/agent.py:{line}harness/harnesses/simple_harness.py:{line} 0 assert right(events[-1]["final"]), events[-1]["final"] @@ -220,7 +220,7 @@ def test_the_model_opens_every_data_format_with_run_python(gemma, tmp_path, ext) attachment = media.Attachment(path.name, "file", str(path), path.stat().st_size) events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=512), max_steps=3, lang="en") - Run(gemma, [], f"Use run_python to open /data/{path.name} and print its first two rows.", settings, events.append, [attachment], tmp_path).run() + SimpleHarness(gemma, [], f"Use run_python to open /data/{path.name} and print its first two rows.", settings, events.append, [attachment], tmp_path).run() results = [e for e in events if e["type"] == "tool_result" and e["name"] == "run_python"] assert any(r["ok"] and "2025" in r["result"] for r in results), results diff --git a/tests/test_render.py b/tests/test_render.py index df9ccf6..689e195 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -5,6 +5,7 @@ import dataclasses import json import re +import tempfile import threading import xml.etree.ElementTree as ET from pathlib import Path @@ -12,11 +13,13 @@ import pytest from app import render +from fake_pi import FakePi from fakes import DRAFTER, INFO, MEDIA_TOKENS, QWEN_INFO, FakeLLM, qwen_tool_call, record_run, tool_call -from harness import agent, families -from harness.agent import Run, RunSettings +from harness import families from harness.backends.backend import TOKEN_DETAILS, Sampling from harness.families import GEMMA4, QWEN3_5 +from harness.harnesses import pi_harness, simple_harness +from harness.harnesses.simple_harness import RunSettings, SimpleHarness from harness.i18n import LANGS from harness.media import Attachment from harness.tools import TOOLS, ToolResult @@ -34,7 +37,7 @@ def approved_run_events(): """A thinking run that asks for approval, gets it, calls a tool and answers.""" fake = FakeLLM(["要查時間" + tool_call("get_current_time"), "好" + ANSWER]) events = [] - run = Run(fake, [], "台北現在幾點？", RunSettings(thinking=True, approve_tools=True), events.append) + run = SimpleHarness(fake, [], "台北現在幾點？", RunSettings(thinking=True, approve_tools=True), events.append) thread = threading.Thread(target=run.run, daemon=True) thread.start() for _ in range(500): @@ -58,6 +61,26 @@ def token(event): LOGPROBS = ("p", "logprob", "top") # what an OpenAI-style API's log probabilities give +BASH = {"tool": "bash", "args": {"command": "date"}, "result": "Mon Oct 26 14:00:00 CST 2026\n"} + + +def pi_events(*turns, approve: bool | None = None, text: str = "現在幾點？", **settings) -> list[dict]: + """A turn run by pi (its stand-in, tests/fake_pi.py), as its events; with `approve`, the user answers the first call so.""" + with pytest.MonkeyPatch.context() as mp, tempfile.TemporaryDirectory() as folder: + FakePi(mp, Path(folder)).say(*turns) + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings("fake", "fake-model", approve_tools=approve is not None, **settings), [], text, events.append, + folder=Path(folder) / "chat", session="c") + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + if approve is not None: + for _ in range(1000): + if request := next((e for e in events if e["type"] == "approval_request"), None): + run.decide(request["call_id"], approve) + break + threading.Event().wait(0.01) + thread.join(20) + return events PHOTO = Attachment("photo.png", "image", "/chat/photo.png", 2048) CLIP = Attachment("clip.mp4", "video", "/chat/clip.mp4", 90_000) SALES = Attachment("sales.csv", "file", str(Path(__file__).resolve().parent.parent / "app/assets/examples/sales.csv"), 942) @@ -82,6 +105,10 @@ def token(event): "qwen_thinking": lambda: record_run( ["要算一下\n\n\n" + qwen_tool_call("calculator", expression="1"), "想好了\n\n\n好。<|im_end|>"], family=QWEN3_5, thinking=True)[1], + "pi": lambda: pi_events({"say": [{"thinking": "用 date。"}, {"text": "我查一下。"}, BASH]}, {"say": [{"text": "下午兩點。"}]}), + "pi_rejected": lambda: pi_events({"say": [BASH]}, {"say": [{"text": "好吧。"}]}, approve=False), + "pi_max_steps": lambda: pi_events(*[{"say": [BASH]}] * 3, max_steps=2), + "pi_error": lambda: pi_events({"say": [], "stop": "error", "error": "Connection error."}), } @@ -100,7 +127,10 @@ def render_everything(data: dict, live: bool, lang: str = "zh") -> list[str]: out.append(render.context_html(step, data["family"], lang=lang)) if data["end"]: out.append(render.stop_html(data, lang=lang)) - if data["settings"]: + if data["settings"] and data["harness"] == "pi": + out.append(render.pi_settings_html(data["settings"], lang=lang)) + out.append(render.pi_tools_html(pi_harness.TOOLS, data["settings"]["tools"], lang=lang)) + elif data["settings"]: out.append(render.knobs_html(data["settings"], INFO["defaults"], lang=lang)) out.append(render.tools_html(TOOLS, data["settings"]["use_tools"], data["family"], lang=lang, offered=data["settings"]["tools"])) return [piece for piece in out if piece] @@ -149,6 +179,10 @@ def test_every_prefix_of_a_run_renders(scenario, lang): "no_details": lambda: as_reported(record_run([tool_call("calculator", expression="6*7"), "It is 42."], user="6*7?", lang="en")[1]), "logprobs": lambda: as_reported( record_run([tool_call("calculator", expression="6*7"), "It is 42."], user="6*7?", lang="en")[1], LOGPROBS), + "pi": lambda: pi_events({"say": [{"thinking": "Use date."}, BASH]}, {"say": [{"text": "It is 2 pm."}]}, text="Time?", lang="en"), + "pi_rejected": lambda: pi_events({"say": [BASH]}, {"say": [{"text": "OK."}]}, approve=False, text="Time?", lang="en"), + "pi_max_steps": lambda: pi_events(*[{"say": [BASH]}] * 3, max_steps=2, text="Time?", lang="en"), + "pi_error": lambda: pi_events({"say": [], "stop": "error", "error": "Connection error."}, text="Time?", lang="en"), } @@ -194,7 +228,7 @@ def test_switching_language_redraws_a_finished_run(): def test_build_of_nothing(): data = render.build([]) - assert data == {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + assert data == {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} def test_build_folds_a_tool_run_into_steps(): @@ -407,9 +441,9 @@ def test_step_card_approval_states(): def test_approval_timeout_is_shown(monkeypatch): - monkeypatch.setattr(agent, "APPROVAL_TIMEOUT", 0.05) + monkeypatch.setattr(simple_harness, "APPROVAL_TIMEOUT", 0.05) events = [] - Run(FakeLLM([tool_call("get_current_time"), ANSWER]), [], "幾點？", RunSettings(approve_tools=True), events.append).run() + SimpleHarness(FakeLLM([tool_call("get_current_time"), ANSWER]), [], "幾點？", RunSettings(approve_tools=True), events.append).run() data = render.build(events) assert list(data["steps"][0]["calls"].values())[0]["approval"] == "timeout" assert "等待確認逾時，harness 當作拒絕並停止迴圈" in render.step_html(data["steps"][0], data, live=False, lang="zh") @@ -424,7 +458,7 @@ def test_approval_timeout_is_shown(monkeypatch): def test_stopping_is_not_shown_as_rejecting(answer, label, not_label): fake = FakeLLM([tool_call("get_current_time"), ANSWER]) events = [] - run = Run(fake, [], "幾點？", RunSettings(approve_tools=True), events.append) + run = SimpleHarness(fake, [], "幾點？", RunSettings(approve_tools=True), events.append) thread = threading.Thread(target=run.run, daemon=True) thread.start() for _ in range(500): @@ -972,3 +1006,102 @@ def test_the_model_tab_lists_what_the_model_reads(): def test_a_tool_left_out_of_the_turn_is_marked(): html = render.tools_html(TOOLS, True, GEMMA4, "en", offered=["calculator"]) assert html.count("not offered to the model in this turn") == len(TOOLS) - 1 + + +# ---------- a run pi ran ---------- + +def test_a_pi_step_shows_the_text_it_streamed_by_channel_and_no_token_numbers(): + data = render.build(SCENARIOS["pi"]()) + assert data["harness"] == "pi" and data["family"] is None + step = data["steps"][0] + assert step["text"] == [{"channel": "thought", "text": "用 date。"}, {"channel": "content", "text": "我查一下。"}, + {"channel": "tool_call", "text": '\nbash {"command": "date"}'}] + card = render.step_html(step, data, lang="en", live=False) + assert '用 date。' in card and 'class="run ch-tool_call"' in card + assert "wrote 20 tokens" in card and "read 1,200 tokens, 200 from the cache" in card and "at pi's prices" not in card + assert "about $0.0021 at pi's prices" in card + assert "the API returned it as JSON" in card and "shell on this computer · returned" in card + assert "Context " not in card and 'class="tok ' not in card and "token id" not in card # no context and no tokens + + +def test_what_pi_streams_is_escaped(): + data = render.build(pi_events({"say": [{"text": ""}]})) + card = render.step_html(data["steps"][0], data, lang="en", live=False) + assert "