diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..7f7ddc5 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,55 @@ +# Checks every pull request and every push to main: ruff's lint (the order of imports with it), mypy's types, and the tests that +# load no weights and touch no network (the ones that need the models, Docker's images, a vLLM server or pi skip). +# mypy and the tests import the whole app, so the job first installs every package uv.lock pins, torch among them. +# A second job builds How it works with pygments alone, as pages.yml does after the merge. +name: Checks + +# A push to a pull request's branch is checked once, as the pull request: on a push to any branch as well, both ran, +# neither cancelled the other (their concurrency groups differ by the event), and torch was installed twice. +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: checks-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.0 + with: + version-file: pyproject.toml # its required-version: no uv older than the fix for the revoked emSign certificates + - id: sync + run: uv sync --locked + # Each runs even when the one before failed, so that one push reports all three. + - run: uv run ruff check + - if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run mypy + - if: ${{ !cancelled() && steps.sync.outcome == 'success' }} + run: uv run pytest + + # What pages.yml runs before it publishes How it works, word for word: keep the two the same. The job above has + # every package installed, so it would not notice app/guide.py, app/render.py, agent/i18n.py, agent/model_families/ + # or a package's __init__.py importing torch, transformers, streamlit or httpx; only the Pages build on main would. + # --isolated keeps uv from reusing a .venv that already has pygments, and with it the whole app: there is none in a + # fresh checkout, but there is in a checkout where these are run by hand. + guide: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.0 + with: + version-file: pyproject.toml # its required-version: no uv older than the fix for the revoked emSign certificates + - run: uv run --isolated --no-project --with pygments --with pytest pytest tests/test_guide.py + # Runs even when the tests failed, so that one push reports both. + - if: ${{ !cancelled() }} + run: uv run --isolated --no-project --with pygments python -m app.guide _site diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index a6a786d..2dd0e4d 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -32,8 +32,9 @@ jobs: # The page needs only pygments, not torch and the rest of the app, and its tests only pytest # besides. They fail when an edit moved or removed a line the page points at, or changed code it # quotes since its text was last reread (tests/guide_excerpts.txt); the page already up then stays. - - run: uv run --no-project --with pygments --with pytest pytest tests/test_guide.py - - run: uv run --no-project --with pygments python -m app.guide _site + # checks.yml's guide job runs these two on every pull request, so keep the two the same. + - run: uv run --isolated --no-project --with pygments --with pytest pytest tests/test_guide.py + - run: uv run --isolated --no-project --with pygments python -m app.guide _site - uses: actions/configure-pages@v6 - uses: actions/upload-pages-artifact@v5 with: diff --git a/.gitignore b/.gitignore index 04d6f43..931339e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ .venv/ __pycache__/ .pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.impeccable/ _site/ /models/ .DS_Store diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/AGENTS.md b/AGENTS.md index f86494a..2857b94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,9 @@ Qwen3.5 0.8B, through Hugging Face `transformers`, or on vLLM: in Docker, starte server at `VLLM_URL`) and draws every step of its agent loop: context, model, harness, environment, stop, down to each token's probability. A question can carry images, sound and video for the model, and, when the sidebar lets the model write code, data files for a `run_python` -tool that runs in a Docker sandbox. It is a teaching project for the +tool that runs in a Docker sandbox. The sidebar can also hand the loop to pi, a coding agent, which asks +either the page's own model (served to it over HTTP on 127.0.0.1, every token still drawn) or a model it has credentials +for (drawn from what pi reports, without tokens). It is a teaching project for the Agent Atlas course. The page and its 運作說明 (How it works) guide are in Traditional Chinese and English. README.md (English) and README.zh-TW.md (Traditional Chinese) are the full account of what the page shows, the architecture, the tests and the known limits; read the parts you are about to change. @@ -15,23 +17,29 @@ page shows, the architecture, the tests and the known limits; read the parts you ```bash uv sync # Python 3.12 (.python-version), packages pinned by uv.lock; uv >= 0.13.4 (required-version in pyproject.toml) uv run pytest # ~40 s; loads no weights, touches no network +uv run ruff check # lint, the order of imports included; --fix sorts them +uv run mypy # types: agent/ and app/ inside every function, tests/ where annotated RUN_MODEL_TESTS=1 uv run pytest tests/test_real_model.py # real Gemma (with its MTP drafter) and Qwen, ~4 min; -k Qwen for one -docker build -t agent-loop-sandbox harness/sandbox # optional: run_python's sandbox; its tests skip without it; bump SANDBOX_VERSION with the Dockerfile +docker build -t agent-loop-sandbox agent/sandbox # optional: run_python's sandbox; its tests skip without it; bump SANDBOX_VERSION with the Dockerfile uv run streamlit run streamlit_app.py # http://localhost:8501; MODEL_ID=Qwen/Qwen3.5-0.8B starts on Qwen VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py # the model on a vLLM server: vllm serve --revision [--enable-scale-out [--return-sampling-mask --logprobs-mode processed_logprobs]] VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # vllm_backend.py against that real server docker compose -f docker-compose/compose.yaml up -d # the vLLM server the sidebar's vLLM (Docker) starts, by hand (+ -f docker-compose/compose.gpu.yaml on NVIDIA GPUs) RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # starts that server with Qwen3.5 and asks it through VLLMBackend, ~2 min +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # the real pi (on the PATH, or PI_COMMAND; pi 1.4 needs Node 22.19+: `nvm install` installs the newest 22 .nvmrc names and switches to it) against a scripted model on 127.0.0.1 uv run python -m app.guide _site # the guide: _site/zh/, _site/en/ and a redirecting _site/index.html UPDATE_GUIDE_SNAPSHOT=1 uv run pytest tests/test_guide.py # after rereading the guide sections a code change touched +uv run --isolated --no-project --with pygments --with pytest pytest tests/test_guide.py # the guide's tests as the Pages build runs them: pygments and pytest alone, not .venv ``` -No linter or formatter is configured. Match the surrounding code by hand and don't reformat lines you -are not changing. +Ruff lints and mypy checks types (both set up in pyproject.toml), and .github/workflows/checks.yml runs them and +`uv run pytest` on every pull request and every push to main, and in a job of its own tests and builds the guide with +pygments alone, as the Pages workflow does. No formatter is configured: match the surrounding code by hand and don't +reformat lines you are not changing. Fix what the two report rather than silencing it; an ignore needs a reason beside it. ## Layout -Two packages: `harness/` runs a turn, `app/` is what a person sees. `app/` imports `harness/`, never the +Two packages: `agent/` runs a turn, `app/` is what a person sees. `app/` imports `agent/`, never the other way round. ``` @@ -41,14 +49,19 @@ app/ the Streamlit app render.py pure functions from the event list to HTML: loop diagram, step cards, token heat and popovers, context guide.py the 運作說明 page: content in both languages, code references by text, static HTML for GitHub Pages assets/ style.css (palette and app), dark.css, guide.css, guide.js (the guide's one inlined script), logo.svg, examples/ (files the examples attach, also offered above the input box) -harness/ the agent, the model and what runs it; no Streamlit - agent.py the harness (Run): prompt → generate → parse → validate → (ask) → run tool → back into context → stop? - backends/ - backend.py Backend: what agent.py and the page ask of whatever runs the model (SimpleBackend, VLLMBackend, tests' FakeLLM); Sampling, Encoded, the token event's fields +agent/ the agent: the harnesses that run its loop, the model, what runs the model, the tools; no Streamlit + harnesses/ + harness.py Harness: what the page asks of whatever runs the loop (run, stop, decide), and the events every harness sends + simple_harness.py this project's harness (SimpleHarness, RunSettings): prompt → generate → parse → validate → (ask) → run tool → back into context → stop? + pi_harness.py PiHarness: pi (PI_COMMAND) in RPC mode for each question, its JSON records turned into the page's events; probe() lists its models + pi_extension.ts the extension every pi the page starts loads: reports each request (a notification), registers the page's model, asks before a tool call (a confirm dialog pi_harness.py answers on stdin) + model_endpoint.py the page's model as an OpenAI-compatible chat completions endpoint on 127.0.0.1, behind a token, for pi; its events go into pi's step + model_backends/ + backend.py Backend: what SimpleHarness and the page ask of whatever runs the model (SimpleBackend, VLLMBackend, tests' FakeLLM); Sampling, Encoded, the token event's fields simple_backend.py SimpleBackend: the hand-written decoding loop (KV cache, temperature / top-k / top-p, MTP draft and verify) vllm_backend.py VLLMBackend (VLLM_URL): the tokenizer and chat template here, the token ids to a vLLM server (/v1/completions, or /inference/v1/generate with --enable-scale-out: media, sampling masks), what it says about each token back vllm_docker.py the sidebar's vLLM (Docker): downloads the model, `docker compose up -d` with it, waits until it answers, `docker compose down`; DockerError in both languages - families/ everything model-specific: one Family per model family; MODELS and DRAFTERS in __init__.py + model_families/ everything model-specific: one Family per model family; MODELS and DRAFTERS in __init__.py tools.py tools: the schema the model reads plus the Python function the harness runs; run_python's docker run media.py attachments: saved per conversation, written into the user message, decoded for the processor (PyAV) i18n.py the two languages: LANGS, pick(lang, zh, en), from_locale(); no dependencies @@ -58,32 +71,37 @@ tests/ pytest; fakes.py is a scripted stand-in for SimpleBacke models/ downloaded models (simple_backend.MODELS_DIR), which git ignores; AGENT_LOOP_MODELS moves them, HF_HUB_CACHE does not ``` -Imports are absolute from the project's folder (`from harness import families`, `from harness.backends.backend +Imports are absolute from the project's folder (`from agent import model_families`, `from agent.model_backends.backend import Backend`, `from app import render`); pytest puts that folder on the path (`pythonpath` in pyproject.toml). ## Rules that keep it working -- **The harness never knows the app.** Nothing in `harness/` imports `app/`, Streamlit or anything that draws; it - reports what happens only as events. `harness/__init__.py`, `harness/families/`, `harness/i18n.py` and - `app/__init__.py` import nothing heavy, because the guide's build loads them with pygments alone. +- **The agent never knows the app.** Nothing in `agent/` imports `app/`, Streamlit or anything that draws; it + reports what happens only as events (tests/test_layout.py checks the imports). `agent/__init__.py`, + `agent/model_families/`, `agent/i18n.py` and `app/__init__.py` import nothing heavy, because the guide's build + loads them with pygments alone. - **app/page.py is imported once, not run as a script.** streamlit_app.py imports it and calls `main()` on every rerun, so everything a rerun does is inside functions, and what the module reads from the environment (`VLLM_URL`, `MODEL_ID`, `AGENT_LOOP_UPLOADS`) is read once per process. Tests change those on the module (`page.VLLM_URL`, `page.UPLOADS`, `page.SimpleBackend`), not in the environment. -- **Events are the interface.** `agent.Run` runs in a background thread and appends one event per +- **Events are the interface.** The page runs a turn only through a `Harness` (harnesses/harness.py): `SimpleHarness`, or + `PiHarness` with pi picked under What runs the loop. It runs in a background thread and appends one event per stage; the order is in the READMEs' Architecture (架構) section. `app/render.py` is a pure function of that list and the page only draws it, which is why stop and approve can rerun the script without interrupting the loop. A new or changed event needs render.py, the `EVENTS` table in app/guide.py (tests check that it lists - every event the harness sends and points at the line that sends it) and the event list in both READMEs. + every event either harness sends and points at the line that sends it) and the event list in both READMEs. + Most events are optional (`harness.SOMETIMES`): pi sends `request` (what it sent the model), and on a provider's model + no `context`, `prefill`, `token` or family but `delta`; on the page's model, model_endpoint.py sends those four into + pi's step. render.py draws a step from what it has, never filling in what the harness did not see. The harness and the page reach the model only through `backend.Backend`; a token event's probabilities, rank, candidates and MTP draft (`backend.TOKEN_DETAILS`) are optional, and render.py leaves out what is missing. -- **Nothing model-specific outside `harness/families/`.** agent.py, simple_backend.py and render.py have no "if Gemma". +- **Nothing model-specific outside `agent/model_families/`.** simple_harness.py, simple_backend.py and render.py have no "if Gemma". Parsing output, the tokens that switch to thinking / tool call / reply, stop tokens, special-token explanations, notes on the files of its checkpoints and recommended sampling all live on the `Family`. Another checkpoint of a supported family is one line in `MODELS` and one in `REVISIONS`, the commit it downloads at (and `DRAFTERS` and its drafter's commit if it has an MTP drafter, and `VLLM_SPECULATIVE` for how vLLM drafts for it on a GPU); a new family is a module like - `harness/families/qwen3_5.py` added to `FAMILIES`. Moving a model to a newer commit means rerunning + `agent/model_families/qwen3_5.py` added to `FAMILIES`. Moving a model to a newer commit means rerunning tests/test_chat_template.py and tests/test_real_model.py and rereading the guide's model folder section, whose numbers come from those files. - **The decoding loop stays exact.** Greedy output equals `model.generate()`. MTP only changes how many @@ -94,33 +112,38 @@ import Backend`, `from app import render`); pytest puts that folder on the path something else: a processed --logprobs-mode is the sampling distribution, never the model's p. - **Escape everything the model writes** before it reaches `st.html`. tests/test_render.py checks it, and also that every prefix of a run's events renders, because the page redraws mid-run. -- **app/guide.py finds code by text, not line number.** `Code("harness/agent.py", "def run(self):")` and inline - `[[harness/agent.py|def run(self):]]` are looked up when the page is built, by their path from the project's folder, +- **app/guide.py finds code by text, not line number.** `Code("agent/harnesses/simple_harness.py", "def run(self):")` and inline + `[[agent/harnesses/simple_harness.py|def run(self):]]` are looked up when the page is built, by their path from the project's folder, and each must match exactly one line. Before renaming, moving or rewriting a line, grep app/guide.py for it; if a reference breaks, tests/test_guide.py fails. Fix the reference, and when the behavior changed, the prose around it too. A file that is not Python (docker-compose/compose.yaml) is quoted line by line, through the line with `end`. Where a part of a section differs by what - runs the model, or by the model, it goes in `Tabs`, one tab per backend or per model (`TAB_KINDS`), not in one paragraph about both. + runs the model, by the model, or by the harness, it goes in `Tabs`, one tab per backend, per model or per harness (`TAB_KINDS`: + `pi-page` and `pi-provider` are kinds of `pi`, as `vllm-docker` is of `vllm`), not in one paragraph about both. Tabs do not nest, so + what the built-in harness and pi on this page's model share (the chat template, the parsers, the backends) stays outside the harness tabs. - **The guide's text is reread whenever the code it quotes changes.** tests/guide_excerpts.txt holds every excerpt and the line behind every chip, without line numbers. When an edit changes any of them, tests/test_guide.py fails and names the sections. Reread each one in both languages, fix what is no longer true, then rewrite the file with `UPDATE_GUIDE_SNAPSHOT=1 uv run pytest tests/test_guide.py` and commit it with the change. Never regenerate it without rereading. - **app/guide.py builds with pygments alone.** The Pages workflow runs - `uv run --no-project --with pygments python -m app.guide _site`, and guide.py imports app.render and - harness.i18n, and render imports harness.families. Keep those (and the packages' `__init__.py`) free of torch, + `uv run --isolated --no-project --with pygments python -m app.guide _site`, and guide.py imports app.render and + agent.i18n, and render imports agent.model_families. Keep those (and the packages' `__init__.py`) free of torch, transformers, streamlit and httpx. Before building, it runs tests/test_guide.py with pygments and pytest alone, - so that file imports only app.guide, harness.i18n and tests/html_check.py (not test_render.py, which imports the - harness and with it PyAV, NumPy, Pillow and httpx). + so that file imports only app.guide, agent.i18n and tests/html_check.py (not test_render.py, which imports the + agent's harnesses and with them PyAV, NumPy, Pillow and httpx). `uv run pytest` has every package installed and + cannot see a heavy import there, so checks.yml's `guide` job runs those two commands on every pull request; + keep it the same as pages.yml. - **Every word a person sees exists in both languages, side by side.** In code it is `pick(lang, "中文", "English")`; in data (special-token notes, stop reasons, tool environments, guide paragraphs, the guide's event table) it is a `{"zh": ..., "en": ...}` dict. Never add a string in one language only. render.py functions take `lang` explicitly and stay pure: the events carry no language, so a switch redraws finished turns too. tests/test_render.py, test_guide.py and - test_app.py fail when Chinese shows up on an English page, and test_guide.py and test_families.py + test_app.py fail when Chinese shows up on an English page, and test_guide.py and test_model_families.py when the two languages drift apart (different refs, links or tokens). - **The model reads harness and tool errors in the run's language.** `RunSettings.lang` is the page's language when the turn started; `Tool.validate`, `family.parse` and the harness's own messages word - errors in it, and `ToolError` / `ParseError` carry both texts. The system prompts in agent.py are the + errors in it, and `ToolError` / `ParseError` carry both texts (with pi, the refusal pi_extension.ts gives the model, through + `AGENT_LOOP_REFUSAL`). The system prompts in simple_harness.py are the same for both languages, so the model's behavior does not depend on the page's language. - **The language switch.** The app starts in the language in the `agent_loop_lang` cookie, else the browser's (`st.context.locale` through `i18n.from_locale`); picking one writes the cookie with an @@ -153,12 +176,26 @@ import Backend`, `from app import render`); pytest puts that folder on the path image build the new one, and keep compose.gpu.yaml's command the same as compose.yaml's but `--skip-mm-profiling`, plus its two flags on GPU memory and two on speculative decoding (tests/test_vllm_docker.py checks it). The app stops the container it started when it quits (`_stop_at_exit`), so a test that fakes a start leaves `vllm_docker._chosen` empty. +- **pi only when asked for, and as itself.** The page starts pi only once What runs the loop is pi, and then never + `docker`. It starts pi without the user's extensions, skills, prompt templates and context files (`pi_harness.FLAGS`), + so the page draws pi's own loop; the credentials are pi's. Each command waits for pi's response before the next (pi + handles them concurrently: a prompt sent beside `set_model` goes to the old model), auto-retry stays off (a retry is + a turn of its own), and Max steps is kept by aborting pi at the turn past it. pi's tools run unsandboxed in the + conversation's folder; the page says so wherever they are offered. model_endpoint.py listens on 127.0.0.1 alone, asks + for its token (pi gets it as `$AGENT_LOOP_TOKEN`), answers only while a turn is served, one request at a time, and + uses the sidebar's decoding, not the request's. A request waits for the step pi opened for it (`_serving_step`), since + pi's records and its HTTP request reach the page by different ways. Read pi's docs (docs/rpc.md, json.md, + rpc-extension-ui.md, custom-provider.md in its package) and its openai-completions provider before changing what is + sent or read. ## Tests - Most tests use `FakeLLM` from tests/fakes.py: same interface as `SimpleBackend`, prompts in a simplified form of each family's chat format, scripted replies streamed token by token. Cover new harness or UI - behavior with it; tests/test_app.py drives the whole page through Streamlit's `AppTest`. + behavior with it; tests/test_app.py drives the whole page through Streamlit's `AppTest`. In pi's place they start + tests/fake_pi.py (`FakePi`), which speaks pi 1.4's RPC protocol from a script and, for the page's model, asks + model_endpoint.py over HTTP as pi does; tests/test_model_endpoint.py asks the endpoint directly. tests/test_pi_harness.py + runs the real pi only with `RUN_PI_TESTS=1`, against a scripted OpenAI-compatible server on 127.0.0.1 and the endpoint. - tests/test_chat_template.py checks each family against the model's real tokenizer and chat template and skips models not yet downloaded into models/. tests/test_real_model.py loads the real weights and runs only with `RUN_MODEL_TESTS=1`. After changing parsing, prompts, stop tokens or the @@ -171,7 +208,7 @@ import Backend`, `from app import render`); pytest puts that folder on the path - Wikipedia is faked over HTTP; the default suite never touches the network. Tests that need Docker run only when `tools.sandbox_problem()` finds it, and skip otherwise. tests/test_vllm_docker.py fakes Docker at the level of `subprocess`; only `RUN_VLLM_DOCKER_TESTS=1` starts a real vLLM container. -- Every format `harness/media.py` accepts has a sample in tests/samples.py (test_media.py fails otherwise), a +- Every format `agent/media.py` accepts has a sample in tests/samples.py (test_media.py fails otherwise), a decoding test, and a real-model test that asks both models about it. A new format needs all three. ## Language and style diff --git a/README.md b/README.md index 5047401..0b454cb 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,12 @@ English | [繁體中文](README.zh-TW.md) -![The top bar links to the How it works page, the switch at the top right changes the language, and the sidebar picks the model and what runs it; on the left is the chat, with the sample files to ask about above the input box, on the right the loop diagram and the tokens of step 1. The pointer rests on the special token <|tool_call>, whose popover says what it is for and shows its probability, its rank and the top 5 candidates.](docs/screenshot-en.png) +![The top bar links to the How it works page, the switch at the top right changes the language, and the sidebar picks what runs the loop (the built-in harness or pi), the model and what runs it; on the left is the chat, with the sample files to ask about above the input box, on the right the loop diagram and the tokens of step 1. The pointer rests on the special token <|tool_call>, whose popover says what it is for and shows its probability, its rank and the top 5 candidates.](docs/screenshot-en.png) This project is for learning: it shows how an agent loop works underneath, in a way more intuitive and visual than reading about it. Behind an agent's chat box runs a loop: step after step, the model proposes an action and the harness carries it out, until it stops. A chat app shows only the question and the answer; this page draws every step as it happens, down to each token the model writes, so the whole mechanism is laid out in front of you. Each part of the page matches a section of the Unit 01 reading of [Agent Atlas](https://cyyeh.github.io/agent-atlas-learning/) (in Traditional Chinese), the course this project belongs to, and [Experiments to try](#experiments-to-try) turns those ideas into things you can try yourself. - Models: pick [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) or [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B) in the sidebar; they run on your machine with Hugging Face `transformers`, or on [vLLM](https://docs.vllm.ai/): pick vLLM (Docker) under What runs the model in the sidebar and the page starts a vLLM server in Docker for you, on the GPU if there is an NVIDIA one and on the CPU otherwise, or set `VLLM_URL` to use a server of your own, on this machine or on a Linux machine with a GPU ([Running the model on a vLLM server](#running-the-model-on-a-vllm-server)) +- The harness: the loop is this project's own (`agent/harnesses/simple_harness.py`), or, picked under What runs the loop in the sidebar, that of [pi](https://pi.dev), a coding agent. pi asks either model above, on transformers or vLLM, which the page serves it on 127.0.0.1, and then every token shows as with the built-in harness; or Claude, GPT or another model pi has credentials for, and then the page draws pi's steps from the events pi reports, without tokens ([pi as the harness](#pi-as-the-harness)) - MTP speculative decoding: one forward pass produces several tokens. With `transformers` you can switch it on in the sidebar for Gemma 4 E2B (it is off by default), with [google/gemma-4-E2B-it-assistant](https://huggingface.co/google/gemma-4-E2B-it-assistant) as the drafter; on vLLM with a GPU both models decode speculatively, Gemma 4 E2B with that drafter and Qwen3.5 with the MTP layer in its own checkpoint, and vLLM in Docker does so by default - Files: attach images and videos (both models) and sound (Gemma 4 E2B, which also takes a recording from the microphone) with the + in the input box, or pick one of the sample files above it (a receipt, a recording, a video, sales data) and ask your own question about it; each becomes a run of tokens in the context, and the page shows them - Writing code (optional): Let the model write code in the sidebar gives the model a `run_python` tool, which runs its Python in a Docker sandbox, reads the data files you attach (CSV, TSV, JSON, text, Markdown, Excel) and draws charts with matplotlib, which appear in the chat. Docker is needed only for this and for vLLM (Docker) @@ -17,20 +18,20 @@ This project is for learning: it shows how an agent loop works underneath, in a On the left is a chat like ChatGPT's; on the right the agent loop is drawn step by step: the model reads the context and proposes an action, the harness carries it out, and the result goes back into the context, until it stops. The model runs on your own computer, or on a vLLM server that sends back each token's log probabilities, so every token's probability, the candidate tokens, the special tokens, the prefill and the generation speed are all in view (on vLLM, as much of it as the server's flags let it send). The models run the same harness, so switching models changes only their text formats, which makes them easy to compare. -[How it works](https://cyyeh.github.io/agent-loop-visualizer/en/) maps each step of one turn, from sending a question to the reply, to the code that does it, with file names, line numbers and excerpts, each a click away from those lines on GitHub. Where the two models, or transformers and vLLM, do a step differently, the step tells each in a tab of its own, and picking one switches every tab of that kind on the page. It lives on GitHub Pages; How it works at the top of the app opens it in a new tab, in the language the app is in. +[How it works](https://cyyeh.github.io/agent-loop-visualizer/en/) maps each step of one turn, from sending a question to the reply, to the code that does it, with file names, line numbers and excerpts, each a click away from those lines on GitHub. Where the two models, transformers and vLLM, or the built-in harness and pi do a step differently, the step tells each in a tab of its own, and picking one switches every tab of that kind on the page. It lives on GitHub Pages; How it works at the top of the app opens it in a new tab, in the language the app is in. ## What the page shows | On the page | What you see | Unit 01 reading (in Chinese) | |---|---|---| | Loop diagram | Which box you are in (context, model, harness, environment, stop) and the arrow that is flowing | §4 An agent is a model in a loop | -| Context | The whole string the model reads in each step: the system prompt, the tool schemas, the conversation and the tool result added after the last step. Hovering over a dark special token says what it is for. An attached image, clip or video is the run of tokens it expands into, drawn as one chip with its count (`<\|image\|>` ×266), and the step card says how many tokens each kind took | §1 The context has a limit, §3 Tool definitions are tokens too | +| Context | The whole string the model reads in each step (with pi on a provider's model, the request pi sent instead, as JSON): the system prompt, the tool schemas, the conversation and the tool result added after the last step. Hovering over a dark special token says what it is for. An attached image, clip or video is the run of tokens it expands into, drawn as one chip with its count (`<\|image\|>` ×266), and the step card says how many tokens each kind took | §1 The context has a limit, §3 Tool definitions are tokens too | | Model | Every generated token, darker the less likely the chosen token was. Hovering over a token shows its probability, its rank, the top 5 candidates and which ones top-k / top-p cut; a special token also says what it is for. With MTP on, tokens the drafter guessed right are underlined, the popover says what the drafter guessed and whether it was kept, and the step card shows how many were kept and how many forward passes the model saved | §1 An LLM samples from a conditional probability distribution; decoding settings | | Harness | The tool calls parsed from the model's own format (Gemma 4 or Qwen3.5), the schema check, warnings about repeated calls, and your Allow or Reject | §3 The model never touches the environment | | Environment | What the tool really returned and how long it took; for `run_python`, the code is written out above and what it printed, or its traceback, below | §2 The environment and observable state | | Stop | Why it stopped and at which token. Gemma stops at `` after an answer (like end_turn) and at `<\|tool_response>` when it wants a tool (like tool_use); Qwen stops at `<\|im_end\|>` either way. Then there are the step limit, the token limit and the user pressing stop | §5 When to stop | | Model tab | The parameter breakdown, the number and kinds of layers, attention heads, the context limit, what the model reads (text, images, sound, video), the MTP drafter, what each decoding setting does, each step's prefill time and tok/s (with MTP, also how many guesses were kept and how many decoding forward passes ran), the id and purpose of every special token of the tokenizer (24 for Gemma, 33 for Qwen), and every file in the model's folder in `models/`: its size, what it is for, what it holds for that model and the values of its key fields | §1, §7 An agent is a system | -| How it works | The 18 stages of one turn (including how an image becomes tokens, how MTP guesses and checks, and what can run the model), each with the code that does it and its line numbers, which open those lines on GitHub, in English and in Chinese. It is on GitHub Pages; the line numbers are looked up in the source when the page is built, the links go to that commit, and it is rebuilt on every push to main, so it follows the code | All of it | +| How it works | The 18 stages of one turn (including how an image becomes tokens, how MTP guesses and checks, and what can run the model), with tabs where the built-in harness and pi differ, each with the code that does it and its line numbers, which open those lines on GitHub, in English and in Chinese. It is on GitHub Pages; the line numbers are looked up in the source when the page is built, the links go to that commit, and it is rebuilt on every push to main, so it follows the code | All of it | ## Quick start @@ -43,6 +44,7 @@ You need: - [uv](https://docs.astral.sh/uv/) 0.13.4 or newer (it installs Python 3.12 for you). Older versions still trust eMudhra's compromised emSign certificates ([the announcement](https://discuss.python.org/t/109576)), so `uv sync` refuses to run with them; `uv self update` (or `brew upgrade uv`) updates it - A network connection: to download the models the first time, and for the Wikipedia tools - Optional: [Docker](https://docs.docker.com/get-started/get-docker/) (Docker Desktop, OrbStack or Colima), only to let the model write code or to run it on vLLM (Docker) +- Optional: [pi](https://pi.dev) and Node.js 22.19 or newer, only to hand the loop to pi ([pi as the harness](#pi-as-the-harness)) ```bash uv sync @@ -50,19 +52,21 @@ uv run hf download google/gemma-4-E2B-it --revision 3e22461f65e89153144f8adb70e3 uv run hf download Qwen/Qwen3.5-0.8B --revision 2fc06364715b967f1860aea9cf38778875588b17 --cache-dir models # about 1.8 GB uv run hf download google/gemma-4-E2B-it-assistant --revision 2d874ef7d29f9a30599a1e4b3c1cbc9595f005df --cache-dir models # the MTP drafter, about 190 MB; otherwise it downloads the first time MTP is switched on uv run streamlit run streamlit_app.py # loads Gemma first; MODEL_ID=Qwen/Qwen3.5-0.8B loads Qwen first -docker build -t agent-loop-sandbox harness/sandbox # optional: the sandbox for Let the model write code (about 400 MB); again whenever harness/sandbox/Dockerfile changes +docker build -t agent-loop-sandbox agent/sandbox # optional: the sandbox for Let the model write code (about 400 MB); again whenever agent/sandbox/Dockerfile changes ``` Your browser opens http://localhost:8501, in its own language (English unless the browser is set to Chinese); the 中文 | English switch at the top right changes it, and the next visit starts in the one you picked. Loading Gemma and warming it up takes about 10–20 seconds the first time, Qwen a few seconds; reloading the page does not load it again. The Model menu at the top of the sidebar switches models; switching starts a new conversation, resets the decoding settings to the new model's recommendations and turns MTP off. MTP speculative decoding in the sidebar loads the drafter the first time it is switched on (in under a second), and it stays in memory after that. -Let the model write code in the sidebar is off by default, and while it is off (and vLLM (Docker) is not picked) the page never looks for Docker. Switched on, it checks for Docker and the `agent-loop-sandbox` image, and if either is missing, or the image was built from an older `harness/sandbox/Dockerfile`, it says in the sidebar how to install Docker or build the image; the model gets `run_python` only once both are there, and the examples that need Python, which appear once it is on, can be pressed only then. Files you attach are saved in a folder per conversation under `~/.cache/agent-loop-visualizer/uploads/` (New chat removes it), which the sandbox sees, read-only, as `/data`. +Let the model write code in the sidebar is off by default, and while it is off (and vLLM (Docker) is not picked) the page never looks for Docker. Switched on, it checks for Docker and the `agent-loop-sandbox` image, and if either is missing, or the image was built from an older `agent/sandbox/Dockerfile`, it says in the sidebar how to install Docker or build the image; the model gets `run_python` only once both are there, and the examples that need Python, which appear once it is on, can be pressed only then. Files you attach are saved in a folder per conversation under `~/.cache/agent-loop-visualizer/uploads/` (New chat removes it), which the sandbox sees, read-only, as `/data`. -The models download into `models/` in the project folder, which git ignores, not into the `~/.cache/huggingface/hub` other projects share; deleting the folder frees the space. Each downloads at a fixed revision, the commit in `REVISIONS` in `harness/families/__init__.py` (the `--revision` above), so whenever you download it you get the files this page was written and tested against, the same on every operating system. To keep them somewhere else, set `AGENT_LOOP_MODELS` to that folder (and give the downloads above the same `--cache-dir`); `AGENT_LOOP_MODELS=$HOME/.cache/huggingface/hub` uses the models already downloaded there. `HF_HUB_CACHE` does not move them, since it is often set once for every project on a machine. +The models download into `models/` in the project folder, which git ignores, not into the `~/.cache/huggingface/hub` other projects share; deleting the folder frees the space. Each downloads at a fixed revision, the commit in `REVISIONS` in `agent/model_families/__init__.py` (the `--revision` above), so whenever you download it you get the files this page was written and tested against, the same on every operating system. To keep them somewhere else, set `AGENT_LOOP_MODELS` to that folder (and give the downloads above the same `--cache-dir`); `AGENT_LOOP_MODELS=$HOME/.cache/huggingface/hub` uses the models already downloaded there. `HF_HUB_CACHE` does not move them, since it is often set once for every project on a machine. To run the model on vLLM instead, pick vLLM (Docker) under What runs the model in the sidebar, and the page starts a vLLM server in Docker for it ([vLLM in Docker, started by the page](#vllm-in-docker-started-by-the-page)); or point the page at a server of your own ([Running the model on a vLLM server](#running-the-model-on-a-vllm-server)). > A 401 while downloading, though the model is public? Most likely the Hugging Face token saved on your machine has expired. Log in again (`uv run hf auth login`), or add `HF_HUB_DISABLE_IMPLICIT_TOKEN=1` to send no token. +> Cloned before `agent/` was called `harness/`? Git ignores the `__pycache__` folders Python wrote there, so a pull leaves them behind, in what looks like an empty `harness/`. Nothing imports it; `git clean -fdX harness` removes it, and only what git ignores in it. + ## Experiments to try 1. **Just a chatbot**: switch off Use tools in the sidebar and ask "What time is it in Taipei right now?". The model sees no tools, so it can only guess or admit it does not know. Switch tools back on, ask again, and compare the two in the Context tab. @@ -81,23 +85,28 @@ To run the model on vLLM instead, pick vLLM (Docker) under What runs the model i ## Architecture -The project has two parts. `harness/` runs a turn: the agent loop, the tools, the model's family and what runs the model. `app/` is what you see: the page, the HTML it draws from the harness's events, and How it works. They depend one way: `app/` builds on `harness/`, and nothing in `harness/` imports `app/` or Streamlit, so the loop runs, and is tested, without a page. +The project has two parts. `agent/` runs a turn: the harness that runs the agent loop, the tools, the model's family and what runs the model. `app/` is what you see: the page, the HTML it draws from the agent's events, and How it works. They depend one way: `app/` builds on `agent/`, and nothing in `agent/` imports `app/` or Streamlit (`test_layout.py` checks), so the loop runs, and is tested, without a page. ``` streamlit_app.py where Streamlit starts: it runs app/page.py's main() on every rerun -app/ what a person sees, built on the harness +app/ what a person sees, built on the agent page.py layout, session state, model menu, language switch, background thread; redraws every 0.5 s while a run is live render.py turns the event list into HTML: the loop diagram, step cards, token heat and popovers, the context view guide.py the How it works page: its content in both languages, line lookups, and the static HTML for GitHub Pages assets/ the page's styles and the guide's script; logo.svg; examples/, the files the examples attach and the sample files above the input box -harness/ what runs a turn; it never imports Streamlit - agent.py the loop: build the prompt → generate → parse → validate → (ask you) → run the tool → back into the context → stop? - backends/ what runs the model +agent/ what runs a turn: the harness, the model, what runs the model, the tools; it never imports Streamlit + harnesses/ what runs the loop + harness.py the Harness the page runs a turn through: run, stop, decide, and the events every harness sends + simple_harness.py this project's loop: build the prompt → generate → parse → validate → (ask you) → run the tool → back into the context → stop? + pi_harness.py pi as the harness: pi in RPC mode, its JSON records turned into the page's events + pi_extension.ts the pi extension the page loads: it reports each request pi sends, registers the page's model, and asks you before pi runs a tool + model_endpoint.py the page's model as an OpenAI-compatible endpoint on 127.0.0.1, for pi: render, encode, generate, and the tokens back to pi + model_backends/ what runs the model backend.py the Backend that the harness and the page run the model through: render, encode, generate, and the fields of a token event simple_backend.py local inference with a hand-written decoding loop; MTP's drafting and checking are here too vllm_backend.py the model on a vLLM server (VLLM_URL): the prompt rendered and tokenized here, the token ids to the server, what it can tell about each token back; media, sampling masks and MTP's guesses on a server started with the flags for them vllm_docker.py vLLM in Docker for the sidebar's vLLM (Docker): docker compose up with the picked model, waiting until it answers; docker compose down - families/ everything that goes with the model: how each model family writes turns, thinking and tool calls, which tokens stop it, what each means + model_families/ everything that goes with the model: how each model family writes turns, thinking and tool calls, which tokens stop it, what each means __init__.py the Family definition, the model menu MODELS, the MTP drafters DRAFTERS, finding a family by config.json's model_type gemma4.py Gemma 4: the output parser and notes on its 24 special tokens qwen3_5.py Qwen3.5: the output parser and notes on its 33 special tokens @@ -113,11 +122,11 @@ tests/ pytest tests; see Tests below models/ the downloaded models (git ignores it; AGENT_LOOP_MODELS moves it) ``` -The agent runs in a background thread and appends an event at every stage; the UI only draws the event list. So when you press Stop or Allow and Streamlit reruns the script, the loop in the background is not interrupted. +The agent runs in a background thread and appends an event at every stage; the UI only draws the event list. So when you press Stop or Allow and Streamlit reruns the script, the loop in the background is not interrupted. The page reaches the loop only through a `Harness` (`harnesses/harness.py`): `run()`, `stop()`, `decide()` and the events it sends. `SimpleHarness` is this project's own, and `PiHarness` hands the loop to pi ([pi as the harness](#pi-as-the-harness)). How it works is a static page: `uv run python -m app.guide _site` writes `_site/en/index.html` and `_site/zh/index.html` (styles and a small script inlined, so they open straight in a browser), and `_site/index.html`, which sends a reader to the language they last picked on the page's switch, or else their browser's. On a push to main, `.github/workflows/pages.yml` runs `tests/test_guide.py`, then does the same and publishes it to GitHub Pages; if the tests fail, the page already up stays as it was. -The events of one run, in order: `run_start`, then for each step `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`, with a tool call also `tool_call` → (`approval_request` → `approval`) → (`tool_start`, when the tool runs) → `tool_result`, and finally `run_end`, after an `error` if something failed: its `message` is the exception's, and a backend's error (a vLLM server's, say) also comes in both languages (`text`), so the page shows the one it is in. `context` carries the prompt as the model reads it, with each attachment expanded into its tokens, and how many tokens each kind of media took (`media_tokens`). `token` carries the token and, when the backend knows them, its probability, its rank, the top 5 candidates and what MTP's drafter guessed (`backend.TOKEN_DETAILS`): `simple_backend.py` knows them all, and the page leaves out whatever a backend does not report rather than showing a number nobody measured. A backend that learns which tokens were MTP's guesses only when the step ends (a vLLM server) says so in `model_end`'s MTP counts (`kept_at`, `turned_down_at`), and the page marks those tokens then. `tool_result` carries the observation the model reads and, apart from it, the images the tool made for you (`images`: the paths of `run_python`'s charts), which the model never reads. +The events of one run, in order: `run_start`, then for each step `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`, with a tool call also `tool_call` → (`approval_request` → `approval`) → (`tool_start`, when the tool runs) → `tool_result`, and finally `run_end`, after an `error` if something failed: its `message` is the exception's, and a backend's error (a vLLM server's, say) also comes in both languages (`text`), so the page shows the one it is in. `context` carries the prompt as the model reads it, with each attachment expanded into its tokens, and how many tokens each kind of media took (`media_tokens`). `token` carries the token and, when the backend knows them, its probability, its rank, the top 5 candidates and what MTP's drafter guessed (`backend.TOKEN_DETAILS`): `simple_backend.py` knows them all, and the page leaves out whatever a backend does not report rather than showing a number nobody measured. A backend that learns which tokens were MTP's guesses only when the step ends (a vLLM server) says so in `model_end`'s MTP counts (`kept_at`, `turned_down_at`), and the page marks those tokens then. `tool_result` carries the observation the model reads and, apart from it, the images the tool made for you (`images`: the paths of `run_python`'s charts), which the model never reads. `run_start` says which harness ran the turn (`harness`: `simple` or `pi`). A step pi runs begins with `request`, the request pi sent the model, as JSON. When pi asks the page's model, the model endpoint sends that step's `context`, `prefill`, `token` and `model_end` as the built-in harness does; when pi asks a provider's model, there is no family and none of those four but `delta`, a piece of the model's output and its channel (thinking, reply, tool call) as it streams, and a `model_end` with the token counts the provider reported (`usage`). ### Two languages @@ -161,11 +170,11 @@ The chat template writes one placeholder where each file goes (`<|image|>` for G ### Writing code in a Docker sandbox -With Let the model write code on and Docker ready, the model gets one more tool, `run_python`. Each call starts a new container from the image `harness/sandbox/Dockerfile` builds: no network, a read-only file system except a small `/tmp`, 512 MB of memory, one CPU, at most 64 processes, no Linux capabilities and your own user, never root (on Linux the mounted folder keeps its permissions, which might not let another user in), with the conversation's folder read-only at `/data`; the code goes in on stdin, and the container is deleted afterwards. Code that runs past 30 seconds is stopped; a nonzero exit, using too much memory or printing nothing (and drawing nothing) comes back to the model as an error, in the turn's language, with the traceback, so it can fix the code in the next step. +With Let the model write code on and Docker ready, the model gets one more tool, `run_python`. Each call starts a new container from the image `agent/sandbox/Dockerfile` builds: no network, a read-only file system except a small `/tmp`, 512 MB of memory, one CPU, at most 64 processes, no Linux capabilities and your own user, never root (on Linux the mounted folder keeps its permissions, which might not let another user in), with the conversation's folder read-only at `/data`; the code goes in on stdin, and the container is deleted afterwards. Code that runs past 30 seconds is stopped; a nonzero exit, using too much memory or printing nothing (and drawing nothing) comes back to the model as an error, in the turn's language, with the traceback, so it can fix the code in the next step. -matplotlib's charts come back too, although the container cannot write anywhere the page can read: the image sets matplotlib's backend to `harness/sandbox/agent_loop_figures.py`, which turns each figure into a PNG at `plt.show()` or when the program ends and, as it exits, writes them after what the code printed, behind a marker line that `run_python` passes in. `run_python` cuts them off what the model reads (at most four), saves them in the conversation's folder as `figure-1.png`, `figure-2.png`… and returns them apart from the observation: the model reads only `{"stdout": …, "figures": ["/data/figure-1.png"]}`, and the page shows the images in the chat and under the step. A label in Chinese is drawn with WenQuanYi Micro Hei, the font the image installs for it. Importing matplotlib makes a call about half a second slower (0.8 s against 0.2 s for plain Python). +matplotlib's charts come back too, although the container cannot write anywhere the page can read: the image sets matplotlib's backend to `agent/sandbox/agent_loop_figures.py`, which turns each figure into a PNG at `plt.show()` or when the program ends and, as it exits, writes them after what the code printed, behind a marker line that `run_python` passes in. `run_python` cuts them off what the model reads (at most four), saves them in the conversation's folder as `figure-1.png`, `figure-2.png`… and returns them apart from the observation: the model reads only `{"stdout": …, "figures": ["/data/figure-1.png"]}`, and the page shows the images in the chat and under the step. A label in Chinese is drawn with WenQuanYi Micro Hei, the font the image installs for it. Importing matplotlib makes a call about half a second slower (0.8 s against 0.2 s for plain Python). -The image installs pip 26.2.2 first, from its wheel by hash, and everything else with it: an older pip also trusts its own copy of certifi, which still has the compromised emSign certificates. The page reads the `agent-loop-sandbox.version` label of the image, and when it is older than `harness/sandbox/Dockerfile` the sidebar asks for a rebuild instead of offering `run_python`. +The image installs pip 26.2.2 first, from its wheel by hash, and everything else with it: an older pip also trusts its own copy of certifi, which still has the compromised emSign certificates. The page reads the `agent-loop-sandbox.version` label of the image, and when it is older than `agent/sandbox/Dockerfile` the sidebar asks for a rebuild instead of offering `run_python`. The model cannot see into a data file, so the harness adds a note to the user's message: where `run_python` finds each file, and the first line of a text file (for a CSV, its column names). Without that line a small model guesses columns in its code, a `Date` that is not there, and fails; with a few rows of data as well, Gemma copied the rows into its code and answered from them alone. `tabulate` is in the image because `DataFrame.to_markdown()`, which small models like to print tables with, needs it. @@ -218,7 +227,7 @@ What the page leaves out it does not make up (`backend.TOKEN_DETAILS`). In detai ### vLLM in Docker, started by the page -Without `VLLM_URL`, the sidebar's What runs the model switches between transformers and vLLM (Docker). Picking vLLM lets go of the model loaded here first (`release_model()` clears the cache and hands the memory back, on CUDA cuBLAS's workspace too, which would otherwise keep the whole model's memory; a switch made while the model loads takes effect once it has loaded, since Streamlit's fast reruns are off), then `harness/backends/vllm_docker.py` downloads the model into `models/` if it is not there yet and starts the server in `docker-compose/` with `docker compose up -d`, showing what Docker and vLLM print meanwhile. Picking transformers again runs `docker compose down`, and so does quitting the app (Ctrl+C), so only one of the two holds the model. What runs the model is one choice for every tab, as the model in memory is: a tab open on transformers follows another that picks vLLM, and back, and a new tab or a reload joins the server with the model it runs, rather than loading the model here beside it. Switching models on vLLM goes through `docker compose up` again, which replaces the container; a rerun with the server already up starts nothing. A start that failed (no Docker, Docker not running, out of memory) says why in the page's language and is tried again only when you press Try again; the switch back to transformers stays in the sidebar. +Without `VLLM_URL`, the sidebar's What runs the model switches between transformers and vLLM (Docker). Picking vLLM lets go of the model loaded here first (`release_model()` clears the cache and hands the memory back, on CUDA cuBLAS's workspace too, which would otherwise keep the whole model's memory; a switch made while the model loads takes effect once it has loaded, since Streamlit's fast reruns are off), then `agent/model_backends/vllm_docker.py` downloads the model into `models/` if it is not there yet and starts the server in `docker-compose/` with `docker compose up -d`, showing what Docker and vLLM print meanwhile. Picking transformers again runs `docker compose down`, and so does quitting the app (Ctrl+C), so only one of the two holds the model. What runs the model is one choice for every tab, as the model in memory is: a tab open on transformers follows another that picks vLLM, and back, and a new tab or a reload joins the server with the model it runs, rather than loading the model here beside it. Switching models on vLLM goes through `docker compose up` again, which replaces the container; a rerun with the server already up starts nothing. A start that failed (no Docker, Docker not running, out of memory) says why in the page's language and is tried again only when you press Try again; the switch back to transformers stays in the sidebar. ```bash # The same server by hand (VLLM_MODEL and VLLM_REVISION default to Qwen3.5 0.8B at its pinned commit); the weights must be in models/ first: @@ -228,15 +237,37 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py ``` - **Where it runs.** Where Docker has the nvidia runtime, `compose.gpu.yaml` goes on top and vLLM's own CUDA image (`vllm/vllm-openai:v0.32.0`) runs on the GPUs; elsewhere, a Mac included, `compose.yaml` runs vLLM's CPU image inside Docker's Linux VM. `AGENT_LOOP_VLLM_DEVICE=cpu` or `gpu` decides instead, and `VLLM_PORT` moves the port (8000). On the GPU the KV cache is 2 GiB, as on the CPU (`--kv-cache-memory-bytes`, `VLLM_KV_CACHE_MEMORY_BYTES`), rather than 92% of the GPU's memory, and vLLM wants only 30% of that memory free to start (`--gpu-memory-utilization`, `VLLM_GPU_MEMORY_UTILIZATION`): at its default 0.92 it refuses on a small card beside the page's own CUDA context, and on a GPU that shares the machine's memory, such as a DGX Spark's. -- **MTP.** On a GPU the server decodes speculatively by default: Gemma 4 with its drafter, Qwen3.5 with the MTP layer in its own checkpoint (`families.VLLM_SPECULATIVE`, handed to Compose as `VLLM_SPECULATIVE_CONFIG`), and each step underlines the guesses it kept. The sidebar's MTP switch turns it off or back on, which starts the server again the other way, and every tab follows. vLLM 0.32's CPU image can decode neither model speculatively, so on a CPU the switch is disabled and says why. Started by hand, the GPU file runs without it. +- **MTP.** On a GPU the server decodes speculatively by default: Gemma 4 with its drafter, Qwen3.5 with the MTP layer in its own checkpoint (`model_families.VLLM_SPECULATIVE`, handed to Compose as `VLLM_SPECULATIVE_CONFIG`), and each step underlines the guesses it kept. The sidebar's MTP switch turns it off or back on, which starts the server again the other way, and every tab follows. vLLM 0.32's CPU image can decode neither model speculatively, so on a CPU the switch is disabled and says why. Started by hand, the GPU file runs without it. - **What the server reads.** It never downloads: `models/` is mounted read-only with `HF_HUB_OFFLINE=1`, so it loads exactly the commit in `REVISIONS`, as the page does. It listens on 127.0.0.1 only, like the page. It is started with `--enable-scale-out`, so the page sees every token's rank and the server takes images, sound and video, and keeps vLLM's default `raw_logprobs`, so the popover shows the model's own probabilities and the top 5. vLLM's compile cache is kept in a Docker volume, so only the first start of a model compiles. - **The CPU image** (`docker-compose/vllm-cpu/`) is `vllm/vllm-openai-cpu:v0.32.0` with two fixes (`patch_vllm.py`): vLLM's own #61992, which came a day after 0.32.0 and without which every model's first request fails on a CPU, and one of ours, which makes the input of a CPU linear layer contiguous, without which the first clip Gemma 4 hears takes the engine down. On the CPU, `compose.yaml` sets the KV cache to 2 GiB (`VLLM_CPU_KVCACHE_SPACE`; otherwise vLLM wants 92% of Docker's memory), skips measuring the image encoder at startup (over 10 minutes on a CPU), and lets one step take up to 30 minutes rather than vLLM's 5 before it gives up on the engine. - **On a DGX Spark** (NVIDIA GB10, whose GPU shares the machine's 128 GB): on the GPU, Qwen3.5 0.8B starts in about 2 minutes and decodes about 113 tok/s, Gemma 4 E2B starts in about 3 minutes and decodes about 41 tok/s, and the real-server tests pass for both, every kind of media Gemma reads included. At vLLM's default `gpu_memory_utilization` (0.92) the server refuses to start there: vLLM takes the machine's available memory for the GPU's free memory, about 108 of 122 GiB. On the same machine's CPU, Qwen3.5 0.8B starts in about 1.3 minutes and decodes about 23 tok/s. - **On a Mac** (Apple M5, 10 cores given to Docker): Qwen3.5 0.8B starts in about 1.5 minutes, takes about 9 GB of Docker's memory and decodes about 10 tok/s. Gemma 4 E2B needs more memory than Docker Desktop's default: with 16 GB it is killed while loading (the page says so and how to give Docker more), with 24 GB it starts in 3 to 5 minutes and decodes about 4.6 tok/s, but checking at load which media the server takes runs its image, audio and video encoders on the CPU and takes about 10 minutes the first time. +### pi as the harness + +The page can also hand the loop to [pi](https://pi.dev), a coding agent with a loop, a system prompt and tools of its own (read, bash, edit, write, grep, find, ls). Pick pi under What runs the loop at the top of the sidebar, then under The model pi asks which model it asks, and the page draws what pi does in the same diagram, step cards and event list. Each step's context section shows the request pi sent the model (its system prompt, its tools and the conversation, as JSON), with what the step added marked. + +- **This page's model** (the default): Gemma 4 E2B or Qwen3.5 0.8B, on transformers or vLLM, picked with the same model menu and What runs the model as for the built-in harness. pi asks models over HTTP alone, so `agent/harnesses/model_endpoint.py` serves the model in memory as an OpenAI-compatible chat completions endpoint on 127.0.0.1, behind a token, and pi's extension registers it as the provider `agent-loop`. Each request is rendered with the model's chat template, encoded and generated as a step of the built-in harness is, and its events go into pi's step: the Context tab shows the string the model read (and, folded under it, the request pi sent), and every token shows with its probability and candidates. The reply streams back to pi, the thinking as `reasoning_content` and the reply as `content`, with the tool calls the family parses out of the model's own format as `tool_calls` at the end. The decoding settings are the sidebar's; Thinking mode reaches the model as pi's thinking level, which the endpoint turns into the chat template's thinking switch. +- **pi's own models**: one pi has credentials for (Claude, GPT, Gemini, or any compatible endpoint in its `models.json`); no model loads here. An API sends back text, not tokens, so there are no token probabilities, candidates or special tokens, and how the API writes the request out for the model nobody says: a step shows pi's request, the text as it streams, the token counts the provider reported (with how many came from the cache) and the cost at pi's prices. + +```bash +nvm install # with nvm: installs the newest Node 22 (.nvmrc) and switches to it; nvm use could pick an older 22 already installed, too old for pi. Install pi under it, since nvm keeps global packages per version +npm install -g @earendil-works/pi-coding-agent # pi 1.4 needs Node.js 22.19 or newer +pi # then /login, once; or set an API key such as ANTHROPIC_API_KEY before starting the app +PI_COMMAND=/path/to/pi uv run streamlit run streamlit_app.py # if pi is not on the PATH (any command works, npx included) +``` + +- **How the page runs it.** `agent/harnesses/pi_harness.py` starts pi for each question in RPC mode (`pi --mode rpc`): commands go to its stdin and records come from its stdout, one JSON object per line. It picks the model and the thinking level, turns pi's own retries off (a retry would be another step, so a failed request is shown as an error instead) and sends the question, each command only once pi has answered the last; one of pi's turns (one request to the model, and the tools its reply calls) is one step. pi keeps the conversation in a session file in the conversation's folder, so the next question continues it; New chat removes it. +- **What it starts pi without.** The user's own extensions, skills, prompt templates and context files (AGENTS.md, CLAUDE.md) stay out (`--no-extensions` and the like), so the page draws pi's own loop, the same on every machine; the models and the credentials are still the user's, in `~/.pi/agent`. +- **Tools.** The sidebar picks which of pi's tools the model sees (read, bash, edit and write by default, as pi offers). They run on this computer, in the conversation's folder, without a sandbox: unlike `run_python`, bash reaches whatever you can. A file you attach is saved in that folder and named in the question; an image also goes with the question to a model that reads images. +- **The extension.** Every pi the page starts loads `agent/harnesses/pi_extension.ts`. Before each request to the model it sends the request as a notification, which in RPC mode is an `extension_ui_request` on stdout and becomes the step's `request`; it registers the page's model when pi asks it; and it is what asks before a tool call (below). +- **Ask me before running a tool.** pi does not ask by itself; with this on, the extension opens a confirm dialog before every tool call. In RPC mode the dialog comes out on stdout, the page shows it as Allow and Reject, and the answer goes back on stdin; a rejected call does not run, and the model reads the refusal in the turn's language. +- **Stopping.** Stop sends pi `abort`. pi has no step limit of its own, so the page keeps Max steps: when pi starts a turn past it, the page aborts pi. An unanswered confirmation stops pi after 5 minutes, as with the built-in harness. +- **Tested** with pi 1.4.0, through a stand-in that speaks its protocol from a script (`tests/fake_pi.py`, which asks the model endpoint over HTTP as pi does) and, with `RUN_PI_TESTS=1`, the real pi, pointed at a scripted model on 127.0.0.1 and at the model endpoint with the fake model behind it. + ### Supporting more models -The harness (`agent.py`), the decoding loop (`simple_backend.py`) and the page (`render.py`) are written once, without a single "if Gemma" branch. Whatever differs between models is gathered in one `Family` object in `families/`: +The harness (`harnesses/simple_harness.py`), the decoding loop (`simple_backend.py`) and the page (`render.py`) are written once, without a single "if Gemma" branch. Whatever differs between models is gathered in one `Family` object in `model_families/`: - Reading the output: `parse` (splits the output into thinking, tool calls and the reply), `thought_prefix`, `channel_switch` (which tokens switch to thinking, a tool call or the reply), `stop_tokens` - Writing the conversation back: `reasoning_key` (the field the thinking is kept in, so the chat template can read it), `text_with_calls` (whether text in front of a tool call is kept) @@ -245,14 +276,14 @@ The harness (`agent.py`), the decoding loop (`simple_backend.py`) and the page ( To add a model: -1. Another checkpoint of a family that is already here (a bigger Gemma 4 or Qwen3.5, say): add a line to `MODELS` in `harness/families/__init__.py`, and one to `REVISIONS` with the commit to download it at. Other Gemma 4 checkpoints have MTP drafters too (`google/gemma-4-*-it-assistant`); add a line to `DRAFTERS` (and the drafter's commit to `REVISIONS`) and the sidebar's MTP switch works for them. -2. A new family: write a module like `harness/families/qwen3_5.py` (a parser for the output plus notes on the special tokens, in both languages) and add its `Family` to `FAMILIES`. Loading a model finds it automatically by the `model_type` in the checkpoint's `config.json`. `tests/test_chat_template.py` checks a new family against the real tokenizer and chat template: every special token has a note, tool calls the template writes into the history parse back, and what the step card marks as new is exactly the tool result. +1. Another checkpoint of a family that is already here (a bigger Gemma 4 or Qwen3.5, say): add a line to `MODELS` in `agent/model_families/__init__.py`, and one to `REVISIONS` with the commit to download it at. Other Gemma 4 checkpoints have MTP drafters too (`google/gemma-4-*-it-assistant`); add a line to `DRAFTERS` (and the drafter's commit to `REVISIONS`) and the sidebar's MTP switch works for them. +2. A new family: write a module like `agent/model_families/qwen3_5.py` (a parser for the output plus notes on the special tokens, in both languages) and add its `Family` to `FAMILIES`. Loading a model finds it automatically by the `model_type` in the checkpoint's `config.json`. `tests/test_chat_template.py` checks a new family against the real tokenizer and chat template: every special token has a note, tool calls the template writes into the history parse back, and what the step card marks as new is exactly the tool result. Or ask for it: on a vLLM server, picking a model of a family the page does not know offers a GitHub issue already filled in, and an issue for any other model you would like to watch is welcome too, at https://github.com/cyyeh/agent-loop-visualizer/issues. ### The two models' formats -Gemma 4 wraps strings in the special token `<|"|>` and leaves keys unquoted, so it is not JSON, and `harness/families/gemma4.py` has its own small parser: +Gemma 4 wraps strings in the special token `<|"|>` and leaves keys unquoted, so it is not JSON, and `agent/model_families/gemma4.py` has its own small parser: ``` <|tool_call>call:get_current_time{timezone:<|"|>Asia/Taipei<|"|>}<|tool_response> @@ -278,33 +309,41 @@ Qwen stops at `<|im_end|>` both after an answer and for a tool, so the harness d ```bash uv run pytest # about 40 s; loads no weights and touches no network +uv run ruff check # lint: likely bugs, unused imports and the order of imports (--fix sorts them) +uv run mypy # types: agent/ and app/ inside every function, tests/ where annotated RUN_MODEL_TESTS=1 uv run pytest tests/test_real_model.py # loads the real Gemma (with its MTP drafter) and Qwen, about 4 minutes; add -k Qwen for one VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # asks a running vLLM server RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # starts vLLM in Docker with Qwen3.5 and asks it +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # starts the real pi (on the PATH, or PI_COMMAND) against a scripted model on 127.0.0.1, and against the model endpoint ``` -Most tests use the fake model in `tests/fakes.py`: it replays scripted output one token at a time and is a `Backend` (`backend.py`) like `SimpleBackend`, so the harness and the page run from start to finish. +Most tests use the fake model in `tests/fakes.py`: it replays scripted output one token at a time and is a `Backend` (`backend.py`) like `SimpleBackend`, so the harness and the page run from start to finish. In pi's place they start `tests/fake_pi.py`, which speaks pi 1.4's RPC protocol from a script, so they need neither Node nor a model. Every file format the page accepts has a test: `tests/samples.py` makes a small file of each one from the examples' files (the receipt in every image format, the recording in every audio format, the rolling ball in every video container, the sales data in every data format). `test_media.py` fails if `media.py` accepts a format without a sample, and `test_real_model.py` asks both models about each one. The tests that need Docker run only when Docker and the sandbox image are there. +On every pull request and every push to main, `.github/workflows/checks.yml` runs `ruff check`, `mypy` and `uv run pytest` on Ubuntu; the tests that need the models, the sandbox image, a vLLM server or pi skip there. Beside them it runs `tests/test_guide.py` and builds How it works with pygments alone, as `pages.yml` does on main, so a pull request that has the page's code import torch or the rest of the app fails before it is merged, not when the page is published. + How it works quotes code, and `tests/guide_excerpts.txt` keeps everything it quotes, without line numbers. When a change touches quoted code, `test_guide.py` fails and names the sections that quote it: reread them, fix what they say if it is no longer true, then rewrite the file with `UPDATE_GUIDE_SNAPSHOT=1 uv run pytest tests/test_guide.py`. | File | What it tests | |---|---| | `test_gemma4.py` | Gemma 4's parser: splitting thinking, tool calls and the reply; `<\|"\|>` strings, nested arguments, all kinds of format errors, in both languages | | `test_qwen3_5.py` | Qwen3.5's parser: XML tool calls, types from the schema, thinking, all kinds of format errors; a whole harness run in Qwen's format | -| `test_families.py` | Every family explains the tokens it uses, in both languages and for the same tokens; `model_type` finds the family; a token shows as the text it stands for, and a byte that is only part of a character as `<0xF0>`; every family explains the files of its checkpoints in both languages, naming the same fields and tokens; every model and drafter downloads at a pinned commit; vLLM drafts for a listed model with the MTP layer in its checkpoint or the drafter transformers uses | +| `test_model_families.py` | Every family explains the tokens it uses, in both languages and for the same tokens; `model_type` finds the family; a token shows as the text it stands for, and a byte that is only part of a character as `<0xF0>`; every family explains the files of its checkpoints in both languages, naming the same fields and tokens; every model and drafter downloads at a pinned commit; vLLM drafts for a listed model with the MTP layer in its checkpoint or the drafter transformers uses | | `test_tools.py` | Schema validation and defaults, the calculator's AST allowlist (rejecting `__import__` and the like), timezones, Wikipedia (with fake HTTP); `run_python`'s locked-down `docker run`, its errors (a traceback, no output, too much memory, too long, no image) and what is missing when Docker is not ready or the image is out of date, in both languages; the charts after the marker saved without overwriting a file, only PNGs, at most four, none from code that failed; the image's version label and its pip installed by hash; with Docker, the real sandbox reads `/data` but cannot change it or reach the network, reads every data format, sends back its charts with Chinese labels drawn, and has a pip (with its certifi) past the emSign fix | | `test_simple_backend.py` | `SimpleBackend` has every method and attribute of a `Backend`, and its tokens carry every field and detail; the hand-written decoding loop: KV cache, stop tokens, the token limit, interruption, channel switches, Chinese characters split across tokens; `encode()` handing the processor each kind of media in order with the family's settings, and a short video all its frames; images and videos through the vision encoder before the prefill, and no position offset left from the last prompt; greedy taking the lower id of two tied tokens, as `generate()` does; the candidates and probabilities of temperature / top-k / top-p; MTP with a scripted fake drafter: kept and rejected guesses, the extra token when all hold, how far the KV cache is cut back, the drafter never seeing dropped guesses, and a check that guesses do not change the distribution when sampling; models downloading at their pinned revision into `models/`, which `AGENT_LOOP_MODELS` moves (a relative one from where the app started) and `HF_HUB_CACHE` does not, letting a model go on CUDA freeing cuBLAS's workspaces before emptying the cache, and a model folder's files, their sizes (through the cache's links) and the fields its notes name | -| `test_agent.py` | The fake model is a `Backend` whose tokens carry what `SimpleBackend`'s do; the harness: event order, tool results back in the context, validation errors, repeated calls, truncated results, every stop reason, Allow / Reject / Stop while waiting; the model reads errors in the turn's language, and the system prompt is the same in both; attachments as parts of the question and a note for `run_python`, the media of every turn going to the processor, the expanded context still marking only the tool result as new, `run_python` getting the conversation's folder, and a turn without `run_python`; the model reading where the charts are while the page gets the images | -| `test_render.py` | Every prefix of a run's events renders in both languages (the page redraws every 0.5 s mid-run), HTML tags pair up, the loop diagram is valid SVG, model output is always escaped, special token popovers and lists; MTP's underlines, popovers, counts and drafter card; a run of one special token as one chip with its count, a cut snippet never splitting a token, how many tokens the media took, code and output over several lines written out, tools left out of a turn, the charts of a run and of each step; the model folder card: the files in order with their sizes and notes, a file the family notes but the checkpoint lacks, the drafter's folder, escaping; tokens from a backend that reports no probabilities, or only log probabilities, drawn with what they carry and no more; no Chinese left on an English page | -| `test_app.py` | Drives the whole page with Streamlit's `AppTest`: Streamlit not collecting garbage after every script run, and a rerun waiting for the run in progress; sending a question, the example buttons, sidebar settings reaching the model, multi-turn chat, the Context and Model tabs, Allow / Reject / Stop, a new chat, resetting the settings, two tabs never running at once, switching models, How it works linking to GitHub Pages in the page's language; MTP off by default, the drafter loaded only when it is switched on, and switched off and disabled for Qwen, which has none; an English browser gets English, the last pick beats the browser's language, switching languages keeps the conversation and the settings, and the model's errors follow the language; each page shows only its own language's examples, an example attaches its file and shows only when the model reads that kind, a sample file picked above the input box goes with the next question typed and is offered only when the model can use it, Let the model write code looks for Docker only once it is on and says what is missing, the sales example runs Python on the attached file, a chart shows in the chat and in the step that drew it, and New chat removes the conversation's files; with `VLLM_URL`, the menu lists what the server serves, the sidebar and the Model tab name the server, MTP is the server's, a server with sampling masks gets no greedy decoding or unlimited top-k from the sidebar, the model reads the media the server takes and the page says which it refused and why, the menu leaves out the LoRA adapters the server serves, marks the models of a family the page does not support and starts on the first it does, and picking one of those says why and offers a GitHub issue filled in in the page's language, naming the model (a folder by its name alone) and never the server's address, an error the server sends mid-run is shown in the page's language, and an unreachable server, at load or later, is said in the page's language; without `VLLM_URL`, What runs the model starts on transformers and leaves Docker alone, picking vLLM (Docker) lets go of the local model (nothing keeps it) and starts the server with it, a rerun starts nothing, another model starts the server with that one, picking transformers again stops the container, a new tab or a reload while vLLM runs joins its server with its model and a tab on transformers follows another that picks vLLM and back (the model loaded here once), on a GPU vLLM starts with MTP on and the switch starts it again without (and back, with the reset button), on a CPU it runs without MTP and the switch says why, a tab follows the MTP another tab picked, and a start that failed is said in the page's language, tried again only on Try again, with the switch back to transformers still there | +| `test_simple_harness.py` | The fake model is a `Backend` whose tokens carry what `SimpleBackend`'s do; the harness: event order, tool results back in the context, validation errors, repeated calls, truncated results, every stop reason, Allow / Reject / Stop while waiting; the model reads errors in the turn's language, and the system prompt is the same in both; attachments as parts of the question and a note for `run_python`, the media of every turn going to the processor, the expanded context still marking only the tool result as new, `run_python` getting the conversation's folder, and a turn without `run_python`; the model reading where the charts are while the page gets the images | +| `test_pi_harness.py` | pi as the harness, with `tests/fake_pi.py` in pi's place: both harnesses are a `Harness`, and pi sends only the events it names; a turn becomes the page's events in the loop's order (the thinking, text and tool-call arguments streamed by channel, the provider's token counts with the cache, the calls and their results); pi started in the conversation's folder on its session, with the sidebar's tools and without the user's extensions, the model set and its retries off before the question goes, no tools at all, attachments named for the model and images sent inline, a repeated call marked, a tool that fails, the token limit, the next question on the same session; asking first through the extension: Allow, Reject with the refusal in the turn's language, Stop while a call waits, a timeout; Stop mid-reply and before pi has started, and the step limit drawing no step past it; a failed request, a refused question, a model pi does not have, pi dying as it starts and pi not installed, said in both languages; which models pi has; with `RUN_PI_TESTS=1`, the real pi against a scripted model on 127.0.0.1, with and without asking first; each step's request with what is new marked and no image bytes; pi asking the page's model through the endpoint (Gemma and Qwen): every token, the context with the tool result as new, the endpoint's own `model_end`, no `delta`, pi's thinking level as the template's thinking, an image reaching the model, Stop stopping the model before its next token, asking first (Allow and Reject, the model reading what came of it), the step limit turning the request past it away at once and leaving the endpoint free; with `RUN_PI_TESTS=1`, the real pi asking the page's model through the endpoint, thinking on and off, and once asking first and stopping at the step limit | +| `test_model_endpoint.py` | The page's model served to pi, over HTTP on 127.0.0.1 with the fake model behind it: only on this machine, only with its token, nothing while no turn is served, one request at a time; a reply streamed as `content` with its usage, a tool call in Gemma's or Qwen's own format sent back as `tool_calls` and not as the reply, thinking as `reasoning_content` when the request turns it on, a call the family cannot parse not sent, the sidebar's token limit as `length`, a failing backend in the finish reason; the request drawn as the step's context (what is new being the tool result), prefill, tokens and end, one outside every step answered undrawn, a stopped turn stopping the model before its next token; the conversation as `SimpleHarness` keeps it (one system message, the arguments as a dict, the thinking under the family's key, a tool result named by its call) and an image saved once under its hash as a part | +| `test_render.py` | Every prefix of a run's events renders in both languages (the page redraws every 0.5 s mid-run), HTML tags pair up, the loop diagram is valid SVG, model output is always escaped, special token popovers and lists; MTP's underlines, popovers, counts and drafter card; a run of one special token as one chip with its count, a cut snippet never splitting a token, how many tokens the media took, code and output over several lines written out, tools left out of a turn, the charts of a run and of each step; the model folder card: the files in order with their sizes and notes, a file the family notes but the checkpoint lacks, the drafter's folder, escaping; tokens from a backend that reports no probabilities, or only log probabilities, drawn with what they carry and no more; a run pi ran: its request as the context section with what each step added marked, the text it streamed by channel and no token numbers, escaped, joined without changing the events, the diagram, the stop card and the step table saying pi runs the loop, and pi's cards in both languages; a run of pi on the page's model drawn with its tokens and context; no Chinese left on an English page | +| `test_app.py` | Drives the whole page with Streamlit's `AppTest`: Streamlit not collecting garbage after every script run, and a rerun waiting for the run in progress; sending a question, the example buttons, sidebar settings reaching the model, multi-turn chat, the Context and Model tabs, Allow / Reject / Stop, a new chat, resetting the settings, two tabs never running at once, switching models, How it works linking to GitHub Pages in the page's language; MTP off by default, the drafter loaded only when it is switched on, and switched off and disabled for Qwen, which has none; an English browser gets English, the last pick beats the browser's language, switching languages keeps the conversation and the settings, and the model's errors follow the language; each page shows only its own language's examples, an example attaches its file and shows only when the model reads that kind, a sample file picked above the input box goes with the next question typed and is offered only when the model can use it, Let the model write code looks for Docker only once it is on and says what is missing, the sales example runs Python on the attached file, a chart shows in the chat and in the step that drew it, and New chat removes the conversation's files; with `VLLM_URL`, the menu lists what the server serves, the sidebar and the Model tab name the server, MTP is the server's, a server with sampling masks gets no greedy decoding or unlimited top-k from the sidebar, the model reads the media the server takes and the page says which it refused and why, the menu leaves out the LoRA adapters the server serves, marks the models of a family the page does not support and starts on the first it does, and picking one of those says why and offers a GitHub issue filled in in the page's language, naming the model (a folder by its name alone) and never the server's address, an error the server sends mid-run is shown in the page's language, and an unreachable server, at load or later, is said in the page's language; without `VLLM_URL`, What runs the model starts on transformers and leaves Docker alone, picking vLLM (Docker) lets go of the local model (nothing keeps it) and starts the server with it, a rerun starts nothing, another model starts the server with that one, picking transformers again stops the container, a new tab or a reload while vLLM runs joins its server with its model and a tab on transformers follows another that picks vLLM and back (the model loaded here once), on a GPU vLLM starts with MTP on and the switch starts it again without (and back, with the reset button), on a CPU it runs without MTP and the switch says why, a tab follows the MTP another tab picked, and a start that failed is said in the page's language, tried again only on Try again, with the switch back to transformers still there; with pi picked under What runs the loop, the intro saying what shows of the model pi asks, pi starting on the page's model with its model menu and decoding settings, a question to it drawing every token, its context and pi's request in the Context tab, and the model's card in the Model tab; on pi's own models, its models and settings in the sidebar and no decoding settings, a question drawn as pi's loop, run in the conversation's folder, the sidebar's settings reaching pi, Allow through the page, Stop, the Context tab showing pi's request and saying what it cannot show, and the Model tab showing pi's model and tools, pi without a model or not installed said with a Try again, Docker never looked for, the built-in harness's model and settings kept across a switch, a switch starting a new conversation, and an English page with no Chinese, on either kind of model | | `test_chat_template.py` | With each model's real tokenizer and chat template (only when downloaded into `models/`): every special token is a single token and has a note, every token's popover shows the text it stands for, tool calls the template writes into the history parse back; a harness run with the real template checks that what the step card marks as new is exactly the tool result (two arguments, thinking mode, text before a tool call), and with the real processor, with an image and a video in the question too; every file in each model's folder (and its drafter's) has a note, and every field a note names is in the file | | `test_vllm_backend.py` | The vLLM backend against a fake server that speaks the streams of vLLM 0.32: `/v1/completions` gets the rendered prompt's token ids, every sidebar setting with every other sampling setting neutral, the stop tokens and a `max_tokens` that fits the server's context; each token's probability and the top 5, the rank only within them; with `--enable-scale-out`, `/inference/v1/generate` and every token's rank, the top 5 by rank though the chosen token comes first; on either route a token tied for first ranked #1, though vLLM's rank counts the tie; with sampling masks, which vLLM sends only with processed log probabilities, the candidates left and the sampling probability but not the model's own probabilities, and temperature 0 and top-k 0 refused on every route, as vLLM refuses them, which the page finds out at load on a server with masks but without `--enable-scale-out` too, while a server that is not ready is no such refusal; the server's `--logprobs-mode` worked out at load with one token, processed log probabilities without masks showing how many were left when some of the top were cut, logits showing only the rank; greedy as one candidate drawn for sure; which media the server takes, each sent as a file with the placeholders and the Context tab showing the server's expansion, a different expansion at generation time an error, a refused kind a plain file; MTP's counts on either route, the per-step ones marking the guesses only when they add up and counting as kept only those up to the stop token; the stop token, the token limit, Stop closing the stream; an unreachable server, a model it does not serve, one started from a folder of its own or with a model of a family the page (or its transformers) does not know or without a config.json, a missing or failing `/inference/v1/generate`, a rank the server took from the logits staying true to the listed probabilities where they tie, its refusals and errors, in both languages, and an error mid-run shown in the page's language; which family a served model is of, from its config.json alone (a folder on this machine read where it is, a repo without one of no family, and with no way to the Hub or from a folder only on the server not told), and the model id and `model_type` an unsupported model's error carries for the page's GitHub issue; warming up, and a step that ends before its first token; the model card's specs, the server, what its numbers are in each logprobs mode, its sampling masks and the media it refused; a whole turn with a tool call and one with an image, every moment of each rendered; with the real tokenizer and chat template (when in `models/`), the server gets exactly the prompt the Context tab shows; with `VLLM_TEST_URL`, a real server, each feature it was started with | | `test_vllm_docker.py` | Starting vLLM in Docker, with Docker faked at the level of `subprocess`: the weights downloaded at their pinned commit before `docker compose up` runs `docker-compose/compose.yaml` with the model, its revision and `models/`, and `compose.gpu.yaml` on top where Docker has the nvidia runtime (or `AGENT_LOOP_VLLM_DEVICE` says so); a server already up for the model not started again, another model replacing it; no Docker, Docker not running, no Compose, `docker compose up` failing, a container killed for memory, one that exits (the exceptions in its log, cause first) and one that never answers, each said in both languages; a rerun while Compose is busy stopping it too; `docker compose down`; the model the app asked for kept through a failed start until it is stopped, and quitting the app stopping the server it started (and leaving Docker alone without one); on the GPU, MTP with Gemma's drafter at its pinned commit (downloaded first) or Qwen's own MTP layer, never on the CPU or unasked, and switching it starting the server again. The compose files: on 127.0.0.1 only, `models/` read-only and offline, Qwen3.5 at its pinned commit by default, the GPU file the same vLLM with the same flags but the CPU-only one, plus a KV cache of the CPU's size, a low `--gpu-memory-utilization`, and speculative decoding off by hand, its counts in detail only with a config (vLLM refuses them without one), Docker Compose handing vLLM the page's config as it is, and Docker Compose reading both (when installed) and mounting the folder a relative `AGENT_LOOP_MODELS` names; with `RUN_VLLM_DOCKER_TESTS=1`, the real server in Docker answering through `VLLMBackend`, with MTP and then without where Docker has a GPU | | `test_media.py` | Every format the page accepts has a sample and is read as what it is (sound is a file for Qwen) and decodes; names saved distinct and plain; the question's parts and the note naming a text file's columns; photos turned upright, sound at the model's rate, a video's frames evenly spaced at their own rate; the type each clip plays back as, and the frames shown for an MKV video | | `test_guide.py` | Every excerpt and line number on How it works resolves and links to its lines on GitHub at the page's commit, the event table points at the line that really sends each event, HTML tags pair up; the code each section quotes matches `tests/guide_excerpts.txt`, which an edit to that code changes and a line merely moving does not; each paragraph points at the same code and sections in both languages, with no Chinese left in the English one; every set of tabs is radio buttons of known backends or models, one panel each, that switch without a script, and an excerpt of `compose.yaml` ends where it says; the built pages have light and dark palettes and loop diagrams and name their commit, and the root page sends readers to the right language | -| `test_real_model.py` | Skipped by default. For each model: the loaded model is a `Backend`; greedy decoding matches `model.generate()`, with text and with an image, a video and (Gemma) a recording, and a text prompt after them too; the real model calls the calculator and answers; it reads the receipt's total in every image format, transcribes the recording in every audio format (Gemma; Qwen gets it as a file), tells which way the ball rolls in every video container, opens every data format with `run_python`, and writes code that runs in the sandbox (with Docker); for Gemma with MTP, greedy output is exactly the same as without it, with an image too, and the drafter's guesses really are kept; on CUDA, a warmed-up model let go hands all its GPU memory back | +| `test_layout.py` | No file in `agent/` imports `app` or Streamlit, at the top or inside a function, read from each file's syntax tree: the agent never knows the app | +| `test_real_model.py` | Skipped by default. For each model: the loaded model is a `Backend`; greedy decoding matches `model.generate()`, with text and with an image, a video and (Gemma) a recording, and a text prompt after them too; the real model calls the calculator and answers; it reads the receipt's total in every image format, transcribes the recording in every audio format (Gemma; Qwen gets it as a file), tells which way the ball rolls in every video container, opens every data format with `run_python`, and writes code that runs in the sandbox (with Docker); for Gemma with MTP, greedy output is exactly the same as without it, with an image too, and the drafter's guesses really are kept; on CUDA, a warmed-up model let go hands all its GPU memory back; with pi installed, the real pi asks the model through the model endpoint, calls bash and answers, every token drawn | ## Known limitations @@ -314,7 +353,7 @@ How it works quotes code, and `tests/guide_excerpts.txt` keeps everything it quo - With MTP on, the model computes 7 positions at once, which gives very slightly different numbers from computing 1; when two tokens are almost equally likely, greedy decoding may now and then pick the other one. - 18 of Qwen3.5's 24 layers are Gated DeltaNet. transformers has two kernels to speed them up, `flash-linear-attention` (Triton) and `causal-conv1d` (CUDA). Both need a CUDA GPU, so neither is a dependency: the PyTorch reference implementation runs instead, with the same results, only slower, and `simple_backend.py` silences transformers' notice about that fallback on every device. With a GPU the difference is small for a 0.8B model. On an NVIDIA GB10 (DGX Spark), flash-linear-attention made prefill about 4× faster (3,571 tokens in 93 ms instead of 368 ms) but decoding barely changed (56 → 60 tok/s), and both paths stayed equally close to float32. causal-conv1d has no wheel for that machine and would have to be compiled. Don't install flash-linear-attention on a Linux machine without a GPU: transformers still picks it, and the first forward pass fails. - On MPS, transformers hangs loading Qwen3.5's weights with several threads, so on MPS `simple_backend.py` loads with one thread (`HF_DEACTIVATE_ASYNC_LOAD=1`), which does not make either model load slower. -- Qwen3.5 0.8B has thinking off by default; on, it easily keeps thinking without stopping, and lowering Max tokens per step helps. It ships no generation_config.json, so its recommended decoding settings are in `harness/families/qwen3_5.py` (the presence penalty it recommends is not implemented here). +- Qwen3.5 0.8B has thinking off by default; on, it easily keeps thinking without stopping, and lowering Max tokens per step helps. It ships no generation_config.json, so its recommended decoding settings are in `agent/model_families/qwen3_5.py` (the presence penalty it recommends is not implemented here). - Only one model is kept in memory. If two tabs pick different models, using them in turn keeps reloading; it is best for every tab to use the same model. - There is one copy of the model, so only one agent runs at a time and other tabs wait. Reloading or closing the tab during a run does not stop the agent; other tabs (including the reloaded one) show it is still running and offer a Stop that agent button. - With Ask me before running a tool on, if no one answers within 5 minutes the harness takes it as a rejection and stops the loop, with "confirmation timed out" as the stop reason. @@ -333,6 +372,10 @@ How it works quotes code, and `tests/guide_excerpts.txt` keeps everything it quo - `vllm_backend.py` follows the protocols of vLLM 0.32, read from its source, and is tested against a fake server that speaks them, with the real tokenizers and in the real page; `/inference/v1/generate` and its sampling masks and per-step counts are marked experimental in vLLM and may change. Its tests against a real server (`VLLM_TEST_URL`) have been run on one GPU only, a DGX Spark's GB10. - vLLM in Docker runs on the CPU on a Mac, and slowly: Gemma 4 E2B decodes about 4.6 tok/s, needs Docker to have about 20 GB of memory or more, and its first load spends about 10 minutes running its media encoders to see which kinds the server takes. `compose.gpu.yaml` has been run on one NVIDIA GPU only, a DGX Spark's GB10. The CPU image patches vLLM 0.32.0 (`docker-compose/vllm-cpu/patch_vllm.py`); a newer vLLM may need neither fix, and the build stops if the code it patches is not the one it was written for. - The app stops vLLM's container when it quits with Ctrl+C or a SIGTERM. Stopped any other way (its terminal closed, `kill -9`), it leaves the container running and holding the model; `docker compose -p agent-loop-vllm down` stops it. +- With pi on its own models, the page sees only what pi reports: no tokens, probabilities or special tokens, and not the string the model reads, only the request pi sends. The token counts are the provider's, tok/s is measured as the reply arrives (the network included), and the cost is at the prices in pi's catalog, an estimate. pi's own models are a provider's: they cost money, and the conversation goes to that provider. +- With pi on the page's model, a tool call the family cannot parse is not sent to pi, and pi's turn ends there: unlike the built-in harness, nothing tells the model what it wrote wrong. pi's system prompt and tool schemas are written for large models; Gemma 4 E2B and Qwen3.5 0.8B follow them less well than the built-in harness's prompt. pi sends images alone of the media, so audio and video attached for pi are plain files in its folder. A request that comes outside every step (pi compacting a long conversation, say) waits up to 10 seconds for one, then is answered without being drawn. +- pi's tools run on this computer, without a sandbox; with bash on, the model can do whatever you can. Ask me before running a tool shows each call first. +- `pi_harness.py` follows pi 1.4's RPC protocol, from its documentation and the records pi 1.4.0 sends; the real-pi tests have been run with pi 1.4.0 only. pi 1.4 needs Node.js 22.19 or newer: under an older Node it does not start, and the page shows what it said. The page asks pi which models it has at most once a minute; after logging in to a provider in pi, Try again lists them. - The `MODEL_ID` environment variable picks the model selected at first; it may also name a checkpoint the menu does not list, as long as it belongs to a supported family (Gemma 4 or Qwen3.5); another family needs one added first, as in Supporting more models. ## License diff --git a/README.zh-TW.md b/README.zh-TW.md index f45f734..e46b9a2 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -4,11 +4,12 @@ [English](README.md) | 繁體中文 -![上方可以切換到「運作說明」頁，右上角可以切換語言，側欄可以選模型和用什麼跑它；左邊是對話，輸入框上方是可以拿來問的範例檔案，右邊是迴圈圖與第 1 步的 token。滑鼠停在 <|tool_call> 這個特殊 token 上，浮窗說明它的用途，並顯示它的機率、排名與前 5 名候選](docs/screenshot-zh.png) +![上方可以切換到「運作說明」頁，右上角可以切換語言，側欄可以選用什麼跑迴圈（內建 harness 或 pi）、模型和用什麼跑它；左邊是對話，輸入框上方是可以拿來問的範例檔案，右邊是迴圈圖與第 1 步的 token。滑鼠停在 <|tool_call> 這個特殊 token 上，浮窗說明它的用途，並顯示它的機率、排名與前 5 名候選](docs/screenshot-zh.png) 這個專案是學習用的：用更直覺、更視覺化的方式，讓你看懂 agent 迴圈在底層怎麼運作。Agent 的聊天框背後跑著一個迴圈：一步接著一步，模型提出動作、harness 去執行，直到停止。一般的聊天 app 只讓你看到問題和答案；這個網頁在每一步發生的當下把它畫出來，細到模型寫下的每一個 token，整個機制攤開在你眼前。畫面上的每個部分都對應 [Agent Atlas 技術與實戰交流](https://cyyeh.github.io/agent-atlas-learning/) unit 01 導讀的一節，本專案就是它的專案之一；[可以試試看的實驗](#可以試試看的實驗)則把這些概念變成可以自己動手試的實驗。 - 模型：側欄可以選 [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) 或 [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B)，在本機用 Hugging Face `transformers` 推論，也可以用 [vLLM](https://docs.vllm.ai/) 跑：在側欄的「用什麼跑模型」選 vLLM (Docker)，網頁會替你在 Docker 裡啟動一台 vLLM 伺服器，有 NVIDIA GPU 就用 GPU，否則用 CPU；或是設定 `VLLM_URL`，連到你自己的伺服器，在這台電腦上或有 GPU 的 Linux 機器上都行（見[在 vLLM 伺服器上跑模型](#在-vllm-伺服器上跑模型)） +- Harness：迴圈是這個專案自己寫的（`agent/harnesses/simple_harness.py`），也可以在側欄的「用什麼跑迴圈」選 [pi](https://pi.dev) 這個 coding agent。pi 可以問上面兩個模型，跑在 transformers 或 vLLM 上，由網頁在 127.0.0.1 上提供給它，這時每個 token 都和內建 harness 一樣看得到；也可以問 Claude、GPT 或其他它登入過的模型，這時網頁照 pi 回報的事件畫出它的每一步，看不到 token（「[pi 當 harness](#pi-當-harness)」） - MTP 推測解碼：一次 forward 產生好幾個 token。用 `transformers` 時，Gemma 4 E2B 可以在側欄打開（預設關閉），用 [google/gemma-4-E2B-it-assistant](https://huggingface.co/google/gemma-4-E2B-it-assistant) 當草稿模型；在有 GPU 的 vLLM 上兩個模型都會推測解碼，Gemma 4 E2B 用同一個草稿模型，Qwen3.5 用它自己 checkpoint 裡的 MTP 層，由網頁在 Docker 裡啟動的 vLLM 預設就打開 - 檔案：用輸入框左邊的 ＋ 附上圖片和影片（兩個模型都行）、聲音（Gemma 4 E2B，也可以直接用麥克風錄音），或是在輸入框上方選一個範例檔案（收據、錄音、影片、銷售資料），自己問它問題；每個檔案在脈絡裡變成一串 token，畫面上看得到 - 寫程式（選用）：在側欄打開「讓模型寫程式」，模型多一個 `run_python` 工具，它寫的 Python 在 Docker 沙箱裡執行，讀得到你附上的資料檔（CSV、TSV、JSON、純文字、Markdown、Excel），也能用 matplotlib 畫圖，圖會出現在聊天區。只有這個功能和 vLLM (Docker) 需要 Docker @@ -17,20 +18,20 @@ 左邊是像 ChatGPT 的聊天介面，右邊一步一步畫出 agent 迴圈：模型讀進脈絡、提出動作、harness 去執行、結果接回脈絡，直到停止。模型跑在你自己的電腦上，或是跑在一台會把每個 token 的 log 機率傳回來的 vLLM 伺服器上，所以每個 token 的機率、候選 token、特殊 token、prefill 和生成速度都看得到（用 vLLM 時，看得到多少要看伺服器開了哪些參數）。不同模型跑的是同一個 harness，換模型時只有它們的文字格式不同，正好可以對照。 -[運作說明](https://cyyeh.github.io/agent-loop-visualizer/zh/)把一輪對話從送出到回覆的每一步對應到程式碼，附上檔名、行號和程式片段，點一下就能在 GitHub 上看到那幾行。兩個模型、或 transformers 和 vLLM 做法不一樣的步驟，分成頁籤各自說明，選了一個，整頁同一類的頁籤都跟著換。它放在 GitHub Pages 上，app 上方的「運作說明」會在新分頁打開它，語言和網頁當下的語言一樣。 +[運作說明](https://cyyeh.github.io/agent-loop-visualizer/zh/)把一輪對話從送出到回覆的每一步對應到程式碼，附上檔名、行號和程式片段，點一下就能在 GitHub 上看到那幾行。兩個模型、transformers 和 vLLM，或內建 harness 和 pi 做法不一樣的步驟，分成頁籤各自說明，選了一個，整頁同一類的頁籤都跟著換。它放在 GitHub Pages 上，app 上方的「運作說明」會在新分頁打開它，語言和網頁當下的語言一樣。 ## 畫面上看得到什麼 | 畫面 | 看得到的東西 | 對應 unit 01 導讀 | |---|---|---| | 迴圈圖 | 目前在脈絡、模型、harness、環境、停止哪一格，正在流動的箭頭 | §4 Agent 就是放在迴圈裡的模型 | -| 脈絡 | 每一步模型讀到的完整字串：system prompt、工具 schema、對話，以及上一步接回來的工具結果。滑過深色的特殊 token 會說明它的用途。附上的圖片、聲音或影片就是它展開成的那一串 token，畫成一個標籤加上個數（`<\|image\|>` ×266），step 卡片也寫出每種媒體佔了幾個 token | §1 脈絡有上限、§3 工具定義也是 token | +| 脈絡 | 每一步模型讀到的完整字串（pi 問 provider 的模型時，改成 pi 送出的請求，寫成 JSON）：system prompt、工具 schema、對話，以及上一步接回來的工具結果。滑過深色的特殊 token 會說明它的用途。附上的圖片、聲音或影片就是它展開成的那一串 token，畫成一個標籤加上個數（`<\|image\|>` ×266），step 卡片也寫出每種媒體佔了幾個 token | §1 脈絡有上限、§3 工具定義也是 token | | 模型 | 每個生成的 token，顏色越深代表被選中的 token 機率越低。滑過 token 看機率、排名、前 5 名候選，以及 top-k / top-p 剃掉了哪些；特殊 token 另外說明它的用途。打開 MTP 時，草稿模型猜中的 token 有底線，浮窗寫出草稿猜了什麼、有沒有被收下，step 卡片顯示收下幾個、主模型少跑了幾次 forward | §1 LLM 是會抽樣的條件機率分布、解碼設定 | | Harness | 從模型自己的格式（Gemma 4 或 Qwen3.5）解析出的工具呼叫、schema 驗證結果、重複呼叫警告、你的允許或拒絕 | §3 模型從不直接碰環境 | | 環境 | 工具實際回傳的結果與耗時；`run_python` 的程式碼寫在上面，印出來的東西或 traceback 寫在下面 | §2 環境與可觀察的狀態 | | 停止 | 停止原因和停止 token。Gemma 回答完停在 ``（相當於 end_turn）、要工具時停在 `<\|tool_response>`（相當於 tool_use）；Qwen 兩種都停在 `<\|im_end\|>`。另外還有步數上限、token 上限、使用者中斷 | §5 什麼時候停 | | 模型分頁 | 參數量拆解、層數與層的種類、注意力頭數、脈絡上限、模型讀得懂什麼（文字、圖片、聲音、影片）、MTP 草稿模型、解碼設定的說明、每一步的 prefill 時間與 tok/s（開 MTP 時加上草稿收下幾個、解碼 forward 幾次），tokenizer 全部特殊 token 的 id 與用途（Gemma 24 個、Qwen 33 個），以及 `models/` 裡這個模型資料夾的每個檔案：大小、用途、在這個模型裡裝了什麼、關鍵欄位的值 | §1、§7 Agent 是系統 | -| 運作說明頁 | 一輪對話的 18 個階段（包括圖片怎麼變成 token、MTP 怎麼猜、怎麼驗證，以及模型可以交給誰跑），每段附上做這件事的程式碼片段和行號，點了會在 GitHub 上打開那幾行，有中文和英文兩版。放在 GitHub Pages，行號在產生頁面時從原始碼查出來，連結指向那個 commit；每次推上 main 都會重新產生，程式改了會跟著更新 | 全部 | +| 運作說明頁 | 一輪對話的 18 個階段（包括圖片怎麼變成 token、MTP 怎麼猜、怎麼驗證，以及模型可以交給誰跑），內建 harness 和 pi 做法不同的地方各有頁籤，每段附上做這件事的程式碼片段和行號，點了會在 GitHub 上打開那幾行，有中文和英文兩版。放在 GitHub Pages，行號在產生頁面時從原始碼查出來，連結指向那個 commit；每次推上 main 都會重新產生，程式改了會跟著更新 | 全部 | ## 快速開始 @@ -43,6 +44,7 @@ - [uv](https://docs.astral.sh/uv/) 0.13.4 以上（會自動裝好 Python 3.12）。更舊的版本還信任 eMudhra 那批外洩的 emSign 憑證（[公告](https://discuss.python.org/t/109576)），所以 `uv sync` 會拒絕執行；用 `uv self update`（或 `brew upgrade uv`）更新 - 網路：第一次下載模型，以及 Wikipedia 工具 - 選用：[Docker](https://docs.docker.com/get-started/get-docker/)（Docker Desktop、OrbStack 或 Colima），只有讓模型寫程式、或用 vLLM (Docker) 跑模型時才需要 +- 選用：[pi](https://pi.dev) 和 Node.js 22.19 以上，只有要把迴圈交給 pi 時才需要（「[pi 當 harness](#pi-當-harness)」） ```bash uv sync @@ -50,19 +52,21 @@ uv run hf download google/gemma-4-E2B-it --revision 3e22461f65e89153144f8adb70e3 uv run hf download Qwen/Qwen3.5-0.8B --revision 2fc06364715b967f1860aea9cf38778875588b17 --cache-dir models # 約 1.8 GB uv run hf download google/gemma-4-E2B-it-assistant --revision 2d874ef7d29f9a30599a1e4b3c1cbc9595f005df --cache-dir models # MTP 草稿模型，約 190 MB；跳過的話，第一次打開 MTP 時會自動下載 uv run streamlit run streamlit_app.py # 預設先載入 Gemma；MODEL_ID=Qwen/Qwen3.5-0.8B 可以先載入 Qwen -docker build -t agent-loop-sandbox harness/sandbox # 選用：「讓模型寫程式」用的沙箱（約 400 MB）；harness/sandbox/Dockerfile 改了要再建一次 +docker build -t agent-loop-sandbox agent/sandbox # 選用：「讓模型寫程式」用的沙箱（約 400 MB）；agent/sandbox/Dockerfile 改了要再建一次 ``` 瀏覽器會打開 http://localhost:8501 ，語言跟著瀏覽器（中文的瀏覽器看到中文，其他看到英文），右上角的「中文 | English」可以切換，下次打開會用你選過的那個。第一次載入 Gemma 加暖機大約 10–20 秒，Qwen 幾秒；之後重新整理頁面不會再載入。側欄最上面的「模型」選單可以換模型，換了會開新對話，解碼設定也換成新模型的建議值，MTP 回到關閉。側欄的「MTP 推測解碼」第一次打開時才載入草稿模型（不到 1 秒），之後一直留在記憶體裡。 -側欄的「讓模型寫程式」預設關閉，關著的時候（也沒選 vLLM (Docker)）網頁完全不會去找 Docker。打開之後才檢查 Docker 和 `agent-loop-sandbox` image，缺了哪一個，或 image 是用舊版的 `harness/sandbox/Dockerfile` 建的，就在側欄說明怎麼安裝 Docker 或建 image；兩個都在，模型才拿得到 `run_python`，要寫 Python 的範例（打開就會出現）也才按得下去。你附上的檔案存在 `~/.cache/agent-loop-visualizer/uploads/` 底下，每段對話一個資料夾（按「新對話」就刪掉），沙箱裡看到的是唯讀的 `/data`。 +側欄的「讓模型寫程式」預設關閉，關著的時候（也沒選 vLLM (Docker)）網頁完全不會去找 Docker。打開之後才檢查 Docker 和 `agent-loop-sandbox` image，缺了哪一個，或 image 是用舊版的 `agent/sandbox/Dockerfile` 建的，就在側欄說明怎麼安裝 Docker 或建 image；兩個都在，模型才拿得到 `run_python`，要寫 Python 的範例（打開就會出現）也才按得下去。你附上的檔案存在 `~/.cache/agent-loop-visualizer/uploads/` 底下，每段對話一個資料夾（按「新對話」就刪掉），沙箱裡看到的是唯讀的 `/data`。 -模型下載到專案資料夾裡的 `models/`（git 會忽略它），不是其他專案共用的 `~/.cache/huggingface/hub`；刪掉這個資料夾就能把空間拿回來。每個模型都照固定的 revision 下載，也就是 `harness/families/__init__.py` 裡 `REVISIONS` 的 commit（上面的 `--revision`），所以不管什麼時候下載，拿到的都是這一頁寫作和測試時用的檔案，在每種作業系統上都一樣。想放在別的地方，就把 `AGENT_LOOP_MODELS` 設成那個資料夾（上面的下載指令的 `--cache-dir` 也改成它）；`AGENT_LOOP_MODELS=$HOME/.cache/huggingface/hub` 會用之前已經下載在那裡的模型。`HF_HUB_CACHE` 不會改變位置，因為它常是整台電腦所有專案共用的設定。 +模型下載到專案資料夾裡的 `models/`（git 會忽略它），不是其他專案共用的 `~/.cache/huggingface/hub`；刪掉這個資料夾就能把空間拿回來。每個模型都照固定的 revision 下載，也就是 `agent/model_families/__init__.py` 裡 `REVISIONS` 的 commit（上面的 `--revision`），所以不管什麼時候下載，拿到的都是這一頁寫作和測試時用的檔案，在每種作業系統上都一樣。想放在別的地方，就把 `AGENT_LOOP_MODELS` 設成那個資料夾（上面的下載指令的 `--cache-dir` 也改成它）；`AGENT_LOOP_MODELS=$HOME/.cache/huggingface/hub` 會用之前已經下載在那裡的模型。`HF_HUB_CACHE` 不會改變位置，因為它常是整台電腦所有專案共用的設定。 想改用 vLLM 跑模型，可以在側欄的「用什麼跑模型」選 vLLM (Docker)，網頁會替你在 Docker 裡啟動一台 vLLM 伺服器（見[由網頁在 Docker 裡啟動 vLLM](#由網頁在-docker-裡啟動-vllm)）；也可以讓網頁連到你自己的伺服器（見[在 vLLM 伺服器上跑模型](#在-vllm-伺服器上跑模型)）。 > 下載時出現 401，可是模型明明是公開的？多半是本機存的 Hugging Face token 過期了。重新登入（`uv run hf auth login`），或加上 `HF_HUB_DISABLE_IMPLICIT_TOKEN=1` 不送 token。 +> 在 `agent/` 還叫 `harness/` 的時候就 clone 了？Python 在那裡寫的 `__pycache__` 資料夾 git 會忽略，所以 pull 之後它們還留著，看起來像一個空的 `harness/`。沒有程式會 import 它；`git clean -fdX harness` 會刪掉它，而且只刪其中 git 忽略的檔案。 + ## 可以試試看的實驗 1. **只是聊天機器人**：在側欄關掉「使用工具」，問「台北現在幾點？」。模型看不到任何工具，只能猜或承認不知道。打開工具再問一次，比較兩次的脈絡分頁。 @@ -81,23 +85,28 @@ docker build -t agent-loop-sandbox harness/sandbox # 選用：「讓模型寫 ## 架構 -專案分成兩塊。`harness/` 跑一輪：agent 迴圈、工具、模型家族，以及跑模型的那一層。`app/` 是你看到的東西：網頁、它從 harness 的事件畫出來的 HTML，還有運作說明。兩塊的依賴只有一個方向：`app/` 建在 `harness/` 上面，`harness/` 裡沒有任何東西 import `app/` 或 Streamlit，所以迴圈不需要網頁就能跑、也能測。 +專案分成兩塊。`agent/` 跑一輪：跑 agent 迴圈的 harness、工具、模型家族，以及跑模型的那一層。`app/` 是你看到的東西：網頁、它從 agent 的事件畫出來的 HTML，還有運作說明。兩塊的依賴只有一個方向：`app/` 建在 `agent/` 上面，`agent/` 裡沒有任何東西 import `app/` 或 Streamlit（`test_layout.py` 會檢查），所以迴圈不需要網頁就能跑、也能測。 ``` streamlit_app.py Streamlit 從這裡開始：每次重跑都執行 app/page.py 的 main() -app/ 你看到的東西，站在 harness 上面 +app/ 你看到的東西，站在 agent 上面 page.py 版面、session state、模型選單、語言切換、背景 thread；執行中每 0.5 秒重畫一次 render.py 把事件列表轉成 HTML：迴圈圖、step 卡片、token 熱度與浮窗、脈絡檢視 guide.py 「運作說明」頁：中英文內容、行號查詢，以及產生放上 GitHub Pages 的靜態 HTML assets/ 網頁的樣式和運作說明的 script；logo.svg；examples/ 是範例附上的檔案，也是輸入框上方的範例檔案 -harness/ 跑一輪的東西；從不 import Streamlit - agent.py 迴圈：組 prompt → 生成 → 解析 → 驗證 →（先問你）→ 執行工具 → 接回脈絡 → 判斷要不要停 - backends/ 跑模型的那一層 +agent/ 跑一輪的東西：harness、模型、跑模型的那一層、工具；從不 import Streamlit + harnesses/ 跑迴圈的那一層 + harness.py 網頁透過它跑一輪的 Harness：run、stop、decide，以及每個 harness 送出的事件 + simple_harness.py 這個專案的迴圈：組 prompt → 生成 → 解析 → 驗證 →（先問你）→ 執行工具 → 接回脈絡 → 判斷要不要停 + pi_harness.py pi 當 harness：用 RPC 模式跑 pi，把它的 JSON 紀錄翻成網頁的事件 + pi_extension.ts 網頁讓 pi 載入的 extension：回報 pi 送出的每個請求、註冊這個網頁的模型、在 pi 執行工具前先問你 + model_endpoint.py 把這個網頁的模型當成 127.0.0.1 上相容 OpenAI 的端點提供給 pi：render、encode、生成，再把 token 傳回 pi + model_backends/ 跑模型的那一層 backend.py harness 和畫面透過它使用模型的 Backend：render、encode、generate，以及 token 事件的欄位 simple_backend.py 本機推論，自己寫的解碼迴圈；MTP 的草稿與驗證也在這裡 vllm_backend.py 在 vLLM 伺服器上跑模型（VLLM_URL）：prompt 在這裡 render、切 token，token id 送給伺服器，它對每個 token 說得出的都傳回來；伺服器加了對應的參數時，還有媒體、sampling mask 和 MTP 的猜測 vllm_docker.py 側欄選 vLLM (Docker)時在 Docker 裡跑 vLLM：用選的模型 docker compose up、等它能回應；換回時 docker compose down - families/ 跟著模型走的東西：每個模型家族怎麼寫回合、思考、工具呼叫，哪些 token 會停、各是什麼意思 + model_families/ 跟著模型走的東西：每個模型家族怎麼寫回合、思考、工具呼叫，哪些 token 會停、各是什麼意思 __init__.py Family 的定義、模型選單 MODELS、MTP 草稿模型 DRAFTERS、依 config.json 的 model_type 找家族 gemma4.py Gemma 4：輸出的 parser 與 24 個特殊 token 的說明 qwen3_5.py Qwen3.5：輸出的 parser 與 33 個特殊 token 的說明 @@ -113,11 +122,11 @@ tests/ pytest 測試，見下面「測試」 models/ 下載的模型（git 會忽略它；AGENT_LOOP_MODELS 可以換位置） ``` -agent 在背景 thread 執行，每走到一個階段就 append 一個事件；UI 只負責把事件列表畫出來。所以按「停止」或「允許」時 Streamlit 重跑 script，背景的迴圈不會被打斷。 +agent 在背景 thread 執行，每走到一個階段就 append 一個事件；UI 只負責把事件列表畫出來。所以按「停止」或「允許」時 Streamlit 重跑 script，背景的迴圈不會被打斷。網頁只透過 `Harness`（`harnesses/harness.py`）跑迴圈：`run()`、`stop()`、`decide()`，以及它送出的事件。`SimpleHarness` 是這個專案自己的，`PiHarness` 則把迴圈交給 pi（「[pi 當 harness](#pi-當-harness)」）。 「運作說明」是一個靜態頁：`uv run python -m app.guide _site` 會寫出 `_site/zh/index.html` 和 `_site/en/index.html`（樣式和一小段 script 都內嵌在裡面，可以直接用瀏覽器打開），以及 `_site/index.html`：它把讀者送到上次在頁面右上角選過的語言，沒選過就照瀏覽器的語言。推上 main 時，`.github/workflows/pages.yml` 先跑 `tests/test_guide.py`，再做同一件事並發佈到 GitHub Pages；測試沒過的話，網站上維持原來那一版。 -一次執行的事件依序是：`run_start`，然後每一步 `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`，有工具呼叫時再加上 `tool_call` →（`approval_request` → `approval`）→（真的執行工具時的 `tool_start`）→ `tool_result`，最後是 `run_end`；出錯時它前面還有一個 `error`：`message` 是例外的訊息，backend 的錯誤（例如 vLLM 伺服器的）另外帶著兩種語言的版本（`text`），畫面顯示目前語言的那一個。`context` 帶的是模型實際讀到的 prompt（每個附件都展開成它的 token），以及每種媒體佔了幾個 token（`media_tokens`）。`token` 帶著這個 token，以及 backend 知道的話，它的機率、排名、前 5 名候選和 MTP 草稿模型猜了什麼（`backend.TOKEN_DETAILS`）：`simple_backend.py` 全都知道；backend 沒有回報的，畫面就不畫，不會拿沒量過的數字來填。等這一步結束才知道哪些 token 是 MTP 猜中的 backend（vLLM 伺服器），把它們放在 `model_end` 的 MTP 統計裡（`kept_at`、`turned_down_at`），畫面到那時才標出來。`tool_result` 帶著模型讀的觀察，另外還有工具做給你看的圖（`images`：`run_python` 畫的圖的路徑），模型讀不到。 +一次執行的事件依序是：`run_start`，然後每一步 `step_start` → `context` → `prefill` → `token` ×N → `model_end` → `parsed`，有工具呼叫時再加上 `tool_call` →（`approval_request` → `approval`）→（真的執行工具時的 `tool_start`）→ `tool_result`，最後是 `run_end`；出錯時它前面還有一個 `error`：`message` 是例外的訊息，backend 的錯誤（例如 vLLM 伺服器的）另外帶著兩種語言的版本（`text`），畫面顯示目前語言的那一個。`context` 帶的是模型實際讀到的 prompt（每個附件都展開成它的 token），以及每種媒體佔了幾個 token（`media_tokens`）。`token` 帶著這個 token，以及 backend 知道的話，它的機率、排名、前 5 名候選和 MTP 草稿模型猜了什麼（`backend.TOKEN_DETAILS`）：`simple_backend.py` 全都知道；backend 沒有回報的，畫面就不畫，不會拿沒量過的數字來填。等這一步結束才知道哪些 token 是 MTP 猜中的 backend（vLLM 伺服器），把它們放在 `model_end` 的 MTP 統計裡（`kept_at`、`turned_down_at`），畫面到那時才標出來。`tool_result` 帶著模型讀的觀察，另外還有工具做給你看的圖（`images`：`run_python` 畫的圖的路徑），模型讀不到。`run_start` 說這一輪是哪個 harness 跑的（`harness`：`simple` 或 `pi`）。pi 跑的 step 先有 `request`，也就是 pi 送給模型的請求，寫成 JSON。pi 問這個網頁的模型時，那一步的 `context`、`prefill`、`token` 和 `model_end` 由模型端點送出，和內建 harness 一樣；pi 問 provider 的模型時，沒有 family，也沒有這四個，改送 `delta`，模型邊寫邊傳回來的一段輸出和它的 channel（思考、回覆、工具呼叫），`model_end` 帶的則是 provider 報的 token 數（`usage`）。 ### 兩種語言 @@ -161,11 +170,11 @@ chat template 在每個檔案的位置寫一個佔位 token（Gemma 是 `<|image ### 在 Docker 沙箱裡寫程式 -打開「讓模型寫程式」而且 Docker 準備好時，模型多一個工具 `run_python`。每次呼叫都從 `harness/sandbox/Dockerfile` 建的 image 開一個新容器：沒有網路，檔案系統唯讀、只有一小塊 `/tmp` 能寫，記憶體 512 MB、一顆 CPU、最多 64 個 process，拿掉所有 Linux capability，用你自己的使用者執行、絕不用 root（在 Linux 上，掛進去的資料夾保留原本的權限，換成別的使用者可能進不去），這段對話的資料夾唯讀掛在 `/data`；程式從 stdin 送進去，跑完容器就刪掉。超過 30 秒就停掉；結束代碼不是 0、記憶體用太多或什麼都沒印（也沒畫圖），都用這一輪的語言當成錯誤回給模型，附上 traceback，模型下一步可以改。 +打開「讓模型寫程式」而且 Docker 準備好時，模型多一個工具 `run_python`。每次呼叫都從 `agent/sandbox/Dockerfile` 建的 image 開一個新容器：沒有網路，檔案系統唯讀、只有一小塊 `/tmp` 能寫，記憶體 512 MB、一顆 CPU、最多 64 個 process，拿掉所有 Linux capability，用你自己的使用者執行、絕不用 root（在 Linux 上，掛進去的資料夾保留原本的權限，換成別的使用者可能進不去），這段對話的資料夾唯讀掛在 `/data`；程式從 stdin 送進去，跑完容器就刪掉。超過 30 秒就停掉；結束代碼不是 0、記憶體用太多或什麼都沒印（也沒畫圖），都用這一輪的語言當成錯誤回給模型，附上 traceback，模型下一步可以改。 -容器寫不進任何頁面讀得到的地方，matplotlib 畫的圖還是回得來：image 把 matplotlib 的 backend 設成 `harness/sandbox/agent_loop_figures.py`，它在 `plt.show()` 或程式結束時把每張圖轉成 PNG，程式結束前接在印出來的東西後面，前面加一行 `run_python` 傳進去的記號。`run_python` 把它們從模型要讀的輸出切掉（最多四張），存進這段對話的資料夾，叫 `figure-1.png`、`figure-2.png`…，和觀察分開回傳：模型只讀到 `{"stdout": …, "figures": ["/data/figure-1.png"]}`，頁面把圖畫在聊天區和那一步底下。中文的標籤用 image 為此裝的文泉驛微米黑畫。載入 matplotlib 讓一次呼叫慢了約半秒（0.8 秒，純 Python 是 0.2 秒）。 +容器寫不進任何頁面讀得到的地方，matplotlib 畫的圖還是回得來：image 把 matplotlib 的 backend 設成 `agent/sandbox/agent_loop_figures.py`，它在 `plt.show()` 或程式結束時把每張圖轉成 PNG，程式結束前接在印出來的東西後面，前面加一行 `run_python` 傳進去的記號。`run_python` 把它們從模型要讀的輸出切掉（最多四張），存進這段對話的資料夾，叫 `figure-1.png`、`figure-2.png`…，和觀察分開回傳：模型只讀到 `{"stdout": …, "figures": ["/data/figure-1.png"]}`，頁面把圖畫在聊天區和那一步底下。中文的標籤用 image 為此裝的文泉驛微米黑畫。載入 matplotlib 讓一次呼叫慢了約半秒（0.8 秒，純 Python 是 0.2 秒）。 -image 先用 wheel 加 hash 裝好 pip 26.2.2，再用它裝其他套件：更舊的 pip 也信任它自己帶的那份 certifi，裡面還有外洩的 emSign 憑證。頁面會讀 image 的 `agent-loop-sandbox.version` label，比 `harness/sandbox/Dockerfile` 舊就在側欄請你重建，不提供 `run_python`。 +image 先用 wheel 加 hash 裝好 pip 26.2.2，再用它裝其他套件：更舊的 pip 也信任它自己帶的那份 certifi，裡面還有外洩的 emSign 憑證。頁面會讀 image 的 `agent-loop-sandbox.version` label，比 `agent/sandbox/Dockerfile` 舊就在側欄請你重建，不提供 `run_python`。 模型看不到資料檔的內容，所以 harness 在使用者訊息後面加一段說明：`run_python` 在哪裡找得到每個檔案，以及文字檔的第一行（CSV 就是欄位名稱）。少了這一行，小模型寫程式時會亂猜欄位，例如不存在的 `Date`，然後出錯；多給幾列資料也不行，Gemma 會把那幾列抄進程式，只用它們算答案。image 裡有 `tabulate`，因為小模型很愛用的 `DataFrame.to_markdown()` 需要它。 @@ -218,7 +227,7 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py ### 由網頁在 Docker 裡啟動 vLLM -沒設 `VLLM_URL` 時，側欄的「用什麼跑模型」可以在 transformers 和 vLLM (Docker)之間切換。選 vLLM 時，網頁先放掉在這裡載入的模型（`release_model()` 清掉快取、把記憶體還給系統；在 CUDA 上連 cuBLAS 的工作區也放掉，不然整個模型的記憶體都還不回去。Streamlit 的 fast reruns 關掉了，所以模型載入到一半時切換的設定，要等載入完才生效），接著 `harness/backends/vllm_docker.py` 把還沒下載的模型下載到 `models/`，用 `docker compose up -d` 啟動 `docker-compose/` 裡的伺服器，等待時顯示 Docker 和 vLLM 印出的內容。換回 transformers 或結束這個程式（Ctrl+C）時執行 `docker compose down`，所以兩邊不會同時佔著模型。「用什麼跑模型」也和記憶體裡的模型一樣，是所有分頁共用的一個選擇：停在 transformers 的分頁，會跟著另一個分頁換到 vLLM 或換回來；新開的分頁或重新整理，也會接上正在跑的伺服器和它的模型，不會在這裡再載入一份。在 vLLM 上換模型會再跑一次 `docker compose up`，它會換掉容器；伺服器已經在跑時，重跑頁面不會再啟動一次。啟動失敗（沒有 Docker、Docker 沒在執行、記憶體不夠）時，頁面用目前的語言說明原因，要按「再試一次」才會重試；換回 transformers 的切換鈕一直留在側欄。 +沒設 `VLLM_URL` 時，側欄的「用什麼跑模型」可以在 transformers 和 vLLM (Docker)之間切換。選 vLLM 時，網頁先放掉在這裡載入的模型（`release_model()` 清掉快取、把記憶體還給系統；在 CUDA 上連 cuBLAS 的工作區也放掉，不然整個模型的記憶體都還不回去。Streamlit 的 fast reruns 關掉了，所以模型載入到一半時切換的設定，要等載入完才生效），接著 `agent/model_backends/vllm_docker.py` 把還沒下載的模型下載到 `models/`，用 `docker compose up -d` 啟動 `docker-compose/` 裡的伺服器，等待時顯示 Docker 和 vLLM 印出的內容。換回 transformers 或結束這個程式（Ctrl+C）時執行 `docker compose down`，所以兩邊不會同時佔著模型。「用什麼跑模型」也和記憶體裡的模型一樣，是所有分頁共用的一個選擇：停在 transformers 的分頁，會跟著另一個分頁換到 vLLM 或換回來；新開的分頁或重新整理，也會接上正在跑的伺服器和它的模型，不會在這裡再載入一份。在 vLLM 上換模型會再跑一次 `docker compose up`，它會換掉容器；伺服器已經在跑時，重跑頁面不會再啟動一次。啟動失敗（沒有 Docker、Docker 沒在執行、記憶體不夠）時，頁面用目前的語言說明原因，要按「再試一次」才會重試；換回 transformers 的切換鈕一直留在側欄。 ```bash # 自己啟動同一台伺服器（VLLM_MODEL 和 VLLM_REVISION 預設是 Qwen3.5 0.8B 和它固定的 commit）；權重要先在 models/ 裡： @@ -228,15 +237,37 @@ VLLM_URL=http://localhost:8000 uv run streamlit run streamlit_app.py ``` - **在哪裡跑。** Docker 有 nvidia runtime 時，再疊上 `compose.gpu.yaml`，用 vLLM 官方的 CUDA image（`vllm/vllm-openai:v0.32.0`）在 GPU 上跑；其他情況（Mac 也是）由 `compose.yaml` 在 Docker 的 Linux 虛擬機裡跑 vLLM 的 CPU image。`AGENT_LOOP_VLLM_DEVICE=cpu` 或 `gpu` 可以直接指定，`VLLM_PORT` 可以換埠號（8000）。在 GPU 上 KV cache 和 CPU 上一樣是 2 GiB（`--kv-cache-memory-bytes`、`VLLM_KV_CACHE_MEMORY_BYTES`），不是 GPU 記憶體的 92%；vLLM 啟動時也只要求 30% 的記憶體是空的（`--gpu-memory-utilization`、`VLLM_GPU_MEMORY_UTILIZATION`）：照它預設的 0.92，小張的顯示卡旁邊還有網頁自己的 CUDA context 時會拒絕啟動，和整台電腦共用記憶體的 GPU（例如 DGX Spark）也是。 -- **MTP。** 在 GPU 上，伺服器預設會推測解碼：Gemma 4 用它的草稿模型，Qwen3.5 用它自己 checkpoint 裡的 MTP 層（`families.VLLM_SPECULATIVE`，用 `VLLM_SPECULATIVE_CONFIG` 交給 Compose），每一步收下的猜測都會加上底線。側欄的 MTP 開關可以關掉或再打開，會用另一種方式重新啟動伺服器，每個分頁也都會跟著。vLLM 0.32 的 CPU 版 image 兩個模型都不能推測解碼，所以在 CPU 上開關不能按，也會說明原因。手動啟動 GPU 版時不開。 +- **MTP。** 在 GPU 上，伺服器預設會推測解碼：Gemma 4 用它的草稿模型，Qwen3.5 用它自己 checkpoint 裡的 MTP 層（`model_families.VLLM_SPECULATIVE`，用 `VLLM_SPECULATIVE_CONFIG` 交給 Compose），每一步收下的猜測都會加上底線。側欄的 MTP 開關可以關掉或再打開，會用另一種方式重新啟動伺服器，每個分頁也都會跟著。vLLM 0.32 的 CPU 版 image 兩個模型都不能推測解碼，所以在 CPU 上開關不能按，也會說明原因。手動啟動 GPU 版時不開。 - **伺服器讀什麼。** 它自己不下載：`models/` 唯讀掛進去、設了 `HF_HUB_OFFLINE=1`，所以載入的正是 `REVISIONS` 裡的那個 commit，和網頁一樣。它跟網頁一樣只聽 127.0.0.1。啟動時加了 `--enable-scale-out`，所以看得到每個 token 的排名，伺服器也收圖片、聲音和影片；`--logprobs-mode` 維持 vLLM 預設的 `raw_logprobs`，浮窗顯示的是模型自己的機率和前 5 名。vLLM 編譯的結果存在一個 Docker volume，所以一個模型只有第一次啟動要編譯。 - **CPU 版的 image**（`docker-compose/vllm-cpu/`）是 `vllm/vllm-openai-cpu:v0.32.0` 加上兩個修正（`patch_vllm.py`）：vLLM 自己的 #61992，晚了 0.32.0 一天才進去，沒有它的話每個模型在 CPU 上的第一個請求都會失敗；另一個是我們自己的，讓 CPU 上線性層的輸入在記憶體裡是連續的，沒有它的話 Gemma 4 聽的第一段聲音就會讓 engine 當掉。在 CPU 上，`compose.yaml` 把 KV cache 設成 2 GiB（`VLLM_CPU_KVCACHE_SPACE`；不設的話 vLLM 要 Docker 記憶體的 92%），啟動時不量圖片編碼器要多少記憶體（在 CPU 上要超過 10 分鐘），並讓一步最多可以跑 30 分鐘，而不是 vLLM 預設的 5 分鐘就放棄 engine。 - **在 DGX Spark 上**（NVIDIA GB10，GPU 和整台電腦共用 128 GB 記憶體）：在 GPU 上，Qwen3.5 0.8B 啟動約 2 分鐘、解碼約 113 tok/s，Gemma 4 E2B 啟動約 3 分鐘、解碼約 41 tok/s，兩個模型都通過接真的伺服器的測試，包括 Gemma 讀得懂的每種媒體。照 vLLM 預設的 `gpu_memory_utilization`（0.92），伺服器在這台電腦上不肯啟動：vLLM 把整台電腦可用的記憶體當成 GPU 空著的記憶體，大約是 122 GiB 裡的 108 GiB。在同一台電腦的 CPU 上，Qwen3.5 0.8B 啟動約 1.3 分鐘、解碼約 23 tok/s。 - **在 Mac 上**（Apple M5，給 Docker 10 個核心）：Qwen3.5 0.8B 啟動約 1.5 分鐘，佔 Docker 約 9 GB 記憶體，解碼約 10 tok/s。Gemma 4 E2B 要的記憶體比 Docker Desktop 預設的多：16 GB 時載入到一半會被停掉（頁面會說明，也說怎麼給 Docker 更多記憶體），24 GB 時 3 到 5 分鐘啟動、解碼約 4.6 tok/s，但載入時問伺服器收哪些媒體，要在 CPU 上跑它的圖片、聲音和影片編碼器，第一次要花大約 10 分鐘。 +### pi 當 harness + +網頁也可以把迴圈交給 [pi](https://pi.dev)，一個有自己的迴圈、system prompt 和工具（read、bash、edit、write、grep、find、ls）的 coding agent。在側欄最上面的「用什麼跑迴圈」選 pi，再在「pi 問的模型」選它問哪個模型，網頁就用同一張迴圈圖、同樣的 step 卡片和事件列表，畫出 pi 做了什麼。每一步的脈絡段顯示 pi 送給模型的請求（它的 system prompt、工具和對話，寫成 JSON），這一步新加的部分另外標出來。 + +- **這個網頁的模型**（預設）：Gemma 4 E2B 或 Qwen3.5 0.8B，跑在 transformers 或 vLLM 上，用和內建 harness 一樣的模型選單和「用什麼跑模型」來選。pi 只會用 HTTP 問模型，所以 `agent/harnesses/model_endpoint.py` 把記憶體裡的模型當成 127.0.0.1 上一個相容 OpenAI 的 chat completions 端點，用一個 token 擋住別的程式，pi 的 extension 再把它註冊成 provider `agent-loop`。每個請求都用模型的 chat template render、encode、生成，和內建 harness 的一步一樣，事件進到 pi 的那一步：「脈絡」分頁顯示模型讀到的字串（下面收著 pi 送出的請求），每個 token 的機率和候選也都看得到。回覆一邊生成一邊傳回 pi：思考是 `reasoning_content`，回覆是 `content`，最後是 family 從模型自己的格式解析出來的工具呼叫（`tool_calls`）。解碼設定是側欄的；「思考模式」變成 pi 的思考等級送出，端點再換成 chat template 的思考開關。 +- **pi 自己的模型**：pi 登入過的模型（Claude、GPT、Gemini，或它 `models.json` 裡任何相容的端點），這台電腦不載入模型。API 傳回來的是文字，不是 token，所以沒有 token 的機率、候選和特殊 token，API 那邊怎麼把請求寫成模型讀的字串也沒有人說：一個 step 顯示 pi 的請求、一段段傳回來的文字、provider 報的 token 數（以及其中多少來自 cache），和照 pi 的價目估的費用。 + +```bash +nvm install # 用 nvm 的話：照 .nvmrc 裝好最新的 Node 22 再換過去；nvm use 可能挑到已經裝好、但對 pi 來說太舊的 22。pi 也要裝在它底下，因為 nvm 的全域套件是每個版本各裝各的 +npm install -g @earendil-works/pi-coding-agent # pi 1.4 需要 Node.js 22.19 以上 +pi # 再輸入 /login 登入一次；或在啟動 app 之前設好 API key，例如 ANTHROPIC_API_KEY +PI_COMMAND=/path/to/pi uv run streamlit run streamlit_app.py # pi 不在 PATH 上的話（任何指令都行，npx 也可以） +``` + +- **網頁怎麼跑它。** `agent/harnesses/pi_harness.py` 每個問題都用 RPC 模式啟動 pi（`pi --mode rpc`）：指令寫進它的 stdin，紀錄從它的 stdout 讀出來，一行一個 JSON。它先選好模型和思考等級、關掉 pi 自己的重試（重試會變成另一個 step，所以請求失敗就當成錯誤顯示），再送出問題，每個指令都等 pi 回應了才送下一個；pi 的一個 turn（向模型送一次請求，加上回覆裡要執行的工具）就是一個 step。pi 把對話存在這段對話資料夾裡的 session 檔，下一個問題就接著它；按「新對話」會刪掉。 +- **啟動 pi 時不帶的東西。** 你自己的 extension、skill、prompt 範本和 context 檔（AGENTS.md、CLAUDE.md）都不載入（`--no-extensions` 之類），所以網頁畫的是 pi 本身的迴圈，每台電腦上都一樣；模型和登入資訊還是你的，在 `~/.pi/agent`。 +- **工具。** 側欄選模型看得到 pi 的哪些工具（預設是 read、bash、edit、write，和 pi 一樣）。它們在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：不像 `run_python`，bash 碰得到你碰得到的一切。附上的檔案存在那個資料夾裡，問題裡會寫出檔名；讀得懂圖片的模型，圖片也會跟著問題一起送過去。 +- **Extension。** 網頁啟動的每個 pi 都載入 `agent/harnesses/pi_extension.ts`。每次向模型送出請求前，它把請求當成一則通知送出，在 RPC 模式下就是 stdout 上的一筆 `extension_ui_request`，變成那一步的 `request`；pi 問這個網頁的模型時，它註冊那個模型；執行工具前先問你的也是它（見下）。 +- **執行工具前先問我。** pi 自己不會問；打開時，extension 在每次執行工具前開一個確認對話框。在 RPC 模式下，對話框出現在 stdout，網頁把它顯示成「允許」和「拒絕」，你的回答再從 stdin 送回去；被拒絕的呼叫不會執行，模型讀到的是用這一輪語言寫的拒絕原因。 +- **停止。** 按停止會送給 pi `abort`。pi 自己沒有步數上限，所以「步數上限」由網頁來守：pi 開始超過上限的那個 turn 時，網頁就請它中止。確認一直沒人回答的話，5 分鐘後停掉 pi，和內建 harness 一樣。 +- **測試**用的是 pi 1.4.0：透過照劇本說它的協定的替身（`tests/fake_pi.py`，它也會像 pi 一樣用 HTTP 問模型端點），以及 `RUN_PI_TESTS=1` 時的真 pi，分別指向 127.0.0.1 上一個照劇本回答的模型，和後面接著假模型的模型端點。 + ### 支援更多模型 -harness（`agent.py`）、解碼迴圈（`simple_backend.py`）和畫面（`render.py`）都只寫一次，裡面沒有任何「如果是 Gemma」的分支。每個模型不一樣的地方集中在 `families/` 的一個 `Family` 物件裡： +harness（`harnesses/simple_harness.py`）、解碼迴圈（`simple_backend.py`）和畫面（`render.py`）都只寫一次，裡面沒有任何「如果是 Gemma」的分支。每個模型不一樣的地方集中在 `model_families/` 的一個 `Family` 物件裡： - 讀輸出：`parse`（把輸出切成思考、工具呼叫、回覆）、`thought_prefix`、`channel_switch`（哪些 token 切到思考、工具呼叫、回覆）、`stop_tokens` - 寫回對話：`reasoning_key`（思考存在哪個欄位，chat template 才讀得到）、`text_with_calls`（工具呼叫前的文字留不留） @@ -245,14 +276,14 @@ harness（`agent.py`）、解碼迴圈（`simple_backend.py`）和畫面（`rend 要加模型： -1. 同一個家族的另一個 checkpoint（例如更大的 Gemma 4 或 Qwen3.5）：在 `harness/families/__init__.py` 的 `MODELS` 加一行，`REVISIONS` 也加一行，寫上要下載的 commit。Gemma 4 的其他 checkpoint 也有對應的 MTP 草稿模型（`google/gemma-4-*-it-assistant`），在 `DRAFTERS` 加一行（草稿模型的 commit 也加進 `REVISIONS`），側欄的 MTP 開關就能用。 -2. 新的家族：照 `harness/families/qwen3_5.py` 寫一個模組（輸出的 parser 加上特殊 token 的說明，中英文都要），把它的 `Family` 加進 `FAMILIES`。載入模型時會用 checkpoint `config.json` 裡的 `model_type` 自動找到它。`tests/test_chat_template.py` 會拿真的 tokenizer 和 chat template 檢查新家族：每個特殊 token 都有說明、模板寫進歷史的工具呼叫讀得回來、step 卡片標成「新加入」的剛好是工具結果。 +1. 同一個家族的另一個 checkpoint（例如更大的 Gemma 4 或 Qwen3.5）：在 `agent/model_families/__init__.py` 的 `MODELS` 加一行，`REVISIONS` 也加一行，寫上要下載的 commit。Gemma 4 的其他 checkpoint 也有對應的 MTP 草稿模型（`google/gemma-4-*-it-assistant`），在 `DRAFTERS` 加一行（草稿模型的 commit 也加進 `REVISIONS`），側欄的 MTP 開關就能用。 +2. 新的家族：照 `agent/model_families/qwen3_5.py` 寫一個模組（輸出的 parser 加上特殊 token 的說明，中英文都要），把它的 `Family` 加進 `FAMILIES`。載入模型時會用 checkpoint `config.json` 裡的 `model_type` 自動找到它。`tests/test_chat_template.py` 會拿真的 tokenizer 和 chat template 檢查新家族：每個特殊 token 都有說明、模板寫進歷史的工具呼叫讀得回來、step 卡片標成「新加入」的剛好是工具結果。 也可以提出需求：在 vLLM 伺服器上選了網頁不認得的家族的模型，網頁會附上一個已經填好的 GitHub issue；其他想看看的模型，也歡迎到 https://github.com/cyyeh/agent-loop-visualizer/issues 提。 ### 兩個模型的格式 -Gemma 4 的字串用 `<|"|>` 這個特殊 token 包起來，key 也不加引號，所以不是 JSON，`harness/families/gemma4.py` 有自己的小 parser： +Gemma 4 的字串用 `<|"|>` 這個特殊 token 包起來，key 也不加引號，所以不是 JSON，`agent/model_families/gemma4.py` 有自己的小 parser： ``` <|tool_call>call:get_current_time{timezone:<|"|>Asia/Taipei<|"|>}<|tool_response> @@ -278,33 +309,41 @@ Qwen 不管回答完或要用工具都停在 `<|im_end|>`，所以 harness 是 ```bash uv run pytest # 約 40 秒，不載入模型權重、不連網路 +uv run ruff check # lint：可能的 bug、沒用到的 import、import 的順序（加 --fix 會排好） +uv run mypy # 型別：agent/ 和 app/ 每個函式都檢查，tests/ 只檢查有標型別的地方 RUN_MODEL_TESTS=1 uv run pytest tests/test_real_model.py # 載入真的 Gemma（含 MTP 草稿模型）和 Qwen，約 4 分鐘；加 -k Qwen 只跑一個 VLLM_TEST_URL=http://localhost:8000 uv run pytest tests/test_vllm_backend.py -k real_server # 問一台正在跑的 vLLM 伺服器 RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real # 用 Qwen3.5 在 Docker 裡啟動 vLLM，問它一個問題 +RUN_PI_TESTS=1 uv run pytest tests/test_pi_harness.py -k real # 啟動真的 pi（PATH 上的，或 PI_COMMAND），對上 127.0.0.1 上照劇本回答的模型，以及模型端點 ``` -大部分測試用 `tests/fakes.py` 的假模型：它照劇本一個 token 一個 token 吐出預先寫好的輸出，和 `SimpleBackend` 一樣是一個 `Backend`（`backend.py`），所以 harness 和網頁可以從頭跑到尾。 +大部分測試用 `tests/fakes.py` 的假模型：它照劇本一個 token 一個 token 吐出預先寫好的輸出，和 `SimpleBackend` 一樣是一個 `Backend`（`backend.py`），所以 harness 和網頁可以從頭跑到尾。需要 pi 的地方，測試啟動的是 `tests/fake_pi.py`，它照劇本說 pi 1.4 的 RPC 協定，所以不需要 Node，也不需要模型。 網頁收的每一種檔案格式都有測試：`tests/samples.py` 用範例的檔案做出每種格式的小檔案（收據的每種圖片格式、錄音的每種聲音格式、滾動的球的每種影片格式、銷售資料的每種資料格式）。`media.py` 多收了一種格式卻沒有對應的範例檔，`test_media.py` 就會失敗；`test_real_model.py` 拿每一種格式問兩個模型。需要 Docker 的測試只在 Docker 和沙箱 image 都在時才跑。 +每個 pull request 和每次推上 main，`.github/workflows/checks.yml` 都會在 Ubuntu 上跑 `ruff check`、`mypy` 和 `uv run pytest`；需要模型、沙箱 image、vLLM 伺服器或 pi 的測試在那裡會跳過。同時它也像 `pages.yml` 在 main 上那樣，只裝 pygments 跑 `tests/test_guide.py` 並產生運作說明，所以 pull request 讓這個頁面的程式 import 了 torch 或 app 其他的套件，合併前就會失敗，不會等到發佈頁面時才發現。 + 運作說明會引用程式碼，`tests/guide_excerpts.txt` 存著它引用的每一段程式，不含行號。改到被引用的程式時，`test_guide.py` 會失敗，並列出引用它的段落：重讀那幾段，把不再正確的說明改掉，再用 `UPDATE_GUIDE_SNAPSHOT=1 uv run pytest tests/test_guide.py` 重寫這個檔案。 | 檔案 | 測什麼 | |---|---| | `test_gemma4.py` | Gemma 4 的 parser：思考、工具呼叫、回覆的切分；`<\|"\|>` 字串、巢狀參數、各種格式錯誤，兩種語言都有 | | `test_qwen3_5.py` | Qwen3.5 的 parser：XML 工具呼叫、照 schema 轉型別、思考、各種格式錯誤；用 Qwen 格式跑完整個 harness | -| `test_families.py` | 每個家族都說明了它用到的 token、中英文說明列出同樣的 token、`model_type` 找得到家族；token 顯示成它代表的文字，只是一個字一部分的 byte 顯示成 `<0xF0>`；每個家族都用中英文說明它 checkpoint 裡的檔案，兩種語言提到同樣的欄位和 token；每個模型和草稿模型都照固定的 commit 下載；vLLM 替清單裡的模型推測解碼時，用的是它 checkpoint 裡的 MTP 層，或 transformers 用的那個草稿模型 | +| `test_model_families.py` | 每個家族都說明了它用到的 token、中英文說明列出同樣的 token、`model_type` 找得到家族；token 顯示成它代表的文字，只是一個字一部分的 byte 顯示成 `<0xF0>`；每個家族都用中英文說明它 checkpoint 裡的檔案，兩種語言提到同樣的欄位和 token；每個模型和草稿模型都照固定的 commit 下載；vLLM 替清單裡的模型推測解碼時，用的是它 checkpoint 裡的 MTP 層，或 transformers 用的那個草稿模型 | | `test_tools.py` | schema 驗證與預設值、計算機的 AST 白名單（拒絕 `__import__` 之類）、時區、Wikipedia（HTTP 用假的）；`run_python` 鎖住的 `docker run`、各種錯誤（traceback、沒有輸出、記憶體太多、跑太久、沒有 image）、Docker 沒準備好或 image 過舊時缺什麼，都有中英文兩種；記號後面的圖存檔時不蓋掉現有檔案、只收 PNG、最多四張、程式出錯就不收；image 的版本 label 和用 hash 安裝的 pip；有 Docker 時，真的沙箱讀得到 `/data` 但改不了、連不上網路，讀得了每一種資料格式，送得回中文標籤畫得出來的圖，pip（和它的 certifi）也是修正 emSign 之後的版本 | | `test_simple_backend.py` | `SimpleBackend` 有 `Backend` 的每個方法和屬性，它的 token 帶著每個欄位和細節；自己寫的解碼迴圈：KV cache、停止 token、token 上限、中斷、channel 切換、被拆開的中文字元；`encode()` 照順序把每種媒體和家族的設定交給 processor、短影片給它全部畫格；圖片和影片在 prefill 前先跑視覺編碼器，上一個 prompt 的位置偏移不會留下來；greedy 遇到兩個 token 打平時和 `generate()` 一樣拿 id 小的；temperature / top-k / top-p 的候選與機率；MTP 用照劇本猜的假草稿模型：猜測的收下與退回、全部收下時多一個 token、KV cache 退回多少、草稿模型看不到作廢的猜測，以及抽樣時檢查猜測不改變分布；模型照固定的 revision 下載到 `models/`（`AGENT_LOOP_MODELS` 可以換位置，相對路徑從啟動 app 的地方算起；`HF_HUB_CACHE` 不行），在 CUDA 上放掉模型時，先放掉 cuBLAS 的工作區再清快取，模型資料夾的檔案、大小（經過快取的連結）和說明提到的欄位 | -| `test_agent.py` | 假模型是一個 `Backend`，它的 token 帶的欄位和 `SimpleBackend` 的一樣；harness：事件順序、工具結果接回脈絡、驗證錯誤、重複呼叫、結果截斷、每一種停止原因、允許 / 拒絕 / 等待時按停止；附件變成問題裡的 part 和給 `run_python` 的說明、每一輪的媒體都交給 processor、展開後的脈絡仍然只把工具結果標成新加入、`run_python` 拿到這段對話的資料夾、沒有 `run_python` 的一輪；模型讀到圖在哪裡，圖本身只給頁面 | -| `test_render.py` | 一次執行的每一個事件前綴都能用兩種語言畫出來（執行中每 0.5 秒重畫）、HTML 標籤成對、迴圈圖是合法 SVG、模型輸出一律跳脫、特殊 token 的說明浮窗與列表；MTP 的底線、浮窗、統計和草稿模型卡片；一串同樣的特殊 token 畫成一個標籤加個數、截斷的片段不會切開 token、媒體佔了幾個 token、好幾行的程式碼和輸出照行寫出來、這一輪沒提供的工具、一次執行和每一步的圖；模型資料夾卡片：檔案的順序、大小和說明、家族有說明但 checkpoint 沒有的檔案、草稿模型的資料夾、跳脫；backend 沒回報機率、或只回報 log 機率時，token 照它帶的內容畫，不多畫；英文畫面上沒有留下中文 | -| `test_app.py` | 用 Streamlit `AppTest` 操作整個網頁：Streamlit 不在每次執行完腳本後做垃圾回收，重跑會等正在跑的那一次；送出問題、範例按鈕、側欄設定傳到模型、多輪對話、脈絡與模型分頁、允許 / 拒絕 / 停止、新對話、還原設定、兩個分頁不能同時跑、換模型、運作說明連到 GitHub Pages 上同語言的那一版；MTP 開關預設關閉、打開才載入草稿模型、換到沒有草稿模型的 Qwen 會關掉並停用；英文瀏覽器看到英文、上次選的語言優先於瀏覽器、切換語言時對話和設定都留著、模型讀到的錯誤跟著語言；每種語言的頁面只有那種語言的範例、範例會附上它的檔案，模型讀得懂那一種才出現、輸入框上方選的範例檔案會附在接下來打的問題，模型用得上的才列出來、「讓模型寫程式」打開了才去找 Docker 並說明缺什麼、銷售資料的範例用 Python 讀附上的檔案、圖畫在聊天區和畫它的那一步、「新對話」刪掉這段對話的檔案；設了 `VLLM_URL` 時，選單列出伺服器提供的模型、側欄和模型分頁寫出伺服器、MTP 由伺服器決定、開了 sampling mask 的伺服器不會從側欄收到 greedy 或不限的 top-k、模型讀伺服器收的媒體，頁面說明它不收哪些和原因，選單不列伺服器上的 LoRA adapter，網頁不支援的家族的模型標上 ⚠、一開始選第一個支援的，選了不支援的會說明原因，並附上用頁面語言填好的 GitHub issue，寫出模型（資料夾只寫名字），不寫伺服器的位址，伺服器跑到一半送來的錯誤用頁面的語言顯示，載入時或之後連不上伺服器，都用頁面的語言說明；沒設 `VLLM_URL` 時，「用什麼跑模型」一開始是 transformers、完全不碰 Docker，選 vLLM (Docker)會放掉本機的模型（沒有東西留著它）再用它啟動伺服器，重跑不會再啟動，換模型就用新模型啟動，換回 transformers 會停掉容器，vLLM 在跑時新開分頁或重新整理會接上它的伺服器和模型，停在 transformers 的分頁會跟著另一個分頁換到 vLLM 再換回來（模型在這裡只載入一次），在 GPU 上 vLLM 預設開 MTP 啟動、切換開關會不開 MTP 重新啟動（按還原建議值會再開），在 CPU 上不開 MTP、開關會說明原因，分頁會跟著另一個分頁選的 MTP，啟動失敗時用頁面的語言說明，只有按「再試一次」才重試，換回 transformers 的切換鈕也還在 | +| `test_simple_harness.py` | 假模型是一個 `Backend`，它的 token 帶的欄位和 `SimpleBackend` 的一樣；harness：事件順序、工具結果接回脈絡、驗證錯誤、重複呼叫、結果截斷、每一種停止原因、允許 / 拒絕 / 等待時按停止；附件變成問題裡的 part 和給 `run_python` 的說明、每一輪的媒體都交給 processor、展開後的脈絡仍然只把工具結果標成新加入、`run_python` 拿到這段對話的資料夾、沒有 `run_python` 的一輪；模型讀到圖在哪裡，圖本身只給頁面 | +| `test_pi_harness.py` | pi 當 harness，用 `tests/fake_pi.py` 代替 pi：兩個 harness 都是 `Harness`，pi 只送介面列出的事件；一輪變成照迴圈順序的網頁事件（依 channel 串流進來的思考、文字和工具呼叫參數、provider 報的 token 數和 cache、呼叫和結果）；pi 在這段對話的資料夾、用它的 session 啟動，帶著側欄的工具、不帶你自己的 extension，送出問題之前先選好模型、關掉重試，完全沒有工具，附件的檔名寫給模型、圖片直接送過去，重複的呼叫有標示，工具失敗，token 上限，下一個問題用同一個 session；透過 extension 先問：允許、拒絕（模型讀到這一輪語言的拒絕原因）、等待時按停止、逾時；回覆到一半按停止和 pi 還沒啟動就按停止，步數上限之後不再畫 step；請求失敗、問題被拒絕、pi 沒有的模型、pi 一啟動就結束和沒有安裝 pi，都用兩種語言說明；pi 有哪些模型；`RUN_PI_TESTS=1` 時，真的 pi 對上 127.0.0.1 上照劇本回答的模型，先問和不先問各一次；每一步的請求標出新加的部分、不含圖片的內容；pi 透過端點問這個網頁的模型（Gemma 和 Qwen）：每個 token、新加的部分是工具結果的脈絡、端點自己的 `model_end`、沒有 `delta`、pi 的思考等級變成模板的思考開關、圖片送得到模型、按停止讓模型在下一個 token 前停下、先問（允許和拒絕，模型讀到結果）、步數上限（超過的那個請求馬上被拒絕，端點不會被佔住）；`RUN_PI_TESTS=1` 時，真的 pi 透過端點問這個網頁的模型，思考開和關各一次，再加上一次先問、跑到步數上限 | +| `test_model_endpoint.py` | 這個網頁的模型提供給 pi，透過 127.0.0.1 上的 HTTP、後面接著假模型：只聽這台電腦、只認它的 token、沒有一輪在跑時不回答、一次一個請求；回覆照 `content` 串流並附上用量，Gemma 或 Qwen 自己格式的工具呼叫以 `tool_calls` 送回、不當成回覆，請求打開思考時思考是 `reasoning_content`，family 解析不出來的呼叫不送，側欄的 token 上限是 `length`，backend 出錯寫在 finish reason；請求畫成那一步的脈絡（新加的是工具結果）、prefill、token 和結束，不屬於任何一步的請求照樣回答但不畫，停止的一輪讓模型在下一個 token 前停下；對話和 `SimpleHarness` 的一樣（一則 system 訊息、參數是 dict、思考放在 family 的 key 下、工具結果帶著它呼叫的名字），圖片照 hash 只存一次、變成一個 part | +| `test_render.py` | 一次執行的每一個事件前綴都能用兩種語言畫出來（執行中每 0.5 秒重畫）、HTML 標籤成對、迴圈圖是合法 SVG、模型輸出一律跳脫、特殊 token 的說明浮窗與列表；MTP 的底線、浮窗、統計和草稿模型卡片；一串同樣的特殊 token 畫成一個標籤加個數、截斷的片段不會切開 token、媒體佔了幾個 token、好幾行的程式碼和輸出照行寫出來、這一輪沒提供的工具、一次執行和每一步的圖；模型資料夾卡片：檔案的順序、大小和說明、家族有說明但 checkpoint 沒有的檔案、草稿模型的資料夾、跳脫；backend 沒回報機率、或只回報 log 機率時，token 照它帶的內容畫，不多畫；pi 跑的一輪：它的請求畫成脈絡段、標出每一步新加的部分，照 channel 畫出串流進來的文字、沒有 token 的數字、一律跳脫、合併時不改動事件，迴圈圖、停止卡片和每一步的表格說是 pi 在跑迴圈，pi 的卡片兩種語言都對；pi 在這個網頁的模型上跑的一輪，畫出它的 token 和脈絡；英文畫面上沒有留下中文 | +| `test_app.py` | 用 Streamlit `AppTest` 操作整個網頁：Streamlit 不在每次執行完腳本後做垃圾回收，重跑會等正在跑的那一次；送出問題、範例按鈕、側欄設定傳到模型、多輪對話、脈絡與模型分頁、允許 / 拒絕 / 停止、新對話、還原設定、兩個分頁不能同時跑、換模型、運作說明連到 GitHub Pages 上同語言的那一版；MTP 開關預設關閉、打開才載入草稿模型、換到沒有草稿模型的 Qwen 會關掉並停用；英文瀏覽器看到英文、上次選的語言優先於瀏覽器、切換語言時對話和設定都留著、模型讀到的錯誤跟著語言；每種語言的頁面只有那種語言的範例、範例會附上它的檔案，模型讀得懂那一種才出現、輸入框上方選的範例檔案會附在接下來打的問題，模型用得上的才列出來、「讓模型寫程式」打開了才去找 Docker 並說明缺什麼、銷售資料的範例用 Python 讀附上的檔案、圖畫在聊天區和畫它的那一步、「新對話」刪掉這段對話的檔案；設了 `VLLM_URL` 時，選單列出伺服器提供的模型、側欄和模型分頁寫出伺服器、MTP 由伺服器決定、開了 sampling mask 的伺服器不會從側欄收到 greedy 或不限的 top-k、模型讀伺服器收的媒體，頁面說明它不收哪些和原因，選單不列伺服器上的 LoRA adapter，網頁不支援的家族的模型標上 ⚠、一開始選第一個支援的，選了不支援的會說明原因，並附上用頁面語言填好的 GitHub issue，寫出模型（資料夾只寫名字），不寫伺服器的位址，伺服器跑到一半送來的錯誤用頁面的語言顯示，載入時或之後連不上伺服器，都用頁面的語言說明；沒設 `VLLM_URL` 時，「用什麼跑模型」一開始是 transformers、完全不碰 Docker，選 vLLM (Docker)會放掉本機的模型（沒有東西留著它）再用它啟動伺服器，重跑不會再啟動，換模型就用新模型啟動，換回 transformers 會停掉容器，vLLM 在跑時新開分頁或重新整理會接上它的伺服器和模型，停在 transformers 的分頁會跟著另一個分頁換到 vLLM 再換回來（模型在這裡只載入一次），在 GPU 上 vLLM 預設開 MTP 啟動、切換開關會不開 MTP 重新啟動（按還原建議值會再開），在 CPU 上不開 MTP、開關會說明原因，分頁會跟著另一個分頁選的 MTP，啟動失敗時用頁面的語言說明，只有按「再試一次」才重試，換回 transformers 的切換鈕也還在；「用什麼跑迴圈」選 pi 時，介紹說的是 pi 問的那個模型看得到什麼，pi 一開始問的是這個網頁的模型，側欄有模型選單和解碼設定，問它的問題畫出每個 token，「脈絡」分頁有它的脈絡和 pi 的請求，「模型」分頁是那個模型的卡片；問 pi 自己的模型時，側欄是它的模型和設定、沒有解碼設定，問題畫成 pi 的迴圈、在這段對話的資料夾裡跑，側欄的設定傳到 pi，在網頁上按「允許」，停止，「脈絡」分頁顯示 pi 的請求並說明看不到什麼、「模型」分頁顯示 pi 的模型和工具，pi 沒有模型或沒有安裝時說明原因並可以「再試一次」，從不找 Docker，換回來時內建 harness 的模型和設定都還在，切換會開新對話，兩種模型的英文畫面都沒有中文 | | `test_chat_template.py` | 用每個模型真的 tokenizer 和 chat template（已經下載到 `models/` 才跑）：特殊 token 都是單一 token、每個特殊 token 都有說明、token 浮窗顯示的是它代表的文字、模板寫進歷史的工具呼叫能被 parser 讀回；用真的模板跑 harness，確認 step 卡片標成「新加入」的剛好是工具結果（兩個參數、思考模式、工具呼叫前有文字），用真的 processor、問題附上圖片和影片時也一樣；每個模型（和草稿模型）資料夾裡的每個檔案都有說明，說明提到的欄位在檔案裡都找得到 | | `test_vllm_backend.py` | vLLM backend 對上一個假伺服器，它說的是 vLLM 0.32 的串流格式：`/v1/completions` 收到 render 好的 prompt 的 token id、側欄的每個設定（其他取樣設定都設成不起作用）、停止 token，以及放得進伺服器脈絡的 `max_tokens`；每個 token 的機率和前 5 名，排名只在前 5 名裡才有；加了 `--enable-scale-out` 時走 `/inference/v1/generate`，每個 token 都有排名，雖然選中的 token 排在清單第一個，前 5 名仍照排名；兩條路上同分第一的 token 都是 #1，雖然 vLLM 的排名把同分的也算進去；有 sampling mask 時（vLLM 只在 processed log 機率下送），有剩下的候選和取樣時的機率，沒有模型自己的機率，temperature 0 和 top-k 0 在每條路上都照 vLLM 的做法被拒絕，有 mask 但沒加 `--enable-scale-out` 的伺服器，網頁載入時也會發現，而還沒準備好的伺服器不算這種拒絕；載入時用一個 token 判斷伺服器的 `--logprobs-mode`，沒有 mask 的 processed log 機率在前幾名有被剃掉時看得到剩下幾個，logits 只有排名；greedy 是只剩一個、一定抽中的候選；伺服器收哪幾種媒體、每個檔案跟著佔位 token 送過去、脈絡分頁顯示伺服器的展開、生成時展開得不一樣就是錯誤、不收的種類成為一般檔案；兩條路上都有 MTP 的統計，每一步的統計對得上時才標出猜中的 token，收下的只算到停止 token 為止；停止 token、token 上限、按停止就關掉串流；連不上伺服器、它沒有提供的模型、用它自己的資料夾、或用網頁（或它的 transformers）不認得的家族的模型、或沒有 config.json 的模型啟動的伺服器、沒有或出錯的 `/inference/v1/generate`、它的拒絕和錯誤、伺服器從 logits 算出的排名在列出的機率打平時仍照列出的機率，兩種語言都有，跑到一半的錯誤用頁面的語言顯示；伺服器提供的模型屬於哪個家族，只讀它的 config.json 就知道（這台電腦上的資料夾就地讀，沒有 config.json 的 repo 不屬於任何家族，連不上 Hub 或只在伺服器上的資料夾就說不出來），不支援的模型的錯誤帶著模型 id 和 `model_type`，給網頁的 GitHub issue 用；暖機，以及第一個 token 之前就結束的一步；模型卡片上的規格、伺服器、每種 logprobs 模式下數字代表什麼、sampling mask 和它不收的媒體；一整輪呼叫工具再回答、一輪附上圖片的問題，每個時刻都畫得出來；用真的 tokenizer 和 chat template（在 `models/` 裡時），伺服器拿到的就是脈絡分頁上的 prompt；設了 `VLLM_TEST_URL` 時問真的伺服器，伺服器加了哪些參數就測哪些 | | `test_vllm_docker.py` | 在 Docker 裡啟動 vLLM，Docker 在 `subprocess` 這一層換成假的：先照固定的 commit 下載權重，再用模型、它的 revision 和 `models/` 以 `docker compose up` 跑 `docker-compose/compose.yaml`，Docker 有 nvidia runtime（或 `AGENT_LOOP_VLLM_DEVICE` 指定）時再疊上 `compose.gpu.yaml`；模型已經在跑的伺服器不會再啟動一次，換模型會換掉它；沒有 Docker、Docker 沒在執行、沒有 Compose、`docker compose up` 失敗、容器因為記憶體不夠被停掉、容器結束（列出記錄裡的例外，原因在前）和伺服器一直沒回應，都用兩種語言說明；Compose 還在忙時頁面重跑，也會停掉它；`docker compose down`；程式要求的模型在啟動失敗後還記著，直到停掉為止；結束程式時停掉它啟動的伺服器（沒有的話不碰 Docker）；在 GPU 上 MTP 用 Gemma 照固定 commit 的草稿模型（先下載）或 Qwen 自己的 MTP 層，在 CPU 上或沒要求時都不開，切換時重新啟動伺服器。compose 檔：只聽 127.0.0.1、`models/` 唯讀又離線、預設是固定 commit 的 Qwen3.5、GPU 版是同一個 vLLM、除了只給 CPU 的那個以外參數都一樣，再加上和 CPU 一樣大的 KV cache、偏低的 `--gpu-memory-utilization`，以及手動啟動時不開的推測解碼（有設定時統計才設成 detailed，沒有設定時 vLLM 不收），Docker Compose 也把網頁給的設定原封不動交給 vLLM，Docker Compose 讀得懂兩個檔、掛載的是相對的 `AGENT_LOOP_MODELS` 指的資料夾（有裝的話）；設了 `RUN_VLLM_DOCKER_TESTS=1` 時，Docker 裡真的伺服器透過 `VLLMBackend` 回答，Docker 有 GPU 時先開 MTP、再不開各一次 | | `test_media.py` | 網頁收的每一種格式都有範例檔、被當成正確的種類（聲音對 Qwen 是一般檔案）、解碼得出來；存檔的名字不重複也不含奇怪字元；問題的 part 和寫出文字檔欄位名稱的說明；照片轉正、聲音改成模型的取樣率、影片畫格平均取、畫格率跟著調整；每種聲音和影片播放時的類型，以及 MKV 影片改顯示的畫格 | | `test_guide.py` | 運作說明頁的每個程式片段和行號都找得到，並連到 GitHub 上頁面那個 commit 的那幾行、事件表指向真的送出事件的那一行、HTML 標籤成對；每段引用的程式和 `tests/guide_excerpts.txt` 一致，改到那段程式才會變，只是換了行號不會；中英文每一段指向同樣的程式和段落，英文版沒有留下中文；每組頁籤都是已知 backend 或模型的 radio button、各有一個面板，沒有 script 也切換得了，`compose.yaml` 的片段停在該停的那一行；產生的靜態頁有亮色和暗色兩套配色與迴圈圖、寫出對應的 commit，根目錄的頁面會送到正確的語言 | -| `test_real_model.py` | 預設跳過。每個模型：載入的模型是一個 `Backend`；greedy 解碼和 `model.generate()` 一致，純文字、附上圖片、影片和（Gemma）錄音都一樣，之後再問純文字也一樣；真的模型能呼叫計算機並回答；每一種圖片格式都讀得出收據的總計、每一種聲音格式都寫得出錄音的內容（Gemma；Qwen 收到的是一般檔案）、每一種影片格式都看得出球往哪邊滾、每一種資料格式都能用 `run_python` 打開，也寫得出在沙箱裡跑得動的程式（有 Docker 時）；Gemma 開 MTP 時 greedy 輸出和不開一模一樣（附上圖片也是），而且草稿真的有被收下；在 CUDA 上，暖機過的模型放掉之後，GPU 記憶體會全部還回去 | +| `test_layout.py` | `agent/` 裡沒有任何檔案 import `app` 或 Streamlit，不論寫在檔案開頭還是函式裡，從每個檔案的語法樹讀出來：agent 不認識網頁 | +| `test_real_model.py` | 預設跳過。每個模型：載入的模型是一個 `Backend`；greedy 解碼和 `model.generate()` 一致，純文字、附上圖片、影片和（Gemma）錄音都一樣，之後再問純文字也一樣；真的模型能呼叫計算機並回答；每一種圖片格式都讀得出收據的總計、每一種聲音格式都寫得出錄音的內容（Gemma；Qwen 收到的是一般檔案）、每一種影片格式都看得出球往哪邊滾、每一種資料格式都能用 `run_python` 打開，也寫得出在沙箱裡跑得動的程式（有 Docker 時）；Gemma 開 MTP 時 greedy 輸出和不開一模一樣（附上圖片也是），而且草稿真的有被收下；在 CUDA 上，暖機過的模型放掉之後，GPU 記憶體會全部還回去；裝了 pi 的話，真的 pi 透過模型端點問這個模型，呼叫 bash 再回答，每個 token 都畫得出來 | ## 已知限制 @@ -314,7 +353,7 @@ RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real - 開 MTP 時，主模型一次算 7 個位置，數值和一次算 1 個會有極小的差異；greedy 遇到兩個 token 機率幾乎一樣時，偶爾會選到不同的那個。 - Qwen3.5 的 24 層裡有 18 層是 Gated DeltaNet。transformers 有兩個加速 kernel：`flash-linear-attention`（Triton）和 `causal-conv1d`（CUDA），都要有 CUDA GPU 才能用，所以都不列為依賴：改用 PyTorch 參考實作，結果一樣，只是比較慢；`simple_backend.py` 在所有裝置上都把 transformers 這個「改用參考實作」的提示關掉了。有 GPU 時，對 0.8B 的模型差別不大：在 NVIDIA GB10（DGX Spark）上，裝了 flash-linear-attention 後 prefill 快了約 4 倍（3,571 個 token 從 368 ms 變成 93 ms），decode 幾乎沒變（56 → 60 tok/s），兩種寫法和 float32 的差距也一樣小。causal-conv1d 在那台機器上沒有 wheel，得自己編譯。沒有 GPU 的 Linux 不要裝 flash-linear-attention：transformers 還是會選它，第一次 forward 就會失敗。 - 在 MPS 上，transformers 用多條 thread 載入 Qwen3.5 的權重會卡住，所以 `simple_backend.py` 在 MPS 上改成單一 thread 載入（`HF_DEACTIVATE_ASYNC_LOAD=1`），兩個模型的載入時間都沒有變慢。 -- Qwen3.5 0.8B 官方預設不開思考模式；開了容易一直想下去停不下來，可以把「每步 token 上限」調低。它沒附 generation_config.json，建議的解碼設定寫在 `harness/families/qwen3_5.py`（官方建議的 presence penalty 這裡沒有實作）。 +- Qwen3.5 0.8B 官方預設不開思考模式；開了容易一直想下去停不下來，可以把「每步 token 上限」調低。它沒附 generation_config.json，建議的解碼設定寫在 `agent/model_families/qwen3_5.py`（官方建議的 presence penalty 這裡沒有實作）。 - 記憶體裡同時只放一個模型。兩個分頁選了不同的模型時，輪流在兩邊操作會一直重新載入，最好所有分頁用同一個模型。 - 只有一份模型，一次只能跑一個 agent，其他分頁要等。執行中重新整理或關掉分頁，agent 不會跟著停；其他分頁（包括重新整理後的同一頁）會顯示它還在跑，並提供「停止那個 agent」按鈕。 - 開著「執行工具前先問我」時，等確認超過 5 分鐘沒有回應，harness 會當作拒絕並停止迴圈，停止原因顯示「等待確認逾時」。 @@ -333,6 +372,10 @@ RUN_VLLM_DOCKER_TESTS=1 uv run pytest tests/test_vllm_docker.py -k real - `vllm_backend.py` 照 vLLM 0.32 原始碼裡的協定寫，測試用的是說這些協定的假伺服器、真的 tokenizer 和真的網頁；`/inference/v1/generate`、它的 sampling mask 和每一步的統計在 vLLM 裡標示為實驗性功能，之後可能會改。對真伺服器的測試（`VLLM_TEST_URL`）只在一種 GPU 上跑過：DGX Spark 的 GB10。 - 在 Mac 上，Docker 裡的 vLLM 是用 CPU 跑，而且慢：Gemma 4 E2B 解碼約 4.6 tok/s，Docker 要有大約 20 GB 以上的記憶體，第一次載入還要花大約 10 分鐘跑它的媒體編碼器，看伺服器收哪些媒體。`compose.gpu.yaml` 只在一種 NVIDIA GPU 上跑過：DGX Spark 的 GB10。CPU 版的 image 修改了 vLLM 0.32.0（`docker-compose/vllm-cpu/patch_vllm.py`）；新版的 vLLM 可能兩個修正都不需要，要修改的程式和寫它時不一樣的話，建 image 時會停下來。 - 用 Ctrl+C 或 SIGTERM 結束程式時，會停掉 vLLM 的容器。用別的方式結束（關掉終端機、`kill -9`）的話，容器會繼續跑、佔著模型，要用 `docker compose -p agent-loop-vllm down` 停掉。 +- pi 問它自己的模型時，網頁只看得到 pi 回報的東西：沒有 token、機率和特殊 token，也看不到模型讀到的字串，只有 pi 送出的請求。token 數是 provider 報的，tok/s 是回覆傳回來時量的（含網路），費用是照 pi 目錄裡的價格估的。pi 自己的模型是 provider 的：要花錢，對話也會送到那個 provider。 +- pi 問這個網頁的模型時，family 解析不出來的工具呼叫不會送給 pi，pi 的這一輪就停在那裡：不像內建 harness，沒有人告訴模型哪裡寫錯。pi 的 system prompt 和工具 schema 是寫給大模型的，Gemma 4 E2B 和 Qwen3.5 0.8B 照著做的程度比不上內建 harness 的 prompt。pi 只送圖片這一種媒體，所以給 pi 的聲音和影片只是它資料夾裡的一般檔案。不屬於任何一步的請求（例如 pi 壓縮很長的對話）最多等 10 秒，之後照樣回答，但不畫出來。 +- pi 的工具在這台電腦上執行，沒有沙箱；打開 bash 的話，你做得到的事模型都做得到。打開「執行工具前先問我」，每個呼叫都會先給你看。 +- `pi_harness.py` 照 pi 1.4 的 RPC 協定寫，根據它的文件和 pi 1.4.0 實際送出的紀錄；真 pi 的測試只用 pi 1.4.0 跑過。pi 1.4 需要 Node.js 22.19 以上：Node 太舊的話它啟動不了，網頁會顯示它說了什麼。網頁最多每分鐘問一次 pi 有哪些模型；在 pi 裡登入 provider 之後，按「再試一次」就會列出來。 - `MODEL_ID` 環境變數決定一開始選哪個模型，也可以指定選單裡沒有的 checkpoint，只要它屬於已經支援的家族（Gemma 4 或 Qwen3.5）；其他家族要先照「支援更多模型」加一個 family。 ## 授權 diff --git a/agent/__init__.py b/agent/__init__.py new file mode 100644 index 0000000..0ef0ac4 --- /dev/null +++ b/agent/__init__.py @@ -0,0 +1,12 @@ +"""The agent: everything that runs a turn, and nothing a person sees. + +harnesses/ is what runs the loop, behind one interface: simple_harness.py is this project's own loop (SimpleHarness): +context, model, parse, tool, observation, stop, with one event per stage for the app to draw; pi_harness.py hands the +loop to pi, a coding agent, and turns what pi reports into the same events. tools.py has the tools SimpleHarness runs, +with run_python's Docker sandbox built from sandbox/. media.py turns the files attached to a question into what the +model reads. model_families/ is everything model-specific, and model_backends/ is what runs the model, behind one interface. +i18n.py has the page's two languages, which the errors the model reads use too. + +Nothing here imports Streamlit, and importing the package loads nothing heavy: the app's guide imports model_families +and i18n with pygments alone. +""" diff --git a/agent/harnesses/__init__.py b/agent/harnesses/__init__.py new file mode 100644 index 0000000..ff3fd58 --- /dev/null +++ b/agent/harnesses/__init__.py @@ -0,0 +1,7 @@ +"""What runs the agent loop, behind the one interface the app knows. + +harness.py is that interface (Harness) and the events every harness sends. simple_harness.py is this project's own +harness (SimpleHarness), which runs the model through a Backend (model_backends/); pi_harness.py hands the loop to pi, a +coding agent that runs its own (PiHarness). pi_extension.ts is the pi extension through which the page sees pi's +requests and asks before pi runs a tool, and model_endpoint.py serves the page's own model to pi over HTTP. +""" diff --git a/agent/harnesses/harness.py b/agent/harnesses/harness.py new file mode 100644 index 0000000..5d0567f --- /dev/null +++ b/agent/harnesses/harness.py @@ -0,0 +1,56 @@ +"""The interface between the page and whatever runs the agent loop. + +The page (app/page.py) runs one turn at a time and reaches the loop only through a Harness: it makes one with a +function that appends each event to the turn's list, calls run() in a background thread, and passes the user's +clicks on with stop() and decide(). render.py draws the list. This project's own harness is +simple_harness.SimpleHarness, which runs the model through a Backend (model_backends/backend.py), so it knows every token; +pi_harness.PiHarness hands the loop to pi, a coding agent that runs its own and says what it does as JSON. Another +agent that reports its loop as events, such as the Claude Agent SDK or Codex, would be one more; this module says +what it has to send. + +A harness that runs the model through a Backend knows the prompt the model reads and each token it writes; one that +asks a model's API sees only the text that comes back. So, like a token event's details (backend.TOKEN_DETAILS), most +events are sent only by a harness that knows what they say, and render.py draws a step from whichever it has: the page +never shows a prompt or a number the harness did not see. + +Nothing here imports anything, so a harness can be written, and the interface read, without the rest. +""" + +from typing import Protocol, runtime_checkable + +# The events every harness sends: run_start first, a step_start each time around the loop, and run_end last (after +# an error event when something failed). run_start carries which harness ran the turn (`harness`: "simple", "pi"), the +# model, its family (a key of model_families.FAMILIES, or None when the harness does not know how the model writes its turns), +# the turn's settings and the tools the model gets; run_end the stop reason, the final reply and the conversation. +ALWAYS = ("run_start", "step_start", "run_end") +# The events a harness sends in a step when it knows what they say, in the order they come: +# request the request sent to the model's API, as JSON, from a harness that asks one (pi) +# context the prompt as the model reads it, from a harness that renders it (SimpleHarness, or pi's model endpoint) +# prefill, token the prompt read, then each token (backend.TOKEN_FIELDS), from a harness that runs the model itself +# delta a piece of the model's output as it streams, with its channel, from one that sees text, not tokens (pi) +# model_end the model's turn ended: why, and how many tokens, with their speed when it was measured +# parsed what the harness read in it: the thinking, the reply, the tool calls +# tool_call a call it will run (or reports as failed), with its environment +# approval_request, approval the call held for the user, and the user's answer +# tool_start, tool_result the call running in its environment, and what came back +# error what went wrong, before run_end +SOMETIMES = ("request", "context", "prefill", "token", "delta", "model_end", "parsed", "tool_call", "approval_request", "approval", + "tool_start", "tool_result", "error") + + +@runtime_checkable +class Harness(Protocol): + """What runs one turn. The page makes one per question (app.page.LiveRun) and runs one at a time.""" + + id: str + + def run(self) -> None: + """Run the turn to its end, sending its events. It returns after run_end and never raises: a failure is an + error event followed by run_end.""" + + def stop(self) -> None: + """Stop the turn as soon as it can: the user pressed Stop. It is called from another thread, and run() still + ends with run_end.""" + + def decide(self, call_id: str, approved: bool) -> None: + """The user's answer to the approval_request for the call `call_id`. It is called from another thread.""" diff --git a/agent/harnesses/model_endpoint.py b/agent/harnesses/model_endpoint.py new file mode 100644 index 0000000..a93d080 --- /dev/null +++ b/agent/harnesses/model_endpoint.py @@ -0,0 +1,285 @@ +"""The page's own model as an OpenAI-compatible chat completions endpoint, for a harness that asks a model's API (pi). + +pi asks models over HTTP, so to have pi ask the model in memory here (SimpleBackend on this machine, or VLLMBackend), this +module serves it on 127.0.0.1, at a port of its own and behind a token. pi_extension.ts registers it in pi as the +provider agent-loop. Each request pi sends is the conversation and the tools in the OpenAI format; the endpoint turns +it into the conversation as SimpleHarness keeps it, renders it with the model's chat template, encodes it and +generates, as SimpleHarness does in a step. Its events (context, prefill, token, model_end) go to the turn being drawn +(Serving), into the step pi is in, so pi's steps show every token and the string the model read. The reply streams +back to pi as server-sent events: the thinking as reasoning_content, the reply as content, and at the end the tool +calls the family parses out of the model's own format as tool_calls, which pi runs. + +pi checks a tool call's arguments itself, so the endpoint sends only the calls it could parse; one it could not is +not sent, and pi ends its turn there, since nothing told the model what went wrong (SimpleHarness would). Sampling, +thinking and the token limit are the sidebar's (Serving), not what the request asks for, except that thinking follows +pi's thinking level (chat_template_kwargs.enable_thinking). +""" + +import base64 +import hashlib +import json +import mimetypes +import secrets +import threading +import time +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from agent import media +from agent.harnesses.simple_harness import common_prefix +from agent.model_backends.backend import Backend, Sampling + +PROVIDER = "agent-loop" # what pi calls the endpoint, as pi_extension.ts registers it +WAIT_FOR_STEP = 10 # seconds a request waits for the turn it belongs to before it is served undrawn + + +@dataclass +class Serving: + """What a request is served with, for one turn: the model and the sidebar's settings, and where its events go.""" + llm: Backend + sampling: Sampling + lang: str # the language the errors of a tool call the family cannot parse are worded in + folder: Path # where the images in a request are saved for the processor: the conversation's folder + step: Callable[[], int | None] # waits for the step the request belongs to; None for a request outside every step + emit: Callable[..., None] # emit(event_type, **data), into the turn's events + stopped: Callable[[], bool] # whether the turn is being stopped + + +class Endpoint: + """The HTTP server, one per process; it serves the turn that called serve(), and refuses requests otherwise.""" + + def __init__(self): + self.token = secrets.token_urlsafe(24) # pi sends it as its API key; anything else on this machine is refused + self._serving: Serving | None = None + self._lock = threading.Lock() # one request at a time: one model + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _handler(self)) + self._server.daemon_threads = True + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self._server.server_address[1]}/v1" + + def serve(self, serving: Serving): + self._serving = serving + + def stop_serving(self): + self._serving = None + + +_endpoint: Endpoint | None = None +_endpoint_lock = threading.Lock() + + +def endpoint() -> Endpoint: + """The process's endpoint, started the first time it is asked for.""" + global _endpoint + with _endpoint_lock: + if _endpoint is None: + _endpoint = Endpoint() + return _endpoint + + +# ---------- the request, as the conversation SimpleHarness keeps ---------- + +def text_of(content) -> str: + if isinstance(content, str) or content is None: + return content or "" + return "".join(p.get("text", "") for p in content if p.get("type") == "text") + + +def image_part(url: str, folder: Path) -> dict | None: + """An image sent as a data URL, saved in the conversation's folder under its hash, so that each request names the + same file; as the part of a user message the chat template writes a placeholder for (media.Attachment.part).""" + if not url.startswith("data:") or ";base64," not in url: + return None + mime, data = url[5:].split(";base64,", 1) + raw = base64.b64decode(data) + folder = folder / "pi-images" + folder.mkdir(parents=True, exist_ok=True) + path = folder / (hashlib.sha256(raw).hexdigest()[:16] + (mimetypes.guess_extension(mime) or ".png")) + if not path.exists(): + path.write_bytes(raw) + return media.Attachment(path.name, "image", str(path), len(raw)).part() + + +def user_content(content, folder: Path) -> str | list[dict]: + if isinstance(content, str) or content is None: + return content or "" + parts = [] + for p in content: + if p.get("type") == "text": + parts.append({"type": "text", "text": p.get("text", "")}) + elif p.get("type") == "image_url" and (image := image_part((p.get("image_url") or {}).get("url", ""), folder)): + parts.append(image) + return parts if any(p["type"] != "text" for p in parts) else "".join(p["text"] for p in parts) + + +def conversation(body: dict, llm: Backend, folder: Path) -> list[dict]: + """The request's messages as SimpleHarness keeps its own (Hugging Face's format, which the chat templates take): + the arguments of a tool call as a dict, the thinking under the family's key, a tool result with its tool's name.""" + family, names = llm.family, {} + out: list[dict] = [] + for m in body.get("messages", []): + role = m.get("role") + if role in ("system", "developer"): + if out and out[-1]["role"] == "system": # a template takes one system message, first + out[-1]["content"] += "\n\n" + text_of(m.get("content")) + else: + out.append({"role": "system", "content": text_of(m.get("content"))}) + elif role == "user": + out.append({"role": "user", "content": user_content(m.get("content"), folder)}) + elif role == "assistant": + calls = [] + for c in m.get("tool_calls") or []: + fn = c.get("function") or {} + try: + args = json.loads(fn.get("arguments") or "{}") + except json.JSONDecodeError: + args = {} + names[c.get("id")] = fn.get("name", "") + calls.append({"id": c.get("id"), "type": "function", "function": {"name": fn.get("name", ""), "arguments": args}}) + # Some templates cannot keep text in front of a tool call (see Family.text_with_calls). + message: dict = {"role": "assistant", "content": text_of(m.get("content")) if family.text_with_calls or not calls else ""} + if calls: + message["tool_calls"] = calls + if thinking := m.get("reasoning_content") or m.get("reasoning"): + message[family.reasoning_key] = thinking + out.append(message) + elif role == "tool": + out.append({"role": "tool", "tool_call_id": m.get("tool_call_id"), "name": m.get("name") or names.get(m.get("tool_call_id"), ""), + "content": text_of(m.get("content"))}) + return out + + +def seen_before(messages: list[dict]) -> list[dict]: + """The conversation as of the model's last message: what the step before this one ended with, before the tool + results that are new now.""" + last = max((i for i, m in enumerate(messages) if m["role"] == "assistant"), default=None) + return messages[: last + 1] if last is not None else [] + + +# ---------- one request ---------- + +def _handler(ep: Endpoint): + # HTTP/1.0, so a stream ends where the connection closes, with no length or chunks to say where. + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _json(self, status: int, data: dict): + body = json.dumps(data).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _authorized(self) -> bool: + if self.headers.get("Authorization") == f"Bearer {ep.token}": + return True + self._json(401, {"error": {"message": "This endpoint serves the Agent Loop Visualizer's pi only."}}) + return False + + def do_GET(self): + if not self._authorized(): + return + serving = ep._serving + self._json(200, {"object": "list", "data": [{"id": serving.llm.model_id, "object": "model"}] if serving else []}) + + def do_POST(self): + if not self._authorized(): + return + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + serving = ep._serving + if self.path.rstrip("/") != "/v1/chat/completions" or serving is None: + self._json(404 if serving else 503, {"error": {"message": "No turn is being served."}}) + return + with ep._lock: + self._complete(body, serving) + + def _complete(self, body: dict, serving: Serving): + llm, family = serving.llm, serving.llm.family + step = serving.step() # pi starts its turn before it sends the request; this waits until the page has drawn it + if step is None and serving.stopped(): # a turn pi is aborted at (past Max steps, or after Stop): not worth a prefill + self._json(503, {"error": {"message": "The turn is being stopped."}}) + return + emit = (lambda kind, **data: serving.emit(kind, step=step, **data)) if step else (lambda kind, **data: None) + tools = body.get("tools") or None + schemas = {t["function"]["name"]: t["function"].get("parameters", {}) for t in tools or []} + thinking = bool((body.get("chat_template_kwargs") or {}).get("enable_thinking")) + try: + messages = conversation(body, llm, serving.folder) + prompt = llm.render(messages, tools, thinking) + encoded = llm.encode(prompt, media.parts(messages)) + except Exception as e: # pi shows it as the request's error + self._json(400, {"error": {"message": f"{type(e).__name__}: {e}"}}) + return + before = seen_before(messages) + # Media sit in user messages, all before what is new; so the new part is the same text, further along (SimpleHarness). + if before: + previous = llm.render(before, tools, thinking, generation_prompt=False) + added_from = len(encoded.text) - len(prompt) + common_prefix(prompt, previous) + else: + added_from = 0 + emit("context", prompt=encoded.text, n_tokens=encoded.n_tokens, added_from=added_from, media_tokens=encoded.media_tokens, + context_length=llm.context_length, roles={role: sum(m["role"] == role for m in messages) for role in dict.fromkeys(m["role"] for m in messages)}) + + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + rid, gone = f"chatcmpl-{uuid.uuid4().hex[:12]}", False + + def send(delta: dict | None = None, finish: str | None = None, usage: dict | None = None) -> bool: + nonlocal gone + chunk = {"id": rid, "object": "chat.completion.chunk", "created": int(time.time()), "model": llm.model_id, + "choices": [] if usage else [{"index": 0, "delta": delta or {}, "finish_reason": finish}]} + if usage: + chunk["usage"] = usage + try: + self.wfile.write(f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode()) + self.wfile.flush() + except OSError: # pi went away (it was aborted): the model stops before its next token + gone = True + return not gone + + send({"role": "assistant"}) + done = None + try: + for ev in llm.generate(encoded, serving.sampling, lambda: gone or serving.stopped()): + kind = ev.pop("type") + if kind == "done": + done = ev + continue + emit(kind, **ev) + if kind == "token" and ev["channel"] in ("thought", "content"): + send({"reasoning_content" if ev["channel"] == "thought" else "content": ev["text"]}) + except Exception as e: # what runs the model failed (a vLLM server, say): a stream cannot say so but by its finish reason, which pi shows + send(finish=f"error: {type(e).__name__}: {e}") + return + assert done is not None # generate() always ends with done + raw = done.pop("raw") + emit("model_end", raw=raw, **done) + if gone: + return + # The chat template may open the thinking itself; that part is in the prompt, not in raw. + prefix = family.thought_prefix if prompt.endswith(family.thought_prefix) else "" + calls = [c for c in family.parse(prefix + raw, schemas, serving.lang)["tool_calls"] if not c["error"]] if tools else [] + if calls: + send({"tool_calls": [{"index": i, "id": f"call_{uuid.uuid4().hex[:8]}", "type": "function", + "function": {"name": c["name"], "arguments": json.dumps(c["arguments"] or {}, ensure_ascii=False)}} + for i, c in enumerate(calls)]}) + finish = "length" if done["finish"] == "max_new_tokens" else "tool_calls" if calls else "stop" + send(finish=finish) + send(usage={"prompt_tokens": encoded.n_tokens, "completion_tokens": done["n_tokens"], "total_tokens": encoded.n_tokens + done["n_tokens"]}) + try: + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + except OSError: + pass + + return Handler diff --git a/agent/harnesses/pi_extension.ts b/agent/harnesses/pi_extension.ts new file mode 100644 index 0000000..768c60a --- /dev/null +++ b/agent/harnesses/pi_extension.ts @@ -0,0 +1,63 @@ +/** + * The Agent Loop Visualizer's extension for pi (agent/harnesses/pi_harness.py), loaded into every pi the page starts. + * It does three things, each set by an environment variable that pi_harness.py sets: + * + * - Before every request pi sends a model, it tells the page what the request holds, as a notification, which in RPC + * mode is an extension_ui_request on stdout (REQUEST_MARK, then the payload as JSON); the page draws it as the step's + * request. + * - With AGENT_LOOP_ENDPOINT, it registers the page's own model, served by model_endpoint.py, as the provider + * agent-loop, so that pi can ask Gemma or Qwen on transformers or vLLM; the thinking level becomes the chat template's + * enable_thinking. + * - With AGENT_LOOP_APPROVE, it asks before every tool call through a confirm dialog, which the page shows as Allow and + * Reject; a rejected call does not run, and the model reads the refusal (AGENT_LOOP_REFUSAL) in the turn's language. + */ + +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +const APPROVE_TITLE = "agent-loop:approve"; // pi_harness.APPROVE_TITLE, how pi_harness.py tells this dialog from any other +const REQUEST_MARK = "agent-loop:request "; // pi_harness.REQUEST_MARK +const env = process.env; + +export default function (pi: ExtensionAPI) { + if (env.AGENT_LOOP_ENDPOINT) { + pi.registerProvider("agent-loop", { + name: "Agent Loop Visualizer", + baseUrl: env.AGENT_LOOP_ENDPOINT, + apiKey: "$AGENT_LOOP_TOKEN", + api: "openai-completions", + models: [ + { + id: env.AGENT_LOOP_MODEL, + name: env.AGENT_LOOP_MODEL, + reasoning: true, + input: (env.AGENT_LOOP_INPUT || "text").split(","), + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: Number(env.AGENT_LOOP_CONTEXT), + maxTokens: Number(env.AGENT_LOOP_MAX_TOKENS), + compat: { + thinkingFormat: "chat-template", + chatTemplateKwargs: { enable_thinking: { $var: "thinking.enabled" } }, + supportsDeveloperRole: false, + supportsStore: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + }, + }, + ], + }); + } + + pi.on("before_provider_request", (event, ctx) => { + ctx.ui.notify(REQUEST_MARK + JSON.stringify(event.payload), "info"); + }); + + if (env.AGENT_LOOP_APPROVE) { + pi.on("tool_call", async (event, ctx) => { + const call = JSON.stringify({ toolCallId: event.toolCallId, toolName: event.toolName, input: event.input }); + if (!(await ctx.ui.confirm(APPROVE_TITLE, call))) { + return { block: true, reason: env.AGENT_LOOP_REFUSAL || "The user refused to run this tool call" }; + } + return undefined; + }); + } +} diff --git a/agent/harnesses/pi_harness.py b/agent/harnesses/pi_harness.py new file mode 100644 index 0000000..82aeefc --- /dev/null +++ b/agent/harnesses/pi_harness.py @@ -0,0 +1,568 @@ +"""pi as the harness: the page hands the turn to pi (https://pi.dev), a coding agent that runs a loop of its own, +and draws what pi says it does. + +pi runs as a child process in RPC mode (`pi --mode rpc`): commands go in on its stdin and records come out on its +stdout, one JSON object per line. For each question PiHarness starts pi in the conversation's folder, picks the model and +the thinking level, sends the question as a prompt and turns pi's events into the page's (harness.py): one of pi's +turns (turn_start to turn_end: one request to the model and the tools its reply calls) is a step, the text the model +streams is delta, the finished assistant message is model_end and parsed, and each tool execution is tool_call, +tool_start and tool_result. pi keeps the conversation in a session file in the conversation's folder, so the next +question starts pi on the same session and the model reads all of it. + +pi asks a model over HTTP, in one of two ways. It can ask the page's own model (PageModel: Gemma or Qwen on +transformers or vLLM), which model_endpoint.py serves on 127.0.0.1 as the provider agent-loop: then the endpoint renders +each request with the model's chat template and generates, and its events (context, prefill, token, model_end) go +into pi's step, which shows every token as SimpleHarness's do. Or it can ask a model at a provider (Anthropic, OpenAI, +Google, ... or any endpoint in its models.json), with the credentials it keeps itself (`pi` then /login, or an API +key in the environment): an API sends back text, not tokens, so a step shows the text as it streams (delta) and the +token counts the provider reported, and no probabilities. Either way pi_extension.ts reports each request pi sends, +and the step shows it (request): pi's system prompt, its tools and the conversation, as JSON. + +pi's tools (read, bash, edit, write, grep, find, ls) run on this machine, in the conversation's folder, without a +sandbox: bash reaches whatever the user can. Ask me before running a tool goes through the same extension, which then +asks before every call. In RPC mode its question is a dialog on stdout (extension_ui_request), which becomes the +page's approval_request; the user's answer goes back on stdin. + +The page starts pi without the user's own extensions, skills, prompt templates and context files (AGENTS.md), so +that what it draws is pi's own loop, the same on every machine; the models and the credentials are still the user's. +""" + +import base64 +import collections +import json +import mimetypes +import os +import queue +import shlex +import subprocess +import tempfile +import threading +import time +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import IO + +from agent import media +from agent.harnesses import model_endpoint +from agent.harnesses.simple_harness import APPROVAL_TIMEOUT, common_prefix +from agent.i18n import pick +from agent.model_backends.backend import Backend, Sampling + +# How to start pi: the `pi` on the PATH, unless PI_COMMAND says otherwise (a path, or "npx -y @earendil-works/pi-coding-agent"). +PI_COMMAND = shlex.split(os.environ.get("PI_COMMAND", "pi")) +INSTALL = "npm install -g @earendil-works/pi-coding-agent" +FLAGS = ["--mode", "rpc", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files"] +FILES = {"zh": "這台電腦的檔案", "en": "files on this computer"} +# pi's built-in tools: what each does and where it runs, as the page says it. +TOOLS = { + "read": ({"zh": "讀一個檔案（文字或圖片）", "en": "reads a file (text or an image)"}, FILES), + "bash": ({"zh": "執行 shell 指令", "en": "runs a shell command"}, {"zh": "這台電腦的 shell", "en": "shell on this computer"}), + "edit": ({"zh": "把檔案裡的一段文字換成另一段", "en": "replaces exact text in a file"}, FILES), + "write": ({"zh": "新建或覆寫一個檔案", "en": "creates or overwrites a file"}, FILES), + "grep": ({"zh": "搜尋檔案的內容", "en": "searches the contents of files"}, FILES), + "find": ({"zh": "用 glob 找檔案的路徑", "en": "finds paths by a glob pattern"}, FILES), + "ls": ({"zh": "列出一個資料夾", "en": "lists a folder"}, FILES), +} +OTHER_TOOL = {"zh": "pi 的工具", "en": "a pi tool"} # where a tool runs that pi has and the page does not know +DEFAULT_TOOLS = ("read", "bash", "edit", "write") # what pi offers the model unless told otherwise +THINKING = ("off", "minimal", "low", "medium", "high") # pi's thinking levels for a model that thinks +EXTENSION = Path(__file__).with_name("pi_extension.ts") +APPROVE_TITLE = "agent-loop:approve" # the title pi_extension.ts gives its confirm dialog +REQUEST_MARK = "agent-loop:request " # how pi_extension.ts starts the notification of a request +SESSION_DIR = ".pi-session" # in the conversation's folder +START_TIMEOUT = 30 # seconds pi may take to start and to answer a command +STOP_GRACE = 10 # seconds pi has to settle after an abort, or to exit after its stdin closes, before it is killed +DIALOGS = ("select", "confirm", "input", "editor") # the extension_ui_request methods that wait for an answer + + +class PiError(Exception): + """pi could not be started, refused a command or exited; `text` says so in both of the page's languages.""" + + def __init__(self, zh: str, en: str): + super().__init__(en) + self.text = {"zh": zh, "en": en} + + +def environment_of(tool: str) -> dict[str, str]: + return TOOLS[tool][1] if tool in TOOLS else OTHER_TOOL + + +def not_found() -> PiError: + command = " ".join(PI_COMMAND) + return PiError( + f"找不到 pi（`{command}`）。用 `{INSTALL}` 安裝它（需要 Node.js 22.19 以上），或用環境變數 `PI_COMMAND` 指定怎麼啟動它。", + f"pi (`{command}`) was not found. Install it with `{INSTALL}` (it needs Node.js 22.19 or later), or say how to start it with `PI_COMMAND`.", + ) + + +class Process: + """One pi in RPC mode: commands are written to its stdin, and a thread reads its stdout into `inbox`.""" + + def __init__(self, args: list[str], cwd: Path, inbox: queue.Queue, env: dict | None = None): + try: + self.proc = subprocess.Popen( + [*PI_COMMAND, *FLAGS, *args], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, env={**os.environ, "PI_SKIP_VERSION_CHECK": "1", **(env or {})}, + ) + except (FileNotFoundError, PermissionError): + raise not_found() from None + assert self.proc.stdin and self.proc.stdout and self.proc.stderr # pipes, as asked + self.stdin = self.proc.stdin + self.inbox = inbox + self.backlog: collections.deque = collections.deque() # what came while request() waited for its response + self.stderr: collections.deque = collections.deque(maxlen=20) + self._lock = threading.Lock() + threading.Thread(target=self._read, args=(self.proc.stdout,), daemon=True).start() + threading.Thread(target=self._read_stderr, args=(self.proc.stderr,), daemon=True).start() + + def _read(self, stdout: IO[bytes]): + # A binary pipe splits lines on LF alone, as RPC's framing asks: a JSON string may hold U+2028. + for line in stdout: + try: + self.inbox.put(("record", json.loads(line))) + except json.JSONDecodeError: + continue + self.inbox.put(("exit", None)) + + def _read_stderr(self, stderr: IO[bytes]): + for line in stderr: + self.stderr.append(line.decode(errors="replace").rstrip()) + + def send(self, command: dict): + with self._lock: + try: + self.stdin.write(json.dumps(command, ensure_ascii=False).encode() + b"\n") + self.stdin.flush() + except (OSError, ValueError): # pi has exited, which its stdout says too + pass + + def request(self, command: dict, timeout: float = START_TIMEOUT) -> dict: + """Send a command and wait for its response; return the response's data. What comes meanwhile is kept, in + order, in `backlog`.""" + rid = uuid.uuid4().hex[:8] + self.send({**command, "id": rid}) + deadline = time.monotonic() + timeout + while True: + try: + kind, value = self.inbox.get(timeout=max(deadline - time.monotonic(), 0.01)) + except queue.Empty: + raise PiError(f"pi 在 {timeout} 秒內沒有回應 `{command['type']}`。", f"pi did not answer `{command['type']}` within {timeout} s.") from None + if kind == "exit": + raise self.exited() + if kind == "record" and value.get("type") == "response" and value.get("id") == rid: + if not value.get("success"): + error = value.get("error") or "" + raise PiError(f"pi 拒絕了 `{command['type']}`：{error}", f"pi refused `{command['type']}`: {error}") + return value.get("data") or {} + self.backlog.append((kind, value)) + + def exited(self) -> PiError: + said = "\n".join(self.stderr) + return PiError("pi 意外結束了。" + (f"它最後說：\n{said}" if said else ""), "pi exited unexpectedly." + (f" Its last words:\n{said}" if said else "")) + + def close(self, grace: float = STOP_GRACE): + """Close stdin, which asks pi to shut down, and kill it if it has not within `grace` seconds.""" + with self._lock: + try: + self.stdin.close() + except OSError: + pass + try: + self.proc.wait(grace) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + +def probe() -> dict: + """What pi can run: its version, the models it has credentials for (pi's own model objects: provider, id, name, + input, reasoning, contextWindow, maxTokens, cost) and the one it starts on by itself ("provider/id", or None). + Raises PiError when pi does not start or answer.""" + try: + done = subprocess.run([*PI_COMMAND, "--version"], capture_output=True, text=True, timeout=START_TIMEOUT) + except (FileNotFoundError, PermissionError): + raise not_found() from None + except subprocess.TimeoutExpired: + raise PiError(f"`{' '.join(PI_COMMAND)} --version` 沒有結束。", f"`{' '.join(PI_COMMAND)} --version` did not finish.") from None + if done.returncode: + said = (done.stderr or done.stdout).strip()[-1000:] + raise PiError(f"pi 沒有啟動：\n{said}", f"pi did not start:\n{said}") + with tempfile.TemporaryDirectory() as folder: # somewhere with no project settings of its own + process = Process(["--no-session", "--no-tools"], Path(folder), queue.Queue()) + try: + models = process.request({"type": "get_available_models"})["models"] + current = process.request({"type": "get_state"}).get("model") + finally: + process.close() + names = [f"{m['provider']}/{m['id']}" for m in models] + start = f"{current['provider']}/{current['id']}" if current else None + return {"version": done.stdout.strip(), "models": models, "default": start if start in names else None} + + +@dataclass +class PageModel: + """The page's own model, for pi to ask through model_endpoint.py instead of a provider's, with the sidebar's decoding.""" + llm: Backend + sampling: Sampling + + +@dataclass +class PiSettings: + provider: str # the model, as pi names it: its provider and its id there (model_endpoint.PROVIDER for the page's own) + model: str + thinking: str = "off" # one of THINKING + tools: tuple[str, ...] = DEFAULT_TOOLS + approve_tools: bool = False + max_steps: int = 6 + lang: str = "zh" # what pi_extension.ts tells the model when the user refuses a call is worded in this language + + +class PiHarness: + """One turn, run by pi (harness.Harness).""" + + def __init__(self, settings: PiSettings, history: list[dict], text: str, emit: Callable[[dict], None], + attachments: Sequence[media.Attachment] = (), folder: Path | None = None, session: str = "chat", page: PageModel | None = None): + self.id = uuid.uuid4().hex[:12] + self.settings = settings + self.page = page # the page's own model, when pi asks it rather than a provider's + self.history = history # only for run_end: pi keeps the conversation in its session + self.text = text + self.attachments = list(attachments) + self.folder = Path(folder or tempfile.mkdtemp()) # the conversation's: pi works in it and keeps its session there + self.session = session # the session's id, the same for every question of the conversation + self._emit = emit + self._start = time.perf_counter() + self._inbox: queue.Queue = queue.Queue() # pi's records, and the user's clicks, for run() to handle in order + self._pi: Process | None = None + self._step, self._open = 0, False # the page's step, and whether pi's current turn is one (not after an abort) + self._turn_t = self._stream_t = 0.0 + self._wrote = False # whether the model has streamed anything yet in this message + self._calls: dict[str, dict] = {} # call id -> name, arguments and step of each call drawn + self._pending: dict[str, tuple[str, float]] = {} # call id -> the dialog's id and when it times out + self._seen: list[tuple[str, str]] = [] + self._last: dict | None = None # the last assistant message of a step + self._error: str | None = None + self._aborting: str | None = None # why the turn is being stopped: user_stopped, max_steps, approval_timeout + self._kill_at: float | None = None # after an abort: when to stop waiting for pi to settle + self._hung = False # pi did not settle in time, so it is killed rather than asked to exit + self._emit_lock = threading.Lock() # the endpoint's thread sends the model's events, beside run()'s + self._unserved = threading.Condition() # a step opened whose request the endpoint has not taken yet + self._waiting: int | None = None + self._request_text = "" # the last request pi sent, as JSON, to mark what the next one adds + + @property + def pi(self) -> Process: + """The pi run() started, which everything after it talks to.""" + assert self._pi is not None + return self._pi + + # --- controls called from the web server --- + + def stop(self): + self._inbox.put(("stop", None)) + + def decide(self, call_id: str, approved: bool): + self._inbox.put(("decide", (call_id, approved))) + + # --- the turn --- + + def emit(self, event_type: str, **data): + with self._emit_lock: + self._emit({"type": event_type, "t": round((time.perf_counter() - self._start) * 1000), **data}) + + def run(self): + s, page = self.settings, self.page + settings = {"model": f"{s.provider}/{s.model}", "thinking": s.thinking, "use_tools": bool(s.tools), "tools": list(s.tools), + "approve_tools": s.approve_tools, "max_steps": s.max_steps, "lang": s.lang, "page_model": page is not None} + if page: # what the step cards and the Model tab say of the decoding + sm = page.sampling + settings |= {"temperature": sm.temperature, "top_p": sm.top_p, "top_k": sm.top_k, "max_new_tokens": sm.max_new_tokens, "mtp": sm.mtp} + self.emit( + "run_start", run_id=self.id, model=page.llm.model_id if page else f"{s.provider}/{s.model}", + family=page.llm.family.key if page else None, harness="pi", settings=settings, tools=list(s.tools), + ) + stop_reason = "error" + try: + self.folder.mkdir(parents=True, exist_ok=True) + self._pi = Process(self._args(), self.folder, self._inbox, self._env()) + # Each waits for its answer: pi handles commands concurrently, and a prompt must not go to the model it starts on. + self._pi.request({"type": "set_model", "provider": s.provider, "modelId": s.model}) + self._pi.request({"type": "set_thinking_level", "level": s.thinking}) + # A failed request is retried by pi as a new turn, which would be a step of its own; the page shows the error instead. + self._pi.request({"type": "set_auto_retry", "enabled": False}) + message, images = self._message() + if not self._stopped_meanwhile(): + self._pi.request({"type": "prompt", "message": message, **({"images": images} if images else {})}) + self._loop() + stop_reason = self._stop_reason() + if stop_reason == "error": + self.emit("error", message=f"pi: {self._error}") + except PiError as e: # pi did not start, refused, or exited + self.emit("error", message=f"PiError: {e}", text={lang: f"PiError: {t}" for lang, t in e.text.items()}) + except Exception as e: # a crash here is shown, not a silent hang + self.emit("error", message=f"{type(e).__name__}: {e}") + finally: + if page: + model_endpoint.endpoint().stop_serving() + if self._pi: + self._pi.close(0 if self._hung else STOP_GRACE) + final = self._text(self._last) if self._last and stop_reason != "error" else "" + self.emit("run_end", stop_reason=stop_reason, final=final, + messages=[*self.history, {"role": "user", "content": self.text}, {"role": "assistant", "content": final}]) + + def _args(self) -> list[str]: + s = self.settings + args = ["--session-dir", str(self.folder / SESSION_DIR), "--session-id", self.session, "--extension", str(EXTENSION)] + return args + (["--tools", ",".join(s.tools)] if s.tools else ["--no-tools"]) + + def _env(self) -> dict[str, str]: + """What pi_extension.ts is told: the refusal's words, whether to ask, and the page's model if pi asks it.""" + s, env = self.settings, {"AGENT_LOOP_REFUSAL": pick(self.settings.lang, "使用者拒絕執行這個工具呼叫", "The user refused to run this tool call")} + if s.approve_tools: + env["AGENT_LOOP_APPROVE"] = "1" + if page := self.page: + ep = model_endpoint.endpoint() + ep.serve(model_endpoint.Serving(page.llm, page.sampling, s.lang, self.folder, self._serving_step, self.emit, + lambda: self._aborting is not None)) + env |= {"AGENT_LOOP_ENDPOINT": ep.url, "AGENT_LOOP_TOKEN": ep.token, "AGENT_LOOP_MODEL": page.llm.model_id, + "AGENT_LOOP_CONTEXT": str(page.llm.context_length), "AGENT_LOOP_MAX_TOKENS": str(page.sampling.max_new_tokens), + # pi sends images alone of what a model reads + "AGENT_LOOP_INPUT": "text,image" if "image" in page.llm.modalities else "text"} + return env + + def _serving_step(self) -> int | None: + """For the endpoint: the step a request pi sent belongs to. pi starts a turn and then sends its request, and the two + reach the page by different ways, so a request waits until the page has opened that step; one that comes outside + every step (pi compacting the conversation, say) is served without being drawn.""" + with self._unserved: + self._unserved.wait_for(lambda: self._waiting is not None or self._aborting is not None, timeout=model_endpoint.WAIT_FOR_STEP) + step, self._waiting = self._waiting, None + return step + + def _message(self) -> tuple[str, list[dict]]: + """The question, with a line for the model on the files attached (in English, as all the model reads from the + page is), and the images it reads inline.""" + images = [{"type": "image", "data": base64.b64encode(Path(a.path).read_bytes()).decode(), "mimeType": mimetypes.guess_type(a.name)[0] or "image/png"} + for a in self.attachments if a.kind == "image"] + files = ", ".join(f"{a.name} ({media.fmt_size(a.size)})" for a in self.attachments) + note = f"Attached, in the current folder: {files}" if files else "" + return "\n\n".join(t for t in (self.text, note) if t), images + + def _stopped_meanwhile(self) -> bool: + """Whether Stop was pressed while pi started, before the question went to it.""" + if any(kind == "stop" for kind, _ in self.pi.backlog): + self._aborting = "user_stopped" + return self._aborting is not None + + def _loop(self): + while True: + if self.pi.backlog: + kind, value = self.pi.backlog.popleft() + else: + try: + kind, value = self._inbox.get(timeout=0.5) + except queue.Empty: + kind, value = "tick", None + now = time.monotonic() + if kind == "exit": + if not self._error and not self._aborting: + raise self.pi.exited() + return + if kind == "stop": + self._abort("user_stopped") + elif kind == "decide": + self._decided(*value) + elif kind == "record" and self._on(value): + return + if any(deadline < now for _, deadline in self._pending.values()): + self._abort("approval_timeout") + if self._kill_at and now > self._kill_at: # pi did not settle after the abort + self._hung = True + return + + def _abort(self, reason: str): + if self._aborting: + return + self._aborting = reason + for call_id, (dialog, _) in list(self._pending.items()): + timed_out = reason == "approval_timeout" + self.emit("approval", step=self._calls[call_id]["step"], call_id=call_id, approved=False, timed_out=timed_out, stopped=not timed_out) + self.pi.send({"type": "extension_ui_response", "id": dialog, "cancelled": True}) + self._pending.clear() + self.pi.send({"type": "abort"}) + self._kill_at = time.monotonic() + STOP_GRACE + with self._unserved: # a request waiting for its step learns now, not after WAIT_FOR_STEP, that none will open + self._unserved.notify_all() + + def _stop_reason(self) -> str: + if self._aborting: + return self._aborting + if self._error: + return "error" + if self._last and self._last.get("stopReason") == "length": + return "max_new_tokens" + return "no_tool_call" + + # --- pi's records --- + + def _on(self, rec: dict) -> bool: + """Turn one of pi's records into the page's events; True once pi has settled.""" + kind = rec.get("type") + if kind == "turn_start": + self._turn_start() + elif kind == "message_start" and self._open and rec["message"].get("role") == "assistant": + self._stream_t, self._wrote = time.perf_counter(), False + elif kind == "message_update" and self._open: + self._update(rec["assistantMessageEvent"]) + elif kind == "message_end" and self._open and rec["message"].get("role") == "assistant": + self._message_end(rec["message"]) + elif kind == "tool_execution_start" and self._open and not rec.get("parentToolCallId"): + self._tool_start(rec["toolCallId"], rec["toolName"], rec.get("args") or {}) + elif kind == "tool_execution_end" and rec.get("toolCallId") in self._calls and not rec.get("parentToolCallId"): + self._tool_end(rec["toolCallId"], rec.get("result") or {}, rec.get("isError", False), rec.get("durationMs")) + elif kind == "extension_ui_request": + self._dialog(rec) + return kind == "agent_settled" + + def _turn_start(self): + if self._aborting: # pi goes on to the turn its tool results start before the abort reaches it + self._open = False + return + if self._step >= self.settings.max_steps: # pi has no step limit of its own; the page's is kept by stopping it + self._abort("max_steps") + self._open = False + return + self._step += 1 + self._open = True + self._turn_t = self._stream_t = time.perf_counter() + self.emit("step_start", step=self._step) + with self._unserved: + self._waiting = self._step + self._unserved.notify_all() + + def _update(self, e: dict): + kind = e.get("type") + if kind == "text_delta": + self._delta("content", e.get("delta", "")) + elif kind == "thinking_delta": + self._delta("thought", e.get("delta", "")) + elif kind == "toolcall_start": + self._delta("tool_call", ("\n" if self._wrote else "") + e.get("toolName", "") + " ") + elif kind == "toolcall_delta": + self._delta("tool_call", e.get("delta", "")) + + def _delta(self, channel: str, text: str): + if text and not self.page: # the page's model sends its tokens instead + self._wrote = True + self.emit("delta", step=self._step, channel=channel, text=text) + + def _message_end(self, message: dict): + now = time.perf_counter() + usage = message.get("usage") or {} + reason = message.get("stopReason") or "" + if reason == "error" and not self._aborting: + self._error = message.get("errorMessage") or reason + finish = "user_stopped" if self._aborting else {"length": "max_new_tokens", "aborted": "user_stopped", "error": "error"}.get(reason, "end_turn") + n, seconds = usage.get("output") or 0, now - self._stream_t + if not self.page: # the page's model sent its own, with what the backend knows + self.emit( + "model_end", step=self._step, finish=finish, stop_piece=None, n_tokens=n, + first_ms=round((self._stream_t - self._turn_t) * 1000), decode_ms=round(seconds * 1000), + # tokens over the time they took to arrive, network included; nothing when they came all at once + tok_per_s=round(n / seconds, 1) if n and seconds > 0.05 else None, mtp=None, + usage={"prompt": (usage.get("input") or 0) + (usage.get("cacheRead") or 0) + (usage.get("cacheWrite") or 0), + "cached": usage.get("cacheRead") or 0, "output": n, "cost": (usage.get("cost") or {}).get("total")}, + ) + blocks = message.get("content") or [] + calls = [{"name": b.get("name", ""), "arguments": b.get("arguments") or {}, "raw_args": json.dumps(b.get("arguments") or {}, ensure_ascii=False), + "error": None} for b in blocks if b.get("type") == "toolCall"] + thinking = "\n\n".join(b["thinking"] for b in blocks if b.get("type") == "thinking" and b.get("thinking")) + self.emit("parsed", step=self._step, thinking=thinking, content=self._text(message), tool_calls=calls) + self._last = message + + @staticmethod + def _text(message: dict) -> str: + return "".join(b.get("text", "") for b in message.get("content") or [] if b.get("type") == "text") + + def _tool_start(self, call_id: str, name: str, args: dict): + key = (name, json.dumps(args, sort_keys=True, ensure_ascii=False)) + repeated = key in self._seen + self._seen.append(key) + self._calls[call_id] = {"name": name, "arguments": args, "step": self._step} + environment = environment_of(name) + self.emit("tool_call", step=self._step, call_id=call_id, name=name, arguments=args, raw_args=json.dumps(args, ensure_ascii=False), + errors=[], repeated=repeated, environment=environment) + if not self.settings.approve_tools: # otherwise it starts once the user allows it (_decided) + self.emit("tool_start", step=self._step, call_id=call_id, name=name, environment=environment) + + def _tool_end(self, call_id: str, result: dict, is_error: bool, ms: float | None): + call = self._calls[call_id] + parts = result.get("content") or [] + text = "\n".join(p.get("text", "") if p.get("type") == "text" else f"[{p.get('type')}]" for p in parts) + truncated = bool((result.get("structuredContent") or {}).get("truncated") or (result.get("details") or {}).get("truncation")) + self.emit("tool_result", step=call["step"], call_id=call_id, name=call["name"], ok=not is_error, result=text, + truncated=truncated, ms=round(ms or 0, 1), images=[]) + + def _dialog(self, rec: dict): + method = rec.get("method") + if method == "notify" and (rec.get("message") or "").startswith(REQUEST_MARK): + if self._open: # not one past an abort + self._request(json.loads(rec["message"][len(REQUEST_MARK):])) + elif method == "confirm" and rec.get("title") == APPROVE_TITLE: + call_id = json.loads(rec.get("message") or "{}").get("toolCallId") + if self._aborting or call_id not in self._calls: + self.pi.send({"type": "extension_ui_response", "id": rec["id"], "cancelled": True}) + return + self._pending[call_id] = (rec["id"], time.monotonic() + APPROVAL_TIMEOUT) + call = self._calls[call_id] + self.emit("approval_request", step=call["step"], call_id=call_id, name=call["name"], arguments=call["arguments"]) + elif method in DIALOGS: # no other extension is loaded, so nothing else should ask; an answer keeps pi from waiting + self.pi.send({"type": "extension_ui_response", "id": rec["id"], "cancelled": True}) + + def _request(self, payload): + """The request pi is sending the model, as JSON, the part that is new since its last request marked: the lines + between what the two begin and end with alike (the step's new messages; the tools after them do not change).""" + text = json.dumps(without_media(payload), ensure_ascii=False, indent=2) + last, self._request_text = self._request_text, text + added_from = added_to = None + if last: + start = common_prefix(text, last) + end = common_prefix(text[::-1], last[::-1]) + end = min(end, len(text) - start, len(last) - start) + added_from, added_to = text.rfind("\n", 0, start) + 1, len(text) - end + line_end = text.find("\n", added_to) + added_to = len(text) if line_end < 0 else line_end + messages: list = next((payload[k] for k in ("messages", "input", "contents") if isinstance(payload, dict) and isinstance(payload.get(k), list)), []) + roles = [m.get("role") for m in messages if isinstance(m, dict) and m.get("role")] + self.emit("request", step=self._step, text=text, added_from=added_from, added_to=added_to, + roles={role: roles.count(role) for role in dict.fromkeys(roles)}) + + def _decided(self, call_id: str, approved: bool): + if call_id not in self._pending: + return + dialog, _ = self._pending.pop(call_id) + call = self._calls[call_id] + self.emit("approval", step=call["step"], call_id=call_id, approved=approved, timed_out=False, stopped=False) + if approved: + self.emit("tool_start", step=call["step"], call_id=call_id, name=call["name"], environment=environment_of(call["name"])) + self.pi.send({"type": "extension_ui_response", "id": dialog, "confirmed": approved}) + + +def without_media(value): + """A request with each image's bytes left out, as the page shows it: a data URL keeps its type, and a long base64 + string says how long it was.""" + if isinstance(value, dict): + return {k: without_media(v) for k, v in value.items()} + if isinstance(value, list): + return [without_media(v) for v in value] + if isinstance(value, str) and value.startswith("data:") and ";base64," in value: + kind, data = value.split(";base64,", 1) + return f"{kind};base64,…({len(data):,} characters)" + if isinstance(value, str) and len(value) > 1000 and not value.strip(BASE64): + return f"…({len(value):,} base64 characters)" + return value + + +BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" diff --git a/harness/agent.py b/agent/harnesses/simple_harness.py similarity index 94% rename from harness/agent.py rename to agent/harnesses/simple_harness.py index 2021aa6..7946d6e 100644 --- a/harness/agent.py +++ b/agent/harnesses/simple_harness.py @@ -1,4 +1,4 @@ -"""The harness: the program around the model. +"""SimpleHarness: this project's own harness, the program around the model. Each step renders the conversation into one prompt, lets the model generate, and reads what came out. A tool call is parsed, validated, optionally approved by the @@ -6,7 +6,8 @@ A reply without a tool call ends the run; so do the step limit, the per-step token limit, repeated malformed calls and the stop button. Every stage is emitted as an event so the browser can draw it. The model is whatever Backend (backend.py) the page -hands it, so the loop is the same whichever runs the model. +hands it, so the loop is the same whichever runs the model. It is one Harness +(harness.py); the page can hand a turn to pi instead (pi_harness.py). A run keeps the page's language from when it started: the errors and refusals the model reads come back in it. The system prompt is the same in both languages. @@ -19,14 +20,14 @@ import threading import time import uuid +from collections.abc import Callable, Sequence from dataclasses import dataclass, field from pathlib import Path -from typing import Callable -from harness import media -from harness.backends.backend import Backend, Sampling -from harness.i18n import pick -from harness.tools import TOOLS, ToolError, ToolResult +from agent import media +from agent.i18n import pick +from agent.model_backends.backend import Backend, Sampling +from agent.tools import TOOLS, ToolError, ToolResult MAX_RESULT_CHARS = 2000 MAX_FORMAT_ERRORS = 3 @@ -62,9 +63,9 @@ def common_prefix(a: str, b: str) -> int: return i -class Run: +class SimpleHarness: def __init__(self, llm: Backend, history: list[dict], user_text: str, settings: RunSettings, emit: Callable[[dict], None], - attachments: list[media.Attachment] = (), files: Path | None = None): + attachments: Sequence[media.Attachment] = (), files: Path | None = None): self.id = uuid.uuid4().hex[:12] self.llm = llm self.settings = settings @@ -102,7 +103,7 @@ def run(self): schemas = {t.name: t.parameters for t in TOOLS.values()} system = {"role": "system", "content": SYSTEM_WITH_TOOLS if tools else SYSTEM_NO_TOOLS} self.emit( - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, settings=_settings_dict(s), tools=list(self.tools), ) @@ -137,6 +138,7 @@ def run(self): done = ev else: self.emit(kind, step=step, **ev) + assert done is not None # generate() always ends with done raw = done.pop("raw") self.emit("model_end", step=step, raw=raw, **done) @@ -228,6 +230,7 @@ def _handle_call(self, step: int, cid: str, call: dict, seen: list) -> str: ) if errors: return self._observe(step, cid, call["name"], {"error": "; ".join(errors)}, ok=False, ms=0) + assert tool is not None # a call to no tool has an error if self.settings.approve_tools: self.emit("approval_request", step=step, call_id=cid, name=call["name"], arguments=args) diff --git a/harness/i18n.py b/agent/i18n.py similarity index 91% rename from harness/i18n.py rename to agent/i18n.py index 530fb17..d66f282 100644 --- a/harness/i18n.py +++ b/agent/i18n.py @@ -8,10 +8,14 @@ the system prompt is the same in both, so the model behaves the same whichever language the page is in. """ +from typing import TypeVar + +T = TypeVar("T") + LANGS = {"zh": "中文", "en": "English"} # language -> its name on the switch -def pick(lang: str, zh: str, en: str) -> str: +def pick(lang: str, zh: T, en: T) -> T: return en if lang == "en" else zh diff --git a/harness/media.py b/agent/media.py similarity index 97% rename from harness/media.py rename to agent/media.py index 6af4f56..019ae42 100644 --- a/harness/media.py +++ b/agent/media.py @@ -144,11 +144,11 @@ def load_video(path: str) -> tuple[np.ndarray, dict]: total = total or MAX_FRAMES # nothing to go by: the first frames, then keep = set(np.linspace(0, total - 1, min(total, MAX_FRAMES)).round().astype(int).tolist()) scale = min(1.0, MAX_SIDE / max(stream.width, stream.height)) - size = {"width": max(2, round(stream.width * scale)), "height": max(2, round(stream.height * scale))} + width, height = max(2, round(stream.width * scale)), max(2, round(stream.height * scale)) frames = [] for i, frame in enumerate(container.decode(video=0)): if i in keep: - frames.append(frame.to_ndarray(format="rgb24", **size)) + frames.append(frame.to_ndarray(format="rgb24", width=width, height=height)) if i >= max(keep): break if not frames: diff --git a/harness/backends/__init__.py b/agent/model_backends/__init__.py similarity index 100% rename from harness/backends/__init__.py rename to agent/model_backends/__init__.py diff --git a/harness/backends/backend.py b/agent/model_backends/backend.py similarity index 87% rename from harness/backends/backend.py rename to agent/model_backends/backend.py index 168f9eb..bbacce1 100644 --- a/harness/backends/backend.py +++ b/agent/model_backends/backend.py @@ -1,8 +1,8 @@ """The interface between the harness and whatever runs the model. -The harness (agent.Run) and the page (app/page.py) reach the model only through a Backend. It renders the -conversation into a prompt with the model's chat template, encodes the prompt into the tokens the model reads, -and generates, yielding the events render.py draws. The app runs one of two: simple_backend.SimpleBackend, transformers +This project's harness (harnesses/simple_harness.SimpleHarness) and the page (app/page.py) reach the model only through +a Backend. It renders the conversation into a prompt with the model's chat template, encodes the prompt into the tokens +the model reads, and generates, yielding the events render.py draws. The app runs one of two: simple_backend.SimpleBackend, transformers on this machine, with a hand-written decoding loop that sees the whole distribution at every token, or vllm_backend.VLLMBackend, a vLLM server. tests/fakes.FakeLLM replays scripted replies. A backend that asks another serving engine, such as SGLang or Ollama, would be one more; this module says what it has to provide. @@ -17,10 +17,11 @@ Nothing here imports torch, so a backend can be written, and the interface read, without it. """ +from collections.abc import Callable, Iterator from dataclasses import dataclass, field -from typing import Callable, Iterator, Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable -from harness import families +from agent import model_families @dataclass @@ -37,7 +38,7 @@ class Encoded: """A prompt as the model reads it (Backend.encode).""" text: str # the prompt, with each placeholder of an image, clip or video expanded into the tokens it takes - ids: object # its token ids, in the form the backend's generate() takes (SimpleBackend: a tensor of shape (1, n_tokens)) + ids: Any # its token ids, in the form the backend's generate() takes (SimpleBackend: a tensor of shape (1, n_tokens)) n_tokens: int # what generate() needs to read the media: the processor's inputs for the encoders (SimpleBackend), or the files # and the server's expansion of the ids (VLLMBackend) @@ -67,11 +68,11 @@ class Backend(Protocol): """What runs the model. The page holds one at a time (app.page.current_llm) and runs one turn on it at a time.""" model_id: str - family: families.Family # how the model writes its turns, thinking and tool calls (families/) + family: model_families.Family # how the model writes its turns, thinking and tool calls (model_families/) modalities: tuple[str, ...] # what it reads besides text: "image", "audio", "video" context_length: int # the most tokens it reads at once info: dict # what the Model tab shows, and in "defaults" the recommended temperature, top_p and top_k - drafter_id: str | None # the MTP drafter that goes with the model (families.DRAFTERS), if any + drafter_id: str | None # the MTP drafter that goes with the model (model_families.DRAFTERS), if any drafter: object | None # that drafter once load_drafter() has loaded it; Sampling.mtp turns it on only then def warmup(self) -> None: diff --git a/harness/backends/simple_backend.py b/agent/model_backends/simple_backend.py similarity index 95% rename from harness/backends/simple_backend.py rename to agent/model_backends/simple_backend.py index 3244dbf..4553a46 100644 --- a/harness/backends/simple_backend.py +++ b/agent/model_backends/simple_backend.py @@ -16,7 +16,7 @@ chat template wrote into the tokens of its file, and the prefill hands the encoders their inputs. After the prefill, decoding is the same as for text. -The models download into MODELS_DIR (the project's models/) at the commits families.REVISIONS pins, +The models download into MODELS_DIR (the project's models/) at the commits model_families.REVISIONS pins, and describe_folder() lists what a model's folder holds for the Model tab. """ @@ -27,15 +27,16 @@ import threading import time from collections import Counter +from collections.abc import Callable, Iterator from pathlib import Path -from typing import Callable, Iterator +from typing import Any import torch from huggingface_hub import snapshot_download from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForMultimodalLM, AutoProcessor -from harness import families, media -from harness.backends.backend import Encoded, Sampling +from agent import media, model_families +from agent.model_backends.backend import Encoded, Sampling TOP_N = 5 ROOT = Path(__file__).resolve().parents[2] # the project's folder @@ -49,7 +50,7 @@ def _download(model_id: str, only: str | None = None) -> str: """The model's folder in MODELS_DIR, after fetching the files it lacks (only `only`, if given), at the revision - families.REVISIONS pins it to. + model_families.REVISIONS pins it to. from_pretrained(local_files_only=True) is no test of the cache: once config.json is cached, a tokenizer whose own files are missing raises a ValueError (Gemma 4) or loads empty (Qwen3.5) rather than an OSError. @@ -57,12 +58,12 @@ def _download(model_id: str, only: str | None = None) -> str: can't be reached it returns the cached folder, so a downloaded model still starts offline.""" if os.path.isdir(model_id): # a checkpoint on disk rather than a Hugging Face id return model_id - return snapshot_download(model_id, revision=families.REVISIONS.get(model_id), allow_patterns=only, cache_dir=MODELS_DIR) + return snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), allow_patterns=only, cache_dir=MODELS_DIR) def describe_folder(folder: str, notes: dict) -> dict: """A checkpoint's folder as the Model tab lists it: where it is, the Hub revision it was downloaded at, and every - file with its size and the values of the fields its family's note on it names (families.FileNote). In the cache a + file with its size and the values of the fields its family's note on it names (model_families.FileNote). In the cache a file is a link into blobs/, and the size is that of the file it links to.""" path = Path(folder).absolute() files = [] @@ -136,12 +137,12 @@ def __init__(self, model_id: str): if self.device == "mps": # Loading weights onto MPS from several threads can hang (seen with Qwen3.5); one thread is fast enough. os.environ.setdefault("HF_DEACTIVATE_ASYNC_LOAD", "1") - self.drafter_id = families.DRAFTERS.get(model_id) - self.drafter = None # loaded the first time MTP is switched on + self.drafter_id = model_families.DRAFTERS.get(model_id) + self.drafter: Any = None # loaded the first time MTP is switched on self._drafter_lock = threading.Lock() start = time.perf_counter() # config.json alone says which family the model belongs to; an unsupported one fails before the weights download. - self.family = families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) + self.family = model_families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) files = self.folder = _download(model_id) # The processor holds the tokenizer and, for a model that reads more than text, what turns files into its inputs. self.processor = AutoProcessor.from_pretrained(files) @@ -222,7 +223,7 @@ def warmup(self, mtp: bool = False): pass def load_drafter(self): - """Load the MTP drafter that goes with this model (families.DRAFTERS), if any, and warm it up.""" + """Load the MTP drafter that goes with this model (model_families.DRAFTERS), if any, and warm it up.""" with self._drafter_lock: if self.drafter is not None or self.drafter_id is None: return @@ -260,7 +261,7 @@ def encode(self, prompt: str, parts: list[dict]) -> Encoded: ids = self.tokenizer(prompt, add_special_tokens=False, return_tensors="pt").input_ids return Encoded(prompt, ids, ids.shape[-1]) files = {kind: [p["path"] for p in parts if p["type"] == kind] for kind in media.MEDIA} - inputs = {} + inputs: dict = {} if files["image"]: inputs["images"] = [media.load_image(path) for path in files["image"]] if files["audio"]: @@ -301,7 +302,7 @@ def generate(self, prompt: Encoded, sampling: Sampling, should_stop: Callable[[] yield {"type": "prefill", "n_tokens": ids.shape[-1], "ms": round(prefill_ms, 1)} decode_start = time.perf_counter() - tokens = self._drafted(out, sampling, mtp) if drafting else self._one_at_a_time(out, sampling) + tokens = self._drafted(out, sampling, mtp) if mtp is not None else self._one_at_a_time(out, sampling) for token_id, details in tokens: if should_stop(): finish = "user_stopped" @@ -367,7 +368,7 @@ def _encoded_media(self, inputs: dict) -> dict: for kind in ("image", "video"): encoder = getattr(base, f"get_{kind}_features", None) args = {k: inputs[k] for k in inspect.signature(encoder).parameters if k in inputs} if encoder else {} - if any(k.startswith("pixel_values") for k in args): + if encoder and any(k.startswith("pixel_values") for k in args): encoded[kind] = encoder(**args, return_dict=True) if not encoded: return inputs @@ -420,7 +421,7 @@ def _draft(self, token_id: int, hidden: torch.Tensor, shared: dict, position: in comes in through its input, the embedding of the last guess next to the drafter's last hidden state.""" embed = self.model.get_input_embeddings() positions = torch.tensor([[position]], device=self.device) - guesses = [] + guesses: list[int] = [] while len(guesses) < self.draft_tokens and token_id not in self.eos_ids: x = torch.cat([embed(torch.tensor([[token_id]], device=self.device)), hidden], -1) out = self.drafter(inputs_embeds=x, position_ids=positions, shared_kv_states=shared) @@ -453,7 +454,7 @@ def _sample(self, logits: torch.Tensor, s: Sampling, draft: int | None = None) - kept = int(((cand_probs.cumsum(-1) - cand_probs) < s.top_p).sum()) cand_ids, cand_probs = cand_ids[:kept], cand_probs[:kept] / cand_probs[:kept].sum() weights = cand_probs - at = (cand_ids == draft).nonzero().flatten() if draft is not None else [] + at: torch.Tensor | list = (cand_ids == draft).nonzero().flatten() if draft is not None else [] if len(at) and float(torch.rand(())) < float(cand_probs[at[0]]): pick = int(at[0]) else: diff --git a/harness/backends/vllm_backend.py b/agent/model_backends/vllm_backend.py similarity index 96% rename from harness/backends/vllm_backend.py rename to agent/model_backends/vllm_backend.py index a658510..31fef29 100644 --- a/harness/backends/vllm_backend.py +++ b/agent/model_backends/vllm_backend.py @@ -5,7 +5,7 @@ same; what changes is how much of the model the page can see, which depends on how the server was started. The prompt stays exact. Everything of the checkpoint but its weights (the config, the tokenizer, the chat -template) is downloaded into models/ at the revision families.REVISIONS pins, the prompt is rendered and +template) is downloaded into models/ at the revision model_families.REVISIONS pins, the prompt is rendered and tokenized here as SimpleBackend does it, and the server gets the token ids, so it reads exactly what the Context tab shows. Each token comes back as its id with its log probability and those of the most likely tokens at its position. What those numbers are depends on the server's --logprobs-mode, which no route tells, so the backend @@ -46,8 +46,8 @@ import time import wave from collections import Counter +from collections.abc import Callable, Generator, Iterator from pathlib import Path -from typing import Callable, Iterator import av import httpx @@ -57,9 +57,9 @@ from PIL import Image from transformers import AutoConfig, AutoProcessor, GenerationConfig -from harness import families, media -from harness.backends.backend import Encoded, Sampling -from harness.backends.simple_backend import MODELS_DIR, TOP_N, describe_folder +from agent import media, model_families +from agent.model_backends.backend import Encoded, Sampling +from agent.model_backends.simple_backend import MODELS_DIR, TOP_N, describe_folder # The weights stay on the server; downloading them here as well would only take up space. WEIGHTS = ["*.safetensors", "*.bin", "*.pt", "*.pth", "*.gguf"] @@ -84,7 +84,7 @@ def __init__(self, zh: str, en: str, status: int | None = None): class UnsupportedModel(ServerError): - """A model of a family the page does not know (families.FAMILIES): `model_id` is the name the server was started + """A model of a family the page does not know (model_families.FAMILIES): `model_id` is the name the server was started with, and `model_type` the one in its config.json, or None when it has none or it could not be read.""" def __init__(self, zh: str, en: str, model_id: str, model_type: str | None): @@ -158,12 +158,12 @@ def _unreachable(client: httpx.Client, error: Exception) -> ServerError: def _checkpoint_files(model_id: str) -> str: - """The model's folder in MODELS_DIR with everything but its weights, at the revision families.REVISIONS pins. + """The model's folder in MODELS_DIR with everything but its weights, at the revision model_families.REVISIONS pins. The server should load the same revision (vllm serve --revision ) so that both read the same tokenizer and chat template.""" if os.path.isdir(model_id): # a checkpoint on disk rather than a Hugging Face id return model_id - return snapshot_download(model_id, revision=families.REVISIONS.get(model_id), cache_dir=MODELS_DIR, ignore_patterns=WEIGHTS) + return snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), cache_dir=MODELS_DIR, ignore_patterns=WEIGHTS) def _model_type(config_json: str) -> str | None: @@ -175,7 +175,7 @@ def _model_type(config_json: str) -> str | None: return None -def family_of(model_id: str) -> families.Family | None: +def family_of(model_id: str) -> model_families.Family | None: """The family of a model the server serves, from its config.json alone, which is all the model menu downloads for it (the rest of its files only when it is picked): None for a family the page does not know, or a checkpoint without a config.json (a repo of GGUF files). Raises OSError or ValueError when that cannot be told: no way to the Hub, a @@ -184,11 +184,11 @@ def family_of(model_id: str) -> families.Family | None: config_json = os.path.join(model_id, "config.json") else: try: - config_json = hf_hub_download(model_id, "config.json", revision=families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) + config_json = hf_hub_download(model_id, "config.json", revision=model_families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) except RemoteEntryNotFoundError: return None model_type = _model_type(config_json) - return next((f for f in families.FAMILIES.values() if model_type in f.model_types), None) + return next((f for f in model_families.FAMILIES.values() if model_type in f.model_types), None) def _generation_config(folder: str, config) -> GenerationConfig: @@ -201,8 +201,8 @@ def _generation_config(folder: str, config) -> GenerationConfig: def unsupported(model_id: str, why: str, model_type: str | None = None) -> UnsupportedModel: - """The error for a model of a family the page does not know (families.FAMILIES).""" - names = [f.name for f in families.FAMILIES.values()] + """The error for a model of a family the page does not know (model_families.FAMILIES).""" + names = [f.name for f in model_families.FAMILIES.values()] return UnsupportedModel(f"這個網頁還不支援 {model_id}（{why}）；目前支援 {'、'.join(names)}。", f"This page does not support {model_id} yet ({why}); it supports {', '.join(names)}.", model_id, model_type) @@ -286,7 +286,8 @@ def drafted_tokens(accepted: list[int], drafted: list[int], n_tokens: int) -> di from the prefill; after it, each verify step gives the guesses it kept and then one token of the model's own, which replaces the next guess when it kept fewer than were drafted. vLLM does not count a step without guesses, so when the steps do not account for every token, nobody can tell which tokens were guessed: None.""" - kept, turned_down, at = [], [], 1 + kept: list[int] = [] + turned_down, at = [], 1 for n_kept, n_drafted in zip(accepted, drafted): kept += range(at, at + n_kept) if n_kept < n_drafted: @@ -339,9 +340,9 @@ def __init__(self, model_id: str, url: str, transport: httpx.BaseTransport | Non config = AutoConfig.from_pretrained(files) except (ValueError, OSError) as e: # a model_type this transformers does not know, or no config.json (a repo of GGUF files) raise unsupported(model_id, str(e), _model_type(os.path.join(files, "config.json"))) from e - if not any(config.model_type in f.model_types for f in families.FAMILIES.values()): + if not any(config.model_type in f.model_types for f in model_families.FAMILIES.values()): raise unsupported(model_id, f"model_type {config.model_type!r}", config.model_type) - self.family = families.for_model_type(config.model_type) + self.family = model_families.for_model_type(config.model_type) # The same tokenizer and chat template SimpleBackend loads, so the prompt is the same string and the same tokens. self.processor = AutoProcessor.from_pretrained(files) self.tokenizer = getattr(self.processor, "tokenizer", self.processor) @@ -353,8 +354,8 @@ def __init__(self, model_id: str, url: str, transport: httpx.BaseTransport | Non self.context_length = card.get("max_model_len") or text_cfg.max_position_embeddings # the server's limit (--max-model-len) self.audio_rate = getattr(getattr(self.processor, "feature_extractor", None), "sampling_rate", AUDIO_RATE) self.media_token_ids = {kind: getattr(config, f"{kind}_token_id", None) for kind in media.MEDIA} - self.drafter_id = None # speculative decoding is the server's (--speculative-config): nothing to switch here - self.drafter = None + self.drafter_id: str | None = None # speculative decoding is the server's (--speculative-config): nothing to switch here + self.drafter: object | None = None # What the server does beyond /v1/completions, and so which of the checkpoint's media the model reads here. readable = checkpoint_modalities(config, self.processor) hi = self.encode(self.render([{"role": "user", "content": "Hi"}], None, False), []).ids @@ -622,7 +623,7 @@ def _lines(self, path: str, body: dict) -> Iterator[dict]: except httpx.HTTPError as e: raise _unreachable(self.client, e) from e - def _stream(self, body: dict, s: Sampling, end: dict) -> Iterator[tuple[int, dict]]: + def _stream(self, body: dict, s: Sampling, end: dict) -> Generator[tuple[int, dict], None, None]: """Each token /v1/completions sends, with what it says about the token. `end` gets the finish reason and, from a server started with --per-request-spec-decode-metrics summary or detailed, the speculative decoding counts.""" for chunk in self._lines("/v1/completions", body): @@ -649,7 +650,7 @@ def _details(self, token_id: int, logprobs: dict, n: int, s: Sampling) -> dict: rank = 1 + sum(v > logprob for _, v in top) if any(i == token_id for i, _ in top) else None return self._token_details(token_id, logprob, listed, rank, None, s) - def _stream_generate(self, prompt: Encoded, s: Sampling, end: dict) -> Iterator[tuple[int, dict]]: + def _stream_generate(self, prompt: Encoded, s: Sampling, end: dict) -> Generator[tuple[int, dict], None, None]: """Each token /inference/v1/generate sends for the step, with what it says about the token, and `end` as in _stream. With media, the server's first chunk has the prompt as it expanded it, which must be the one encode() was told and the Context tab shows.""" @@ -708,7 +709,7 @@ def _token_details(self, token_id: int, logprob: float, listed: dict[int, float] depends on the server's --logprobs-mode. Only the model's own log probabilities are p, logprob and the top 5. Processed ones are the distribution the token was drawn from, so the token's is its sampling probability, and the tokens top-k and top-p cut come back as UNREPORTED. Logits are neither.""" - details = {"rank": rank} if rank else {} + details: dict = {"rank": rank} if rank else {} if self.logprobs_mode == "raw_logprobs": # Equally likely tokens go lowest id first, as in SimpleBackend. ranked = sorted(listed.items(), key=lambda c: (-c[1], c[0])) diff --git a/harness/backends/vllm_docker.py b/agent/model_backends/vllm_docker.py similarity index 94% rename from harness/backends/vllm_docker.py rename to agent/model_backends/vllm_docker.py index 1e28cb7..48b5f1c 100644 --- a/harness/backends/vllm_docker.py +++ b/agent/model_backends/vllm_docker.py @@ -8,10 +8,10 @@ Docker Compose keeps track of the container: `docker compose up -d` with the model already running leaves it be, with another model replaces it, and the first time pulls or builds the image. Where Docker has the nvidia runtime (Linux with the NVIDIA Container Toolkit), compose.gpu.yaml goes on top and runs vLLM's CUDA image on the GPU, where -it can also decode speculatively (MTP, families.VLLM_SPECULATIVE), which the page asks for by default; +it can also decode speculatively (MTP, model_families.VLLM_SPECULATIVE), which the page asks for by default; elsewhere, a Mac say, compose.yaml runs vLLM's CPU image (vllm-cpu/). AGENT_LOOP_VLLM_DEVICE=cpu or gpu decides instead. -The server never downloads anything. The weights come into models/ here first, at the commit families.REVISIONS pins, +The server never downloads anything. The weights come into models/ here first, at the commit model_families.REVISIONS pins, and the container reads that folder read-only, so it loads the very files the page renders the prompt with. As with run_python's sandbox, the page runs `docker` only once vLLM is picked. """ @@ -24,15 +24,15 @@ import threading import time from collections import deque +from collections.abc import Callable from pathlib import Path -from typing import Callable import httpx from huggingface_hub import snapshot_download -from harness import families -from harness.backends.simple_backend import MODELS_DIR -from harness.tools import DOCKER_INSTALL +from agent import model_families +from agent.model_backends.simple_backend import MODELS_DIR +from agent.tools import DOCKER_INSTALL COMPOSE_DIR = Path(__file__).resolve().parents[2] / "docker-compose" # compose.yaml, compose.gpu.yaml and vllm-cpu/ PROJECT = "agent-loop-vllm" # the name in compose.yaml @@ -85,6 +85,7 @@ def _stream(args: list[str], env: dict, on_line: Callable[[str], None]) -> tuple process = subprocess.Popen(["docker", *args], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env) except FileNotFoundError as e: raise _not_installed() from e + assert process.stdout is not None # a pipe, as asked tail: deque[str] = deque(maxlen=LOG_LINES) try: for line in process.stdout: @@ -108,19 +109,19 @@ def _env(model_id: str, config: dict | None = None) -> dict: """What the compose files read: the model, the commit to load it at, the folder it was downloaded into, and on the GPU how it is decoded speculatively ("None" is vLLM's word for not at all), with each step's counts in detail, which vLLM refuses without a config.""" - return {**os.environ, "VLLM_MODEL": model_id, "VLLM_REVISION": families.REVISIONS.get(model_id, "main"), + return {**os.environ, "VLLM_MODEL": model_id, "VLLM_REVISION": model_families.REVISIONS.get(model_id, "main"), "AGENT_LOOP_MODELS": str(MODELS_DIR), "VLLM_SPECULATIVE_CONFIG": json.dumps(config) if config else "None", "VLLM_SPEC_DECODE_METRICS": "detailed" if config else "none"} def speculative(model_id: str, where: str) -> dict | None: - """The --speculative-config of model_id on `where` (families.VLLM_SPECULATIVE, a drafter at its pinned commit), or + """The --speculative-config of model_id on `where` (model_families.VLLM_SPECULATIVE, a drafter at its pinned commit), or None: on the CPU, where vLLM 0.32's image runs neither model's MTP, and for a model without one.""" - config = families.VLLM_SPECULATIVE.get(model_id) + config: dict | None = model_families.VLLM_SPECULATIVE.get(model_id) if where != "gpu" or not config: return None if drafter := config.get("model"): - return {**config, "revision": families.REVISIONS.get(drafter, "main")} + return {**config, "revision": model_families.REVISIONS.get(drafter, "main")} return dict(config) @@ -188,7 +189,7 @@ def start(model_id: str, say: Callable[[str, str], None], mtp: bool = False) -> raise DockerError("這台電腦的 Docker 沒有 Compose（`docker compose`）。請更新 Docker，或另外安裝 Docker Compose。", "This computer's Docker has no Compose (`docker compose`). Update Docker, or install Docker Compose.") say("download", "") - snapshot_download(model_id, revision=families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) + snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) if config and (drafter := config.get("model")): # the server reads models/ offline, its drafter's weights too snapshot_download(drafter, revision=config["revision"], cache_dir=MODELS_DIR) env = _env(model_id, config) diff --git a/harness/families/__init__.py b/agent/model_families/__init__.py similarity index 94% rename from harness/families/__init__.py rename to agent/model_families/__init__.py index 9806195..b2ab7a5 100644 --- a/harness/families/__init__.py +++ b/agent/model_families/__init__.py @@ -1,6 +1,6 @@ """Model families: everything that changes when the model changes. -The harness (agent.py), the decoding loop (simple_backend.py) and the page (render.py) are written once. +This project's harness (harnesses/simple_harness.py), the decoding loop (simple_backend.py) and the page (render.py) are written once. What differs between Gemma 4 and Qwen3.5 is how the model writes its turns, its thinking and its tool calls, which tokens stop it, and what its control tokens mean. A family module describes all of that in one Family, and everything else asks the family. @@ -14,9 +14,9 @@ """ import re +from collections.abc import Callable from dataclasses import dataclass, field from functools import cached_property -from typing import Callable # The models in the page's model menu: Hugging Face id -> label. A label is the same in both of the page's # languages: Streamlit's selectbox sends its label back, and one that changed with the language would become the value. @@ -123,8 +123,8 @@ def special_re(self) -> re.Pattern: return re.compile("|".join(re.escape(p) for p in pieces)) -from harness.families.gemma4 import GEMMA4 # noqa: E402 (the family modules import Family from here) -from harness.families.qwen3_5 import QWEN3_5 # noqa: E402 +from agent.model_families.gemma4 import GEMMA4 # noqa: E402 (the family modules import Family from here) +from agent.model_families.qwen3_5 import QWEN3_5 # noqa: E402 FAMILIES = {f.key: f for f in (GEMMA4, QWEN3_5)} @@ -139,6 +139,6 @@ def for_model_type(model_type: str) -> Family: return family known = ", ".join(t for f in FAMILIES.values() for t in f.model_types) raise ValueError( - f"Models of model_type {model_type!r} are not supported; supported: {known}. families/__init__.py says how to add a family. " - f"（不支援 model_type {model_type!r} 的模型；目前支援：{known}。新增模型家族的方法見 families/__init__.py。）" + f"Models of model_type {model_type!r} are not supported; supported: {known}. model_families/__init__.py says how to add a family. " + f"（不支援 model_type {model_type!r} 的模型；目前支援：{known}。新增模型家族的方法見 model_families/__init__.py。）" ) diff --git a/harness/families/gemma4.py b/agent/model_families/gemma4.py similarity index 99% rename from harness/families/gemma4.py rename to agent/model_families/gemma4.py index f44e410..8a0a403 100644 --- a/harness/families/gemma4.py +++ b/agent/model_families/gemma4.py @@ -8,8 +8,8 @@ import json import re -from harness.families import Family, FileNote -from harness.i18n import pick +from agent.i18n import pick +from agent.model_families import Family, FileNote STR = '<|"|>' THOUGHT_OPEN = "<|channel>thought\n" @@ -138,7 +138,7 @@ def key(self) -> str: def obj(self) -> dict: self.pos += 1 - out = {} + out: dict = {} self.skip_ws() if self.peek() == "}": self.pos += 1 @@ -157,7 +157,7 @@ def obj(self) -> dict: def arr(self) -> list: self.pos += 1 - out = [] + out: list = [] self.skip_ws() if self.peek() == "]": self.pos += 1 diff --git a/harness/families/qwen3_5.py b/agent/model_families/qwen3_5.py similarity index 99% rename from harness/families/qwen3_5.py rename to agent/model_families/qwen3_5.py index 359e11a..a85fde7 100644 --- a/harness/families/qwen3_5.py +++ b/agent/model_families/qwen3_5.py @@ -17,8 +17,8 @@ import json import re -from harness.families import Family, FileNote -from harness.i18n import pick +from agent.i18n import pick +from agent.model_families import Family, FileNote THINK_OPEN, THINK_CLOSE = "", "" CALL_OPEN = "" diff --git a/harness/sandbox/Dockerfile b/agent/sandbox/Dockerfile similarity index 96% rename from harness/sandbox/Dockerfile rename to agent/sandbox/Dockerfile index d150777..2bf7f47 100644 --- a/harness/sandbox/Dockerfile +++ b/agent/sandbox/Dockerfile @@ -3,7 +3,7 @@ # for to look at a table, and matplotlib, whose charts come back to the page (agent_loop_figures.py), with a # font for Chinese labels. Build it once, before letting the model write code, and again when this file changes: # -# docker build -t agent-loop-sandbox harness/sandbox +# docker build -t agent-loop-sandbox agent/sandbox # # Every call starts a new container from this image with no network, a read-only file system, limited # memory, CPU and processes, as an unprivileged user, and the conversation's files read-only at /data. diff --git a/harness/sandbox/agent_loop_figures.py b/agent/sandbox/agent_loop_figures.py similarity index 100% rename from harness/sandbox/agent_loop_figures.py rename to agent/sandbox/agent_loop_figures.py diff --git a/harness/sandbox/matplotlibrc b/agent/sandbox/matplotlibrc similarity index 100% rename from harness/sandbox/matplotlibrc rename to agent/sandbox/matplotlibrc diff --git a/harness/tools.py b/agent/tools.py similarity index 98% rename from harness/tools.py rename to agent/tools.py index c971f61..29841f6 100644 --- a/harness/tools.py +++ b/agent/tools.py @@ -24,19 +24,20 @@ import re import subprocess import uuid +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Callable +from typing import Any from urllib.parse import quote from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx -from harness.i18n import pick +from agent.i18n import pick WIKI_HEADERS = {"User-Agent": f"agent-loop-visualizer/0.1 (https://github.com/cyyeh/agent-atlas-learning) httpx/{httpx.__version__}"} -SANDBOX_IMAGE = "agent-loop-sandbox" # docker build -t agent-loop-sandbox harness/sandbox +SANDBOX_IMAGE = "agent-loop-sandbox" # docker build -t agent-loop-sandbox agent/sandbox SANDBOX_VERSION = "2" # the agent-loop-sandbox.version label in sandbox/Dockerfile; for an older image the page asks for a rebuild DOCKER_INSTALL = "https://docs.docker.com/get-started/get-docker/" SANDBOX_TIMEOUT = 30 # seconds a piece of code may run @@ -110,7 +111,7 @@ def validate(self, args: dict, lang: str = "zh") -> tuple[dict, list[str]]: ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: operator.pow, } -UNARY_OPS = {ast.UAdd: operator.pos, ast.USub: operator.neg} +UNARY_OPS: dict[type, Callable] = {ast.UAdd: operator.pos, ast.USub: operator.neg} FUNCS = { name: getattr(math, name) for name in ["sqrt", "sin", "cos", "tan", "asin", "acos", "atan", "log", "log2", "log10", "exp", "floor", "ceil", "factorial"] @@ -258,7 +259,7 @@ def sandbox_command(name: str, files: Path | None) -> list[str]: def sandbox_problem() -> dict[str, str] | None: """Why run_python cannot run right now, in both languages (Markdown), or None when Docker and an up-to-date sandbox image are there.""" - build = f"`docker build -t {SANDBOX_IMAGE} harness/sandbox`" + build = f"`docker build -t {SANDBOX_IMAGE} agent/sandbox`" label = '{{ index .Config.Labels "agent-loop-sandbox.version" }}' try: found = subprocess.run(["docker", "image", "inspect", "--format", label, SANDBOX_IMAGE], capture_output=True, text=True, timeout=10) diff --git a/app/__init__.py b/app/__init__.py index 150814c..acdf89a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,10 +1,10 @@ -"""The app: what a person sees, on top of the harness. +"""The app: what a person sees, on top of the agent. page.py is the Streamlit page: the chat on the left, the loop on the right, the sidebar's settings, and the background thread a turn runs in. render.py turns the harness's events into the HTML the page draws, as pure functions, so the page can redraw any moment of a run. guide.py builds 運作說明 (How it works), a static page on the loop and the code behind it. assets/ holds the stylesheets, the logo and the example files. -Nothing here imports torch or transformers at import time but page.py, through the harness's backends: guide.py and +Nothing here imports torch or transformers at import time but page.py, through the agent's model backends: guide.py and render.py build with pygments alone (.github/workflows/pages.yml). """ diff --git a/app/assets/guide.css b/app/assets/guide.css index b833e48..b6b7bce 100644 --- a/app/assets/guide.css +++ b/app/assets/guide.css @@ -157,7 +157,7 @@ body { margin: 0; background: var(--page); color: var(--ink); font: 16px/1.6 "So .guide h1 { font-size: 24px; } .g-head .g-num { width: 28px; height: 28px; font-size: 13px; } .src pre { font-size: 11.5px; } - /* A chip between full-width brackets, （harness/…:123）, cannot move to the next line on its own; let it break instead. */ + /* A chip between full-width brackets, （agent/…:123）, cannot move to the next line on its own; let it break instead. */ .guide code.ref { white-space: normal; } .g-table code.ref { white-space: nowrap; } /* the table scrolls sideways anyway */ .g-tab-bar small { display: none; } /* the class names; without them more of the row fits on the screen */ diff --git a/app/assets/guide.js b/app/assets/guide.js index 22ac100..1d15b52 100644 --- a/app/assets/guide.js +++ b/app/assets/guide.js @@ -1,8 +1,8 @@ /* The 運作說明 page (guide.py) inlines this after the page body. The table of contents stays at the top of the window as one line; this keeps that line on the section you are reading and closes the open list after a jump, a click elsewhere or Escape. Without it the list still works. - It has every set of backend tabs show the backend picked last, and every set of model tabs the - model, and remembers those picks and the language picked on the 中文 | English switch, for the + It has every set of backend tabs show the backend picked last, every set of model tabs the + model, and every set of harness tabs the harness, and remembers those picks and the language picked on the 中文 | English switch, for the next visit and index.html to go by. A row of tabs wider than the screen scrolls sideways; this fades the side with more tabs and keeps the picked one in view. Without it each set of tabs still switches on its own, and its row still scrolls. */ @@ -35,7 +35,7 @@ document.addEventListener("keydown", (e) => { toc.querySelector("summary").focus(); } }); -// Tabs, one per backend or one per model: picking one in a set picks the same in every other set of that kind that +// Tabs, one per backend, per model or per harness: picking one in a set picks the same in every other set of that kind that // has it. A key with a dash is a narrower one of the key before it ("vllm-docker" of "vllm", any vLLM server), and the // two stand for each other: a set already on a narrower one stays on it when the broader one is picked. A set without // the pick stays as it is. diff --git a/app/assets/style.css b/app/assets/style.css index fa69c62..0f4caeb 100644 --- a/app/assets/style.css +++ b/app/assets/style.css @@ -139,6 +139,9 @@ .tok:hover { outline: 1.5px solid var(--ink); } .tok.ch-thought { font-style: italic; color: var(--muted); } .tok.ch-tool_call { font-family: var(--mono); font-size: 12.5px; color: var(--c-harness); } +/* Text streamed by a harness that sees no tokens (pi): the same channels, without token boundaries. */ +.run.ch-thought { font-style: italic; color: var(--muted); } +.run.ch-tool_call { font-family: var(--mono); font-size: 12.5px; color: var(--c-harness); } .tok.special, .sp { font: 600 11px/1 var(--mono); font-style: normal; diff --git a/app/guide.py b/app/guide.py index c1aea71..7337d08 100644 --- a/app/guide.py +++ b/app/guide.py @@ -10,7 +10,7 @@ switch before, or else their browser's. On every push to main, .github/workflows/pages.yml runs tests/test_guide.py, then builds and publishes them. -The content points at code by text, not by line number: Code("harness/agent.py", "def run(self):") is +The content points at code by text, not by line number: Code("agent/harnesses/simple_harness.py", "def run(self):") is looked up in the file every time the page is built, so the line numbers on the page follow the code when it changes, and each file:line chip and excerpt links to its lines on GitHub at the commit the page was built from. A start text must match exactly one line of its file; tests/test_guide.py @@ -41,8 +41,8 @@ from pygments.formatters import HtmlFormatter from pygments.lexers import get_lexer_for_filename +from agent.i18n import LANGS, pick from app import render -from harness.i18n import LANGS, pick ROOT = Path(__file__).resolve().parent.parent # the project's folder, which the files the guide quotes are named from ASSETS = Path(__file__).parent / "assets" @@ -59,10 +59,10 @@ } REF_RE = re.compile(r"\[\[([\w./-]+)\|(.+?)\]\]") # The tabs a part of a section can come in: one per backend where what runs the model changes it, one per model where -# the model does. Each kind has the name of its row of tabs, and each tab its label and the small print beside it (the +# the model does, and one per harness where what runs the loop does. Each kind has the name of its row of tabs, and each tab its label and the small print beside it (the # class that runs the model, or the family's module). A key with a dash is a narrower one of the key before the dash: -# "vllm-docker" and "vllm-url" are kinds of "vllm" (any vLLM server, where the two do not differ), and guide.js takes -# either for the other. +# "vllm-docker" and "vllm-url" are kinds of "vllm" (any vLLM server, where the two do not differ), "pi-page" and +# "pi-provider" of "pi" (pi asking this page's model or one of its own), and guide.js takes either for the other. TAB_KINDS = { "backend": ({"zh": "用什麼跑模型", "en": "What runs the model"}, { "transformers": ({"zh": "transformers", "en": "transformers"}, "SimpleBackend"), @@ -76,6 +76,12 @@ "qwen3_5": ({"zh": "Qwen3.5", "en": "Qwen3.5"}, "qwen3_5.py"), "gemma4-drafter": ({"zh": "Gemma 4 的草稿模型", "en": "Gemma 4's drafter"}, "MTP"), }), + "harness": ({"zh": "用什麼跑迴圈", "en": "What runs the loop"}, { + "simple": ({"zh": "內建 harness", "en": "Built-in harness"}, "SimpleHarness"), + "pi": ({"zh": "pi", "en": "pi"}, "PiHarness"), + "pi-page": ({"zh": "pi · 這個網頁的模型", "en": "pi · this page's model"}, "model_endpoint.py"), + "pi-provider": ({"zh": "pi · pi 自己的模型", "en": "pi · pi's own models"}, "models.json · /login"), + }), } MAX_TABS = 4 # guide.css hides the panels of the tabs not picked, by place, up to this many @@ -143,12 +149,13 @@ def find_line(file: str, text: str) -> int: def statement_end(tree: ast.Module, line: int) -> int: """Last line of the outermost statement that starts on `line`; failing that, of the innermost statement around it (the line is the middle of a call or a dict); failing that, the line.""" - statements = [n for n in ast.walk(tree) if isinstance(n, ast.stmt)] - starting = [n for n in statements if n.lineno == line] + # ast.parse gives every statement an end_lineno; typeshed allows None, for nodes made by hand. + ends = {n: n.end_lineno or n.lineno for n in ast.walk(tree) if isinstance(n, ast.stmt)} + starting = [ends[n] for n in ends if n.lineno == line] if starting: - return max(n.end_lineno for n in starting) - around = [n for n in statements if n.lineno <= line <= n.end_lineno and not hasattr(n, "body")] - return min(around, key=lambda n: n.end_lineno - n.lineno).end_lineno if around else line + return max(starting) + around = [n for n in ends if n.lineno <= line <= ends[n] and not hasattr(n, "body")] + return ends[min(around, key=lambda n: ends[n] - n.lineno)] if around else line def span(code: Code) -> tuple[int, int]: @@ -373,9 +380,10 @@ def page_html(commit: str, lang: str) -> str: def stylesheet() -> str: """The palette from style.css, dark.css's for a reader in dark mode, then the page's own rules.""" - palette = re.search(r"^:root \{.*?^\}", (ASSETS / "style.css").read_text(encoding="utf-8"), re.S | re.M).group() + palette = re.search(r"^:root \{.*?^\}", (ASSETS / "style.css").read_text(encoding="utf-8"), re.S | re.M) + assert palette, "style.css has no :root palette" dark = (ASSETS / "dark.css").read_text(encoding="utf-8") - return f"{palette}\n@media (prefers-color-scheme: dark) {{\n{dark}}}\n{(ASSETS / 'guide.css').read_text(encoding='utf-8')}" + return f"{palette.group()}\n@media (prefers-color-scheme: dark) {{\n{dark}}}\n{(ASSETS / 'guide.css').read_text(encoding='utf-8')}" def document_html(commit: str, lang: str) -> str: @@ -415,18 +423,18 @@ def build(out: Path) -> list[Path]: INTRO = { "zh": "從你按下送出到看到回覆，中間發生的每一件事，照實際執行的順序走一遍。app 的側欄可以選 Gemma 4 E2B 或 Qwen3.5 0.8B，" - "兩個模型跑的是同一個迴圈。問題可以附上圖片、聲音或影片（{{media}}），打開「讓模型寫程式」時，" + "兩個模型跑的是同一個迴圈；迴圈由這個專案自己的 harness 跑，也可以交給 pi 這個 coding agent（{{overview}}）。問題可以附上圖片、聲音或影片（{{media}}），打開「讓模型寫程式」時，" "模型還能在 Docker 沙箱裡執行 Python（{{execute}}）。模型可以在這台電腦上用 transformers 跑，也可以交給一台 vLLM 伺服器（{{backends}}）。" - "兩個模型或兩種跑法不一樣的地方，分成頁籤（tab）各自說明，一個模型或一種 backend 一個；在任何一段選了一個，整頁同一類的頁籤都跟著換。每段都附上做這件事的程式碼；`harness/agent.py:98` 這類標記是檔案的路徑和行號，" + "兩個模型、兩種跑法或兩個 harness 不一樣的地方，分成頁籤（tab）各自說明，一個模型、一種 backend 或一個 harness 一個；在任何一段選了一個，整頁同一類的頁籤都跟著換。每段都附上做這件事的程式碼；`agent/harnesses/simple_harness.py:99` 這類標記是檔案的路徑和行號，" "滑過去會顯示那一行，點旁邊的 GitHub 圖示會在新分頁打開那個檔案、跳到那一行，程式片段標題上的圖示也一樣。" "行號是產生這一頁時從原始碼查出來的，連結也指向那個 commit；每次推上 main，GitHub Actions 都會重新產生，程式改了也會跟著更新。", "en": "Everything that happens between pressing send and seeing the reply, in the order it really runs. The app's sidebar picks " - "Gemma 4 E2B or Qwen3.5 0.8B; both models run the same loop. " + "Gemma 4 E2B or Qwen3.5 0.8B; both models run the same loop, which this project's own harness runs, or pi, a coding agent ({{overview}}). " "A question can carry images, sound or video ({{media}}), and with Let the model write code on, the model can also run Python in a Docker sandbox ({{execute}}). " - "The model runs on this machine with transformers, or on a vLLM server ({{backends}}). Where the two models, or the two ways of running " - "them, differ, a section tells each in a tab of its own, one per model or per backend, and picking one in any section switches every tab " + "The model runs on this machine with transformers, or on a vLLM server ({{backends}}). Where the two models, the two ways of running " + "them or the two harnesses differ, a section tells each in a tab of its own, one per model, per backend or per harness, and picking one in any section switches every tab " "of that kind on the page. " - "Every part comes with the code that does it; a mark like `harness/agent.py:98` is a file's path and a line number, hovering over it shows that line, " + "Every part comes with the code that does it; a mark like `agent/harnesses/simple_harness.py:99` is a file's path and a line number, hovering over it shows that line, " "and the GitHub icon beside it opens the file at that line in a new tab, as the icon on an excerpt's title does for its lines. " "The line numbers are looked up in the source when this page is built, and the links go to that commit; GitHub Actions rebuilds it on every push to main, so they follow the code as it changes.", } @@ -434,8 +442,8 @@ def build(out: Path) -> list[Path]: SECTIONS = [ Section("overview", "ui", {"zh": "總覽：兩條 thread、一張事件列表", "en": "Overview: two threads, one event list"}, - {"zh": "你送出一個問題就是一輪（一個 `Run`）；一輪裡是一個迴圈，每一圈叫一個 step。", - "en": "Each question you send is one turn (one `Run`); inside a turn is a loop, and each time around it is a step."}, ( + {"zh": "你送出一個問題就是一輪（一個 `SimpleHarness`）；一輪裡是一個迴圈，每一圈叫一個 step。", + "en": "Each question you send is one turn (one `SimpleHarness`); inside a turn is a loop, and each time around it is a step."}, ( {"zh": "每個 step 做同一件事：harness 把整段對話 render 成一個字串（脈絡）→ 模型一次推論、一個 token 一個 token 生成 → " "harness 解析輸出。有工具呼叫，就驗證、（需要時問你）、在環境執行，把結果接回對話，進下一圈；" "沒有工具呼叫，就把回覆交給你，這一輪結束。", @@ -448,58 +456,95 @@ def build(out: Path) -> list[Path]: "en": "The app runs on two threads. Streamlit's script thread draws the page; the agent runs on a background thread " "([[app/page.py|self.thread = threading.Thread]]). The two talk through just three things:"}, ( - {"zh": "agent 每走到一個階段，就把一個事件 append 到 `events` 列表（[[harness/agent.py|def emit(self, event_type]]）；畫面只負責把這張列表畫出來。", - "en": "At every stage the agent appends an event to the `events` list ([[harness/agent.py|def emit(self, event_type]]); the page only draws that list."}, - {"zh": "按「停止」時，畫面呼叫 `Run.stop()`（[[harness/agent.py|def stop(self):]]）。", - "en": "Pressing Stop calls `Run.stop()` ([[harness/agent.py|def stop(self):]])."}, - {"zh": "按「允許」或「拒絕」時，畫面呼叫 `Run.decide()`（[[harness/agent.py|def decide(self, call_id]]）。", - "en": "Pressing Allow or Reject calls `Run.decide()` ([[harness/agent.py|def decide(self, call_id]])."}, + {"zh": "agent 每走到一個階段，就把一個事件 append 到 `events` 列表（[[agent/harnesses/simple_harness.py|def emit(self, event_type]]）；畫面只負責把這張列表畫出來。", + "en": "At every stage the agent appends an event to the `events` list ([[agent/harnesses/simple_harness.py|def emit(self, event_type]]); the page only draws that list."}, + {"zh": "按「停止」時，畫面呼叫 `SimpleHarness.stop()`（[[agent/harnesses/simple_harness.py|def stop(self):]]）。", + "en": "Pressing Stop calls `SimpleHarness.stop()` ([[agent/harnesses/simple_harness.py|def stop(self):]])."}, + {"zh": "按「允許」或「拒絕」時，畫面呼叫 `SimpleHarness.decide()`（[[agent/harnesses/simple_harness.py|def decide(self, call_id]]）。", + "en": "Pressing Allow or Reject calls `SimpleHarness.decide()` ([[agent/harnesses/simple_harness.py|def decide(self, call_id]])."}, ), - {"zh": "專案分成兩塊。`harness/` 跑一輪：迴圈、工具、模型，以及跑模型的那一層，它從不 import Streamlit。" - "`app/` 是你看到的東西，站在 harness 上面：開始一輪、把 harness 送出的事件畫出來、把你的點擊傳回去。" - "根目錄的 `streamlit_app.py` 只負責啟動 `app/page.py`。`harness/` 裡每個檔案的分工：", - "en": "The project has two parts. `harness/` runs a turn: the loop, the tools, the model and what runs it, and it never imports Streamlit. " - "`app/` is what you see, on top of the harness: it starts a turn, draws the events the harness sends and passes your clicks back. " - "`streamlit_app.py` at the top only starts `app/page.py`. What each file in `harness/` does:"}, + {"zh": "這三樣就是畫面對「跑迴圈的那一層」的全部要求，`Harness`（[[agent/harnesses/harness.py|class Harness(Protocol):]]）。" + "有兩個 harness 跑得了它：這個專案自己寫的 `SimpleHarness`，以及側欄的「用什麼跑迴圈」選 pi 時的 `PiHarness`，它把迴圈交給 pi 這個 coding agent。" + "兩者做法不同的地方，下面每段都分成頁籤；pi 問這個網頁的模型和問它自己的模型也不一樣時，再各分一個頁籤。" + "另一個會把自己的迴圈報成事件的 agent，例如 Claude Agent SDK 或 Codex，也可以照這樣再寫一個 harness。", + "en": "These three are all the page asks of whatever runs the loop, the `Harness` ([[agent/harnesses/harness.py|class Harness(Protocol):]]). " + "Two harnesses run it: this project's own, `SimpleHarness`, and, with pi picked under What runs the loop in the sidebar, `PiHarness`, which hands " + "the loop to pi, a coding agent. Where the two do a part differently, the sections below tell it in a tab for each; where pi asking this page's " + "model and pi asking one of its own differ too, in a tab for each of those. Another agent that reports its loop as events, such as the " + "Claude Agent SDK or Codex, could be one more harness written the same way."}, + Tabs({ + "simple": ( + {"zh": "`SimpleHarness.run()` 就是迴圈本身（[[agent/harnesses/simple_harness.py|def run(self):]]）：它 render 脈絡、讓模型生成、解析輸出、執行工具，" + "每走到一個階段就送出一個事件。模型在這台電腦上用 transformers 跑，或在 vLLM 伺服器上（{{backends}}），每個 token 都看得到。", + "en": "`SimpleHarness.run()` is the loop itself ([[agent/harnesses/simple_harness.py|def run(self):]]): it renders the context, has the model " + "generate, parses the output and runs the tools, sending an event at every stage. The model runs on this machine with transformers, or on a " + "vLLM server ({{backends}}), and every token shows."}, + ), + "pi": ( + {"zh": "`PiHarness` 每個問題都用 RPC 模式啟動一個 pi 子行程（[[agent/harnesses/pi_harness.py|class Process:]]）：指令從 stdin 送進去，紀錄從 stdout 讀出來，" + "一行一個 JSON。迴圈是 pi 在跑，`PiHarness` 只把 pi 的紀錄翻成頁面的事件；pi 的一個 turn（向模型送一次請求，加上回覆裡要執行的工具）就是一個 step。", + "en": "`PiHarness` starts pi as a child process in RPC mode for each question ([[agent/harnesses/pi_harness.py|class Process:]]): commands go in on stdin, " + "and records come out on stdout, one JSON object per line. pi runs the loop, and `PiHarness` only turns pi's records into the page's events; one " + "of pi's turns (one request to the model, and the tools its reply calls) is a step."}, + Code("agent/harnesses/pi_harness.py", "def _on(self, rec: dict) -> bool:"), + {"zh": "pi 可以問這個網頁的模型，也可以問它自己登入過的模型（Claude、GPT…），由側欄的「pi 問的模型」決定。問這個網頁的模型時，" + "`model_endpoint.py` 把記憶體裡的模型當成一個相容 OpenAI 的端點提供給 pi，每個 token 照樣看得到；問 pi 自己的模型時，provider 的 API 只傳回文字。", + "en": "pi asks either this page's model or one it is logged in to (Claude, GPT…), as the sidebar's The model pi asks says. On this page's model, " + "`model_endpoint.py` serves the model in memory to pi as an OpenAI-compatible endpoint, and every token still shows; on one of pi's own, the " + "provider's API sends back text alone."}, + ), + }), + {"zh": "專案分成兩塊。`agent/` 跑一輪：跑迴圈的 harness、工具、模型，以及跑模型的那一層，它從不 import Streamlit。" + "`app/` 是你看到的東西，站在 agent 上面：開始一輪、把 harness 送出的事件畫出來、把你的點擊傳回去。" + "根目錄的 `streamlit_app.py` 只負責啟動 `app/page.py`。`agent/` 裡每個檔案的分工：", + "en": "The project has two parts. `agent/` runs a turn: the harness that runs the loop, the tools, the model and what runs it, and it never imports Streamlit. " + "`app/` is what you see, on top of the agent: it starts a turn, draws the events the harness sends and passes your clicks back. " + "`streamlit_app.py` at the top only starts `app/page.py`. What each file in `agent/` does:"}, ( - {"zh": "`harness/agent.py`：harness 的迴圈本體 `Run.run()`（[[harness/agent.py|def run(self):]]）。", - "en": "`harness/agent.py`: the harness's loop itself, `Run.run()` ([[harness/agent.py|def run(self):]])."}, - {"zh": "`harness/backends/backend.py`：harness 和畫面對「跑模型的那一層」的全部要求，`Backend`（[[harness/backends/backend.py|class Backend(Protocol):]]）：" + {"zh": "`agent/harnesses/`：跑迴圈的那一層。`harness.py` 是畫面對它的要求 `Harness`，以及每個 harness 送出的事件；" + "`simple_harness.py` 是這個專案的 harness，迴圈本體 `SimpleHarness.run()`（[[agent/harnesses/simple_harness.py|def run(self):]]）；" + "`pi_harness.py` 把迴圈交給 pi；`pi_extension.ts` 是網頁讓 pi 載入的 extension，回報 pi 送出的每個請求、在 pi 執行工具前先問你；" + "`model_endpoint.py` 把這個網頁的模型當成一個相容 OpenAI 的端點提供給 pi。", + "en": "`agent/harnesses/`: what runs the loop. `harness.py` is what the page asks of it, `Harness`, and the events every harness sends; " + "`simple_harness.py` is this project's harness, with the loop itself, `SimpleHarness.run()` ([[agent/harnesses/simple_harness.py|def run(self):]]); " + "`pi_harness.py` hands the loop to pi; `pi_extension.ts` is the extension the page has pi load, which reports every request pi sends and asks you " + "before pi runs a tool; and `model_endpoint.py` serves this page's model to pi as an OpenAI-compatible endpoint."}, + {"zh": "`agent/model_backends/backend.py`：harness 和畫面對「跑模型的那一層」的全部要求，`Backend`（[[agent/model_backends/backend.py|class Backend(Protocol):]]）：" "`render()`、`encode()`、`generate()`，以及 token 事件裡哪些欄位一定有、哪些要 backend 知道才有。" "`simple_backend.py` 的 `SimpleBackend` 是其中一種，`vllm_backend.py` 是另一種（{{backends}}）。", - "en": "`harness/backends/backend.py`: everything the harness and the page ask of whatever runs the model, `Backend` ([[harness/backends/backend.py|class Backend(Protocol):]]): " + "en": "`agent/model_backends/backend.py`: everything the harness and the page ask of whatever runs the model, `Backend` ([[agent/model_backends/backend.py|class Backend(Protocol):]]): " "`render()`, `encode()` and `generate()`, and which fields of a token event are always there and which only when the backend knows them. " "`SimpleBackend` in `simple_backend.py` is one backend, and `vllm_backend.py` another ({{backends}})."}, - {"zh": "`harness/backends/simple_backend.py`：本機推論，自己寫的解碼迴圈 `generate()`（[[harness/backends/simple_backend.py|def generate(self, prompt]]）和抽樣 `_sample()`；" + {"zh": "`agent/model_backends/simple_backend.py`：本機推論，自己寫的解碼迴圈 `generate()`（[[agent/model_backends/simple_backend.py|def generate(self, prompt]]）和抽樣 `_sample()`；" "打開 MTP 時再加上一個草稿模型（{{mtp}}）。", - "en": "`harness/backends/simple_backend.py`: local inference, with a hand-written decoding loop, `generate()` ([[harness/backends/simple_backend.py|def generate(self, prompt]]), " + "en": "`agent/model_backends/simple_backend.py`: local inference, with a hand-written decoding loop, `generate()` ([[agent/model_backends/simple_backend.py|def generate(self, prompt]]), " "and sampling, `_sample()`; with MTP on, a drafter joins in ({{mtp}})."}, - {"zh": "`harness/backends/vllm_backend.py`：模型在 vLLM 伺服器上跑時用的 `Backend`（[[harness/backends/vllm_backend.py|class VLLMBackend:]]），" + {"zh": "`agent/model_backends/vllm_backend.py`：模型在 vLLM 伺服器上跑時用的 `Backend`（[[agent/model_backends/vllm_backend.py|class VLLMBackend:]]），" "伺服器是 `VLLM_URL` 指的那台，或是畫面在 Docker 裡啟動的那台。prompt 照樣在這裡 render、切成 token，" "token id 送到 vLLM 伺服器，每個 token 帶著它的 log 機率和前 5 名候選回來；取樣時剃掉了哪些候選，留在伺服器上。", - "en": "`harness/backends/vllm_backend.py`: the `Backend` for a model on a vLLM server ([[harness/backends/vllm_backend.py|class VLLMBackend:]]), " + "en": "`agent/model_backends/vllm_backend.py`: the `Backend` for a model on a vLLM server ([[agent/model_backends/vllm_backend.py|class VLLMBackend:]]), " "the one at `VLLM_URL` or the one the page starts in Docker. The prompt is still rendered " "and tokenized here, the token ids go to a vLLM server, and each token comes back with its log probability and the top 5 candidates; " "which candidates the sampling cut stays on the server."}, - {"zh": "`harness/backends/vllm_docker.py`：沒設 `VLLM_URL`、側欄又選了 vLLM 時，照 `docker-compose/compose.yaml` 用 Docker Compose 啟動 vLLM 伺服器，" - "等到它能回應（[[harness/backends/vllm_docker.py|def start(model_id]]），在 GPU 上預設開 MTP 推測解碼；換回 transformers 時停掉它（{{backends}}）。", - "en": "`harness/backends/vllm_docker.py`: without `VLLM_URL`, when the sidebar picks vLLM, it starts a vLLM server with Docker Compose from " - "`docker-compose/compose.yaml` and waits until it answers ([[harness/backends/vllm_docker.py|def start(model_id]]), on a GPU with MTP " + {"zh": "`agent/model_backends/vllm_docker.py`：沒設 `VLLM_URL`、側欄又選了 vLLM 時，照 `docker-compose/compose.yaml` 用 Docker Compose 啟動 vLLM 伺服器，" + "等到它能回應（[[agent/model_backends/vllm_docker.py|def start(model_id]]），在 GPU 上預設開 MTP 推測解碼；換回 transformers 時停掉它（{{backends}}）。", + "en": "`agent/model_backends/vllm_docker.py`: without `VLLM_URL`, when the sidebar picks vLLM, it starts a vLLM server with Docker Compose from " + "`docker-compose/compose.yaml` and waits until it answers ([[agent/model_backends/vllm_docker.py|def start(model_id]]), on a GPU with MTP " "speculative decoding on by default; picking transformers again stops it ({{backends}})."}, - {"zh": "`harness/families/`：跟著模型走的東西。`gemma4.py`、`qwen3_5.py` 各自負責讀懂自己模型的輸出，並說明它的特殊 token（{{families}}）。", - "en": "`harness/families/`: everything that goes with the model. `gemma4.py` and `qwen3_5.py` each read their own model's output " + {"zh": "`agent/model_families/`：跟著模型走的東西。`gemma4.py`、`qwen3_5.py` 各自負責讀懂自己模型的輸出，並說明它的特殊 token（{{families}}）。", + "en": "`agent/model_families/`: everything that goes with the model. `gemma4.py` and `qwen3_5.py` each read their own model's output " "and explain its special tokens ({{families}})."}, - {"zh": "`harness/tools.py`：每個工具的 schema（給模型看）和 Python 函式（給 harness 執行），全部列在 `TOOLS`（[[harness/tools.py|TOOLS = {]]）。" - "其中 `run_python` 在 `harness/sandbox/Dockerfile` 建的 Docker 容器裡跑模型寫的程式（{{execute}}）。", - "en": "`harness/tools.py`: each tool's schema (for the model to read) and Python function (for the harness to run), " - "all listed in `TOOLS` ([[harness/tools.py|TOOLS = {]]). One of them, `run_python`, runs the model's code in a Docker container " - "built from `harness/sandbox/Dockerfile` ({{execute}})."}, - {"zh": "`harness/media.py`：問題附上的檔案：存進這段對話的資料夾、寫進使用者訊息，再解碼成模型的 processor 要的格式（{{media}}）。", - "en": "`harness/media.py`: the files attached to a question: it saves them in the conversation's folder, writes them into the user's " + {"zh": "`agent/tools.py`：每個工具的 schema（給模型看）和 Python 函式（給 harness 執行），全部列在 `TOOLS`（[[agent/tools.py|TOOLS = {]]）。" + "其中 `run_python` 在 `agent/sandbox/Dockerfile` 建的 Docker 容器裡跑模型寫的程式（{{execute}}）。", + "en": "`agent/tools.py`: each tool's schema (for the model to read) and Python function (for the harness to run), " + "all listed in `TOOLS` ([[agent/tools.py|TOOLS = {]]). One of them, `run_python`, runs the model's code in a Docker container " + "built from `agent/sandbox/Dockerfile` ({{execute}})."}, + {"zh": "`agent/media.py`：問題附上的檔案：存進這段對話的資料夾、寫進使用者訊息，再解碼成模型的 processor 要的格式（{{media}}）。", + "en": "`agent/media.py`: the files attached to a question: it saves them in the conversation's folder, writes them into the user's " "message and decodes them for the model's processor ({{media}})."}, - {"zh": "`harness/i18n.py`：頁面的兩種語言，繁體中文和英文；`pick()` 照語言挑出其中一段文字（[[harness/i18n.py|def pick(]]）。" + {"zh": "`agent/i18n.py`：頁面的兩種語言，繁體中文和英文；`pick()` 照語言挑出其中一段文字（[[agent/i18n.py|def pick(]]）。" "harness 也用它：模型讀到的錯誤訊息，用這一輪開始時頁面的語言寫。", - "en": "`harness/i18n.py`: the page's two languages, Traditional Chinese and English; `pick()` chooses the text for one of them ([[harness/i18n.py|def pick(]]). " + "en": "`agent/i18n.py`: the page's two languages, Traditional Chinese and English; `pick()` chooses the text for one of them ([[agent/i18n.py|def pick(]]). " "The harness uses it too: the errors the model reads are in the language the page was in when the turn started."}, ), {"zh": "`app/` 裡：", "en": "In `app/`:"}, @@ -547,7 +592,7 @@ def build(out: Path) -> list[Path]: }), {"zh": "這些差異全部寫進 `Family`，其他程式只問 family，沒有任何 `if 是 Gemma` 的分支：", "en": "All of these differences go into a `Family`, and the rest of the code just asks the family; there is no `if Gemma` branch anywhere:"}, - Code("harness/families/__init__.py", "class Family:", end="def special_re(self)", + Code("agent/model_families/__init__.py", "class Family:", end="def special_re(self)", caption={"zh": "欄位分三組：讀模型輸出用的、把對話寫回 chat template 用的、畫面用的。最後幾個有預設值，只有某些家族需要，" "例如 Qwen 在 `processor_kwargs` 裡給 processor 的影片設定；`file_notes` 和 `drafter_file_notes` 說明模型資料夾裡的每個檔案（{{files}}）。", "en": "The fields come in three groups: for reading the model's output, for writing the conversation back for the chat template, and for the page. " @@ -562,17 +607,17 @@ def build(out: Path) -> list[Path]: "every turn, so without the family the model would keep writing after its answer. Qwen's tokenizer does not mark `` " "and `` as special either, and the family adds those too. The tokenizer comes inside the model's processor, which " "also turns images, sound and video into the model's inputs; which of those a model reads, the checkpoint says ({{media}})."}, - Code("harness/backends/simple_backend.py", "self.family = families.for_model_type(", end="self.special_ids = set("), - {"zh": "要加模型時：同一個家族的另一個 checkpoint（例如更大的 Gemma 4），在 `MODELS` 加一行（[[harness/families/__init__.py|MODELS = {]]），" + Code("agent/model_backends/simple_backend.py", "self.family = model_families.for_model_type(", end="self.special_ids = set("), + {"zh": "要加模型時：同一個家族的另一個 checkpoint（例如更大的 Gemma 4），在 `MODELS` 加一行（[[agent/model_families/__init__.py|MODELS = {]]），" "`REVISIONS` 再加一行它要下載的 commit；有 MTP 草稿模型的話，`DRAFTERS` 和草稿模型的 commit 也要加，`VLLM_SPECULATIVE` 寫 vLLM 在 GPU 上怎麼替它推測解碼。" - "新的家族，照 `harness/families/qwen3_5.py` 寫一個模組，內容是一個輸出的 parser 和特殊 token 的說明（中英文各一份），" - "加進 `FAMILIES`（[[harness/families/__init__.py|FAMILIES = {]]），`model_type` 對得上就會自動選到。", + "新的家族，照 `agent/model_families/qwen3_5.py` 寫一個模組，內容是一個輸出的 parser 和特殊 token 的說明（中英文各一份），" + "加進 `FAMILIES`（[[agent/model_families/__init__.py|FAMILIES = {]]），`model_type` 對得上就會自動選到。", "en": "To add a model: another checkpoint of a family that is already here (a bigger Gemma 4, say) is one line in `MODELS` " - "([[harness/families/__init__.py|MODELS = {]]) and one in `REVISIONS`, the commit it downloads at; with an MTP drafter, also `DRAFTERS` " - "and the drafter's commit, and `VLLM_SPECULATIVE` for how vLLM drafts for it on a GPU. A new family is a module like `harness/families/qwen3_5.py`, with a parser for the output and " - "notes on the special tokens (in Chinese and in English), added to `FAMILIES` ([[harness/families/__init__.py|FAMILIES = {]]); " + "([[agent/model_families/__init__.py|MODELS = {]]) and one in `REVISIONS`, the commit it downloads at; with an MTP drafter, also `DRAFTERS` " + "and the drafter's commit, and `VLLM_SPECULATIVE` for how vLLM drafts for it on a GPU. A new family is a module like `agent/model_families/qwen3_5.py`, with a parser for the output and " + "notes on the special tokens (in Chinese and in English), added to `FAMILIES` ([[agent/model_families/__init__.py|FAMILIES = {]]); " "a matching `model_type` picks it automatically."}, - Code("harness/families/qwen3_5.py", "QWEN3_5 = Family(", + Code("agent/model_families/qwen3_5.py", "QWEN3_5 = Family(", caption={"zh": "Qwen3.5 的 family 幾乎都是資料，只有 `parse` 是程式。", "en": "Qwen3.5's family is almost all data; only `parse` is code."}), )), Section("files", "model", @@ -581,12 +626,12 @@ def build(out: Path) -> list[Path]: "兩個模型和 Gemma 的草稿模型分別說明。", "en": "A model's files are in a folder of its own in `models/`: transformers loads the whole model from it, vLLM reads the tokenizer and the chat template there too, " "and in Docker the weights as well. Here is each file, for both models and Gemma's drafter."}, ( - {"zh": "不管用什麼跑模型，下載都是 `snapshot_download()` 下載到 `MODELS_DIR`（[[harness/backends/simple_backend.py|MODELS_DIR = Path(]]）：預設是專案裡的 `models/`，git 不會追蹤它；" - "設定 `AGENT_LOOP_MODELS` 就換到別的地方（`HF_HUB_CACHE` 不行，因為它常是整台電腦所有專案共用的設定）。下載的是 `REVISIONS` 固定的 commit（[[harness/families/__init__.py|REVISIONS = {]]），不是當下的 `main`，" + {"zh": "不管用什麼跑模型，下載都是 `snapshot_download()` 下載到 `MODELS_DIR`（[[agent/model_backends/simple_backend.py|MODELS_DIR = Path(]]）：預設是專案裡的 `models/`，git 不會追蹤它；" + "設定 `AGENT_LOOP_MODELS` 就換到別的地方（`HF_HUB_CACHE` 不行，因為它常是整台電腦所有專案共用的設定）。下載的是 `REVISIONS` 固定的 commit（[[agent/model_families/__init__.py|REVISIONS = {]]），不是當下的 `main`，" "所以不管什麼時候下載，拿到的都是這一頁寫作和測試時用的檔案。已經下載過的檔案不會再抓，連不上 Hub 時直接用資料夾裡現有的。", - "en": "Whatever runs the model, a download is `snapshot_download()` into `MODELS_DIR` ([[harness/backends/simple_backend.py|MODELS_DIR = Path(]]): by default the project's `models/`, " + "en": "Whatever runs the model, a download is `snapshot_download()` into `MODELS_DIR` ([[agent/model_backends/simple_backend.py|MODELS_DIR = Path(]]): by default the project's `models/`, " "which git ignores; `AGENT_LOOP_MODELS` moves it elsewhere (`HF_HUB_CACHE` does not, since it is often set once for every project on a machine). It downloads the commit pinned in `REVISIONS` " - "([[harness/families/__init__.py|REVISIONS = {]]), not whatever `main` is by then, so whenever it runs, it gets the files this page was " + "([[agent/model_families/__init__.py|REVISIONS = {]]), not whatever `main` is by then, so whenever it runs, it gets the files this page was " "written and tested against. Files already there are not fetched again, and when the Hub cannot be reached, what the folder holds is used."}, {"zh": "`models/` 裡是 Hugging Face 的快取格式：", "en": "Inside, `models/` is laid out as Hugging Face's cache:"}, ( @@ -617,19 +662,19 @@ def build(out: Path) -> list[Path]: "en": "`SimpleBackend` has `_download()` fetch the whole checkpoint, the weights included, and loads all of it on this machine. It fetches `config.json` " "alone first, to tell the model's family from it, so a model the page does not support fails before gigabytes of weights come down ({{families}}). " "The MTP drafter downloads the same way, the first time MTP is switched on ({{mtp}})."}, - Code("harness/backends/simple_backend.py", "def _download("), + Code("agent/model_backends/simple_backend.py", "def _download("), ), "vllm-docker": ( {"zh": "啟動伺服器之前，`vllm_docker.start()` 先把整個 checkpoint 下載到 `models/`，權重也在內；GPU 上開 MTP 時，連 Gemma 4 的草稿模型也一起下載" "（Qwen3.5 的 MTP 層就在它自己的權重裡）。伺服器自己什麼都不下載：", "en": "Before it starts the server, `vllm_docker.start()` downloads the whole checkpoint into `models/`, the weights included, and with MTP on a GPU " "Gemma 4's drafter too (Qwen3.5's MTP layer is in its own weights). The server itself downloads nothing:"}, - Code("harness/backends/vllm_docker.py", 'say("download", "")', end="snapshot_download(drafter"), + Code("agent/model_backends/vllm_docker.py", 'say("download", "")', end="snapshot_download(drafter"), {"zh": "`docker-compose/compose.yaml` 把這個資料夾唯讀掛進容器的 `/models`，Hugging Face 的快取指到那裡，而且不連網路。" - "`_env()` 把完整路徑的 `MODELS_DIR` 當成 `AGENT_LOOP_MODELS` 交給 Docker Compose（[[harness/backends/vllm_docker.py|\"AGENT_LOOP_MODELS\": str(MODELS_DIR)]]），" + "`_env()` 把完整路徑的 `MODELS_DIR` 當成 `AGENT_LOOP_MODELS` 交給 Docker Compose（[[agent/model_backends/vllm_docker.py|\"AGENT_LOOP_MODELS\": str(MODELS_DIR)]]），" "所以 `models/` 換了地方，掛上的也是那裡。vLLM 編譯模型的結果（第一次啟動最久的那一段）存在另一個 Docker volume，不放在 `models/`：", "en": "`docker-compose/compose.yaml` mounts that folder in the container at `/models`, read-only, and points Hugging Face's cache there, with no network. " - "`_env()` hands Docker Compose the full path of `MODELS_DIR` as `AGENT_LOOP_MODELS` ([[harness/backends/vllm_docker.py|\"AGENT_LOOP_MODELS\": str(MODELS_DIR)]]), " + "`_env()` hands Docker Compose the full path of `MODELS_DIR` as `AGENT_LOOP_MODELS` ([[agent/model_backends/vllm_docker.py|\"AGENT_LOOP_MODELS\": str(MODELS_DIR)]]), " "so a `models/` moved elsewhere is the one mounted. What vLLM compiles for the model, the first start's longest wait, goes in a Docker volume of its own, " "not in `models/`:"}, Code("docker-compose/compose.yaml", "- ${AGENT_LOOP_MODELS:-../models}:/models:ro", end='HF_HUB_OFFLINE: "1"'), @@ -648,10 +693,10 @@ def build(out: Path) -> list[Path]: ), "vllm-url": ( {"zh": "`VLLM_URL` 的伺服器自己有一份模型，在它跑的那台電腦上，所以網頁只把權重以外的檔案下載到 `models/`" - "（`WEIGHTS` 列的那幾種不抓，[[harness/backends/vllm_backend.py|WEIGHTS = []]），用的是同一個 commit：", + "（`WEIGHTS` 列的那幾種不抓，[[agent/model_backends/vllm_backend.py|WEIGHTS = []]），用的是同一個 commit：", "en": "A server at `VLLM_URL` has a copy of the model of its own, on the machine it runs on, so the page downloads every file but the weights into `models/` " - "(none of the kinds `WEIGHTS` lists, [[harness/backends/vllm_backend.py|WEIGHTS = []]), at the same commit:"}, - Code("harness/backends/vllm_backend.py", "def _checkpoint_files("), + "(none of the kinds `WEIGHTS` lists, [[agent/model_backends/vllm_backend.py|WEIGHTS = []]), at the same commit:"}, + Code("agent/model_backends/vllm_backend.py", "def _checkpoint_files("), {"zh": "兩邊要讀到同樣的 tokenizer 和 chat template，伺服器就要用 `REVISIONS` 的 commit 啟動，例如 `vllm serve Qwen/Qwen3.5-0.8B --revision 2fc0636…`。" "網頁照伺服器啟動時的名字（`/v1/models` 的 `root`）去下載，所以從伺服器自己的資料夾啟動的（`vllm serve /data/gemma`），" "要這台電腦的同一個路徑也有同一個資料夾，網頁才讀得到，而且就地讀、不下載；否則網頁會說它為什麼載入不了。", @@ -660,34 +705,34 @@ def build(out: Path) -> list[Path]: "so a server started from a folder of its own (`vllm serve /data/gemma`) works only when the same folder is at the same path on this machine, " "which the page then reads where it is, downloading nothing; otherwise the page says why it cannot load."}, {"zh": "在那之前，模型選單只下載伺服器上每個模型的 `config.json`，用它的 `model_type` 看出是哪個 family、網頁支不支援" - "（[[harness/backends/vllm_backend.py|def family_of(]]）；其他檔案等選了那個模型才下載。", + "（[[agent/model_backends/vllm_backend.py|def family_of(]]）；其他檔案等選了那個模型才下載。", "en": "Before that, the model menu downloads only the `config.json` of each model the server serves, whose `model_type` tells its family, and whether the " - "page supports it ([[harness/backends/vllm_backend.py|def family_of(]]); the other files come once that model is picked."}, + "page supports it ([[agent/model_backends/vllm_backend.py|def family_of(]]); the other files come once that model is picked."}, ), }), {"zh": "載入時，每個檔案由誰讀：", "en": "When the model loads, which code reads which file:"}, Tabs({ "transformers": ( - {"zh": "同一個 revision 在 macOS、Linux 加 CUDA 或只有 CPU 的電腦上，都是一模一樣的檔案。不同的是載入的方式：`pick_device()` 選裝置（[[harness/backends/simple_backend.py|def pick_device()]]），" + {"zh": "同一個 revision 在 macOS、Linux 加 CUDA 或只有 CPU 的電腦上，都是一模一樣的檔案。不同的是載入的方式：`pick_device()` 選裝置（[[agent/model_backends/simple_backend.py|def pick_device()]]），" "權重在 GPU 上照檔案的 bfloat16 載入，在 CPU 上轉成 float32，記憶體是兩倍。", "en": "A revision is exactly the same files on macOS, on Linux with CUDA or on a computer with only a CPU. What differs is how they load: " - "`pick_device()` picks the device ([[harness/backends/simple_backend.py|def pick_device()]]), and the weights load in the files' bfloat16 on a GPU but become " + "`pick_device()` picks the device ([[agent/model_backends/simple_backend.py|def pick_device()]]), and the weights load in the files' bfloat16 on a GPU but become " "float32 on a CPU, twice the memory."}, ( - {"zh": "`config.json`：`AutoConfig` 先讀它，`model_type` 決定 family（[[harness/backends/simple_backend.py|self.family = families.for_model_type(]]），模型分頁上方的規格也來自它。", - "en": "`config.json`: `AutoConfig` reads it first, and its `model_type` picks the family ([[harness/backends/simple_backend.py|self.family = families.for_model_type(]]); " + {"zh": "`config.json`：`AutoConfig` 先讀它，`model_type` 決定 family（[[agent/model_backends/simple_backend.py|self.family = model_families.for_model_type(]]），模型分頁上方的規格也來自它。", + "en": "`config.json`: `AutoConfig` reads it first, and its `model_type` picks the family ([[agent/model_backends/simple_backend.py|self.family = model_families.for_model_type(]]); " "the specs at the top of the Model tab come from it too."}, {"zh": "`tokenizer.json`、`tokenizer_config.json`、`chat_template.jinja` 和 processor 的設定檔：`AutoProcessor` 把它們讀成一個 processor" - "（[[harness/backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]）；chat template 由 `render()` 執行（[[harness/backends/simple_backend.py|def render(self, messages]]）。", + "（[[agent/model_backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]）；chat template 由 `render()` 執行（[[agent/model_backends/simple_backend.py|def render(self, messages]]）。", "en": "`tokenizer.json`, `tokenizer_config.json`, `chat_template.jinja` and the processor's settings: `AutoProcessor` reads them into one " - "processor ([[harness/backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]); `render()` runs the chat template ([[harness/backends/simple_backend.py|def render(self, messages]])."}, + "processor ([[agent/model_backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]); `render()` runs the chat template ([[agent/model_backends/simple_backend.py|def render(self, messages]])."}, {"zh": "權重：`AutoModelForMultimodalLM` 照 `config.json` 建好模型，再把 `.safetensors` 裡的 tensor 依名稱填進去" - "（[[harness/backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]]）。", + "（[[agent/model_backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]]）。", "en": "The weights: `AutoModelForMultimodalLM` builds the model from `config.json` and fills it with the tensors of the `.safetensors` file " - "by name ([[harness/backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]])."}, - {"zh": "`generation_config.json`：停止 token（[[harness/backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]]）和建議的抽樣設定（[[harness/backends/simple_backend.py|def _sampling_defaults(self)]]）。", - "en": "`generation_config.json`: the stop tokens ([[harness/backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]]) and the recommended sampling " - "settings ([[harness/backends/simple_backend.py|def _sampling_defaults(self)]])."}, + "by name ([[agent/model_backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]])."}, + {"zh": "`generation_config.json`：停止 token（[[agent/model_backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]]）和建議的抽樣設定（[[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]]）。", + "en": "`generation_config.json`: the stop tokens ([[agent/model_backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]]) and the recommended sampling " + "settings ([[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]])."}, {"zh": "`README.md`、`LICENSE`、`.gitattributes`：給人看的模型卡、授權和 Hub 上 git 的設定，app 不讀。", "en": "`README.md`, `LICENSE` and `.gitattributes`: the model card, the license and the settings of the Hub's git, for people; the app does not read them."}, ), @@ -696,23 +741,23 @@ def build(out: Path) -> list[Path]: {"zh": "`VLLMBackend` 讀的是同一份檔案、用同樣的方式讀，只有權重不在這裡讀：", "en": "`VLLMBackend` reads the same files the same way, all but the weights:"}, ( - {"zh": "`config.json`：`AutoConfig` 讀它，`model_type` 決定 family（[[harness/backends/vllm_backend.py|self.family = families.for_model_type(config.model_type)]]），" - "模型分頁上方的規格也來自它；脈絡上限則是伺服器說的（`--max-model-len`，[[harness/backends/vllm_backend.py|self.context_length = card.get(]]）。", - "en": "`config.json`: `AutoConfig` reads it, and its `model_type` picks the family ([[harness/backends/vllm_backend.py|self.family = families.for_model_type(config.model_type)]]); " + {"zh": "`config.json`：`AutoConfig` 讀它，`model_type` 決定 family（[[agent/model_backends/vllm_backend.py|self.family = model_families.for_model_type(config.model_type)]]），" + "模型分頁上方的規格也來自它；脈絡上限則是伺服器說的（`--max-model-len`，[[agent/model_backends/vllm_backend.py|self.context_length = card.get(]]）。", + "en": "`config.json`: `AutoConfig` reads it, and its `model_type` picks the family ([[agent/model_backends/vllm_backend.py|self.family = model_families.for_model_type(config.model_type)]]); " "the specs at the top of the Model tab come from it too, but the context limit is the server's (`--max-model-len`, " - "[[harness/backends/vllm_backend.py|self.context_length = card.get(]])."}, + "[[agent/model_backends/vllm_backend.py|self.context_length = card.get(]])."}, {"zh": "`tokenizer.json`、`tokenizer_config.json`、`chat_template.jinja` 和 processor 的設定檔：`AutoProcessor` 讀成一個 processor，和 `SimpleBackend` 一樣" - "（[[harness/backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]），所以 chat template 在這裡執行，不在伺服器上（{{context}}）。" + "（[[agent/model_backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]），所以 chat template 在這裡執行，不在伺服器上（{{context}}）。" "伺服器也從它那一份（在 Docker 裡就是這個資料夾）載入 tokenizer 和 processor，用來展開媒體（{{media}}）。", "en": "`tokenizer.json`, `tokenizer_config.json`, `chat_template.jinja` and the processor's settings: `AutoProcessor` reads them into one processor, as " - "`SimpleBackend` does ([[harness/backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]), so the chat template runs here, " + "`SimpleBackend` does ([[agent/model_backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]]), so the chat template runs here, " "not on the server ({{context}}). The server loads a tokenizer and a processor from its copy too (in Docker, this very folder), to expand the media ({{media}})."}, {"zh": "權重：伺服器在它的裝置上、用它的精度載入，網頁從來不載入，所以模型分頁沒有裝置、精度和參數量（{{backends}}）。", "en": "The weights: the server loads them, on its device and in its precision, and the page never does, so the Model tab has no device, precision or " "parameter count ({{backends}})."}, - {"zh": "`generation_config.json`：網頁用它的停止 token 和建議的抽樣設定（[[harness/backends/vllm_backend.py|def _generation_config(]]）。" + {"zh": "`generation_config.json`：網頁用它的停止 token 和建議的抽樣設定（[[agent/model_backends/vllm_backend.py|def _generation_config(]]）。" "伺服器也讀它，當成自己的抽樣預設值，所以每個請求都把它們設成不起作用（{{sampling}}）。", - "en": "`generation_config.json`: the page takes the stop tokens and the recommended sampling settings from it ([[harness/backends/vllm_backend.py|def _generation_config(]]). " + "en": "`generation_config.json`: the page takes the stop tokens and the recommended sampling settings from it ([[agent/model_backends/vllm_backend.py|def _generation_config(]]). " "The server reads it too, for sampling defaults of its own, which every request sets to do nothing ({{sampling}})."}, {"zh": "`README.md`、`LICENSE`、`.gitattributes`：網頁和伺服器都不讀。", "en": "`README.md`, `LICENSE` and `.gitattributes`: neither the page nor the server reads them."}, @@ -770,9 +815,9 @@ def build(out: Path) -> list[Path]: "limit of 262,144; `vision_config` is a 12-layer vision encoder that images and video share; there is no sound encoder. " "`mtp_num_hidden_layers: 1` says the weights come with an MTP layer."}, {"zh": "沒有 `generation_config.json`：停止 token 只剩 `config.json` 的 `<|endoftext|>`，所以 family 補上 `<|im_end|>`；" - "建議的抽樣設定（T 1.0、top-p 1.0、top-k 20）也由 family 提供（[[harness/families/qwen3_5.py|sampling={]]）。", + "建議的抽樣設定（T 1.0、top-p 1.0、top-k 20）也由 family 提供（[[agent/model_families/qwen3_5.py|sampling={]]）。", "en": "No `generation_config.json`: the only stop token left is `config.json`'s `<|endoftext|>`, so the family adds `<|im_end|>`; the " - "recommended sampling settings (T 1.0, top-p 1.0, top-k 20) come from the family too ([[harness/families/qwen3_5.py|sampling={]])."}, + "recommended sampling settings (T 1.0, top-p 1.0, top-k 20) come from the family too ([[agent/model_families/qwen3_5.py|sampling={]])."}, {"zh": "`model.safetensors-00001-of-00001.safetensors`（1.7 GB）：只有一個分片，檔名照分片的格式取；`model.safetensors.index.json` 記下 488 個 tensor 各在哪個分片。" "0.87B 個參數：24 層 0.50B、embedding 0.25B、視覺編碼器 0.10B，還有 20M 的 MTP 層（`mtp.`），transformers 載入時跳過。" "大多是 bfloat16，Gated DeltaNet 層的 `A_log` 和 norm 以 float32 存。", @@ -808,9 +853,9 @@ def build(out: Path) -> list[Path]: "(`backbone_hidden_size`); all 4 read the model's keys and values (`num_kv_shared_layers: 4`). Its output is clustered: the 262,144 " "tokens are sorted into 2,048 clusters of 128, the clusters are scored first, and logits are computed only for the 4,096 tokens of " "the best 32 ({{mtp}})."}, - {"zh": "`generation_config.json`：`num_assistant_tokens: 6`，每次猜 6 個 token（[[harness/backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]]）。", + {"zh": "`generation_config.json`：`num_assistant_tokens: 6`，每次猜 6 個 token（[[agent/model_backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]]）。", "en": "`generation_config.json`: `num_assistant_tokens: 6`, six guesses at a time " - "([[harness/backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]])."}, + "([[agent/model_backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]])."}, {"zh": "`model.safetensors`（158 MB）：77.7M 個參數。67M 是 `embed_tokens`（262,144 個 token × 256 維），當作輸出的表；" "`pre_projection` 把主模型的 token embedding 接上 hidden state（3,072 維）壓成 256 維，`post_projection` 再放大回 1,536；4 層本身只有約 9M。" "`masked_embedding.token_ordering` 是 262,144 個整數，依群列出 token。", @@ -825,15 +870,15 @@ def build(out: Path) -> list[Path]: ), }), {"zh": "模型分頁最下面的「模型資料夾」列出目前載入的模型的這些檔案：`describe_folder()` 讀出每個檔案的大小，以及 family 的說明提到的欄位在檔案裡的值" - "（[[harness/backends/simple_backend.py|def describe_folder(]]）。說明本身寫在 family 裡（Gemma：[[harness/families/gemma4.py|file_notes=FILES,]]；Qwen：[[harness/families/qwen3_5.py|file_notes=FILES,]]），" + "（[[agent/model_backends/simple_backend.py|def describe_folder(]]）。說明本身寫在 family 裡（Gemma：[[agent/model_families/gemma4.py|file_notes=FILES,]]；Qwen：[[agent/model_families/qwen3_5.py|file_notes=FILES,]]），" "所以換成同一家族的另一個 checkpoint，數字會跟著它的檔案變。模型在 vLLM 上跑時，列的是網頁讀的那個資料夾：權重不在裡面時（`VLLM_URL` 的伺服器），" "寫成「只在伺服器上」（[[app/render.py|elif served and kind == \"model.safetensors\":]]）。", "en": "The model folder card at the bottom of the Model tab lists these files for the model that is loaded: `describe_folder()` reads each " - "file's size and the values of the fields the family's notes mention ([[harness/backends/simple_backend.py|def describe_folder(]]). The notes themselves are on the " - "family (Gemma: [[harness/families/gemma4.py|file_notes=FILES,]]; Qwen: [[harness/families/qwen3_5.py|file_notes=FILES,]]), so another checkpoint of " + "file's size and the values of the fields the family's notes mention ([[agent/model_backends/simple_backend.py|def describe_folder(]]). The notes themselves are on the " + "family (Gemma: [[agent/model_families/gemma4.py|file_notes=FILES,]]; Qwen: [[agent/model_families/qwen3_5.py|file_notes=FILES,]]), so another checkpoint of " "the family shows its own numbers. With the model on vLLM, it lists the folder the page reads, and weights that are not in it (with a server at `VLLM_URL`) " "are on the server only ([[app/render.py|elif served and kind == \"model.safetensors\":]])."}, - Code("harness/families/__init__.py", "class FileNote:"), + Code("agent/model_families/__init__.py", "class FileNote:"), )), Section("submit", "ui", {"zh": "送出問題：載入模型、開一條 thread 跑 agent", "en": "Sending a question: load the model, start a thread for the agent"}, @@ -843,27 +888,42 @@ def build(out: Path) -> list[Path]: "`load_llm()` 用 `st.cache_resource(max_entries=1)` 快取，記憶體裡同時只留一個模型。換模型或換「用什麼跑模型」時，" "`release_model()` 先清掉快取、把記憶體還給系統，下一個模型才載入，兩個不會同時佔著記憶體。Streamlit 的 fast reruns 也關掉了（`.streamlit/config.toml`）：重跑會等正在跑的那一次到下一個 Streamlit 呼叫，而不是在新的 thread 裡和它同時跑，所以模型載入到一半時切換的設定，要等載入完才生效，不會在放掉之後才又載入完、留在記憶體裡。" "第一次用到某個模型時，把它建議的 temperature、top-p、top-k 寫進側欄；Qwen3.5 的 checkpoint 沒附 generation_config.json，建議值來自 family" - "（[[harness/backends/simple_backend.py|def _sampling_defaults(self)]]）。側欄的模型選單換了之後會開新對話（[[app/page.py|key=\"model_id\", on_change=switch_model]]）。", + "（[[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]]）。側欄的模型選單換了之後會開新對話（[[app/page.py|key=\"model_id\", on_change=switch_model]]）。", "en": "On every rerun, `current_llm()` first gets a `Backend` for the model picked in the sidebar, and the rest of the page takes it for just that ({{backends}}). " "`load_llm()` is cached with `st.cache_resource(max_entries=1)`, so only one model stays in memory. Switching models, or what runs them, first has " "`release_model()` clear that cache and hand the memory back, and only then does the next model load, so the two never hold memory at once. " "Streamlit's fast reruns are off too (`.streamlit/config.toml`): a rerun waits for the run in progress to reach its next Streamlit call rather than running beside it in a new thread, so a switch made while a model loads takes effect once it has loaded, and the model does not finish loading after it was let go. " "The first time a model is used, its recommended temperature, top-p and top-k go into the sidebar; the Qwen3.5 checkpoint ships no " - "generation_config.json, so its recommendations come from the family ([[harness/backends/simple_backend.py|def _sampling_defaults(self)]]). " + "generation_config.json, so its recommendations come from the family ([[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]]). " "Switching models in the sidebar's menu starts a new conversation ([[app/page.py|key=\"model_id\", on_change=switch_model]])."}, Code("app/page.py", "@st.cache_resource(max_entries=1", end="def release_model():"), - {"zh": "其他的看是用什麼跑模型：", "en": "The rest depends on what runs the model:"}, + Tabs({ + "simple": ( + {"zh": "內建 harness 跑的就是這個模型：側欄最上面選模型，再選用什麼跑它。", + "en": "The built-in harness runs this very model: the sidebar picks the model at its top, then what runs it."}, + ), + "pi": ( + {"zh": "選了 pi 時，`pi_picker()` 先啟動一次 pi，問它有哪些模型（[[agent/harnesses/pi_harness.py|def probe()]]），最多每分鐘問一次。" + "「pi 問的模型」選這個網頁的模型時，接著照內建 harness 的方式選模型、載入它；選 pi 自己的模型時，列出 pi 有登入資訊的那些，這台電腦什麼都不載入。", + "en": "With pi picked, `pi_picker()` first starts pi once to ask which models it has ([[agent/harnesses/pi_harness.py|def probe()]]), at most once a minute. " + "With This page's model picked under The model pi asks, it then picks and loads the model as the built-in harness does; with pi's own models, it " + "lists those pi has credentials for, and nothing loads on this machine."}, + Code("app/page.py", 'if st.session_state.pi_source == "page":', end="llm=current_llm())"), + ), + }), + {"zh": "載入這個網頁的模型時，其他的看是用什麼跑它（內建 harness 和 pi 問這個網頁的模型時都一樣）：", + "en": "Loading the page's model, the rest depends on what runs it (for the built-in harness and for pi on this page's model alike):"}, Tabs({ "transformers": ( {"zh": "`load_llm()` 建一個 `SimpleBackend`，把模型載入這台電腦（{{files}}）。剛從 vLLM (Docker) 換回 transformers 時，`current_llm()` 先停掉 vLLM 的容器，再載入模型。" "在 CUDA 上，記憶體還給系統之前要先放掉 cuBLAS 的工作區：它從第一次矩陣乘法起就一直留著，而且是切在 transformers 替權重預留的那一大塊記憶體裡，" - "不放掉的話，那一整塊都還不回去（[[harness/backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]]）。" - "換模型時 MTP 回到關閉；打開「MTP 推測解碼」時，`current_llm()` 第一次會順便載入草稿模型（[[harness/backends/simple_backend.py|def load_drafter(self):]]）。", + "不放掉的話，那一整塊都還不回去（[[agent/model_backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]]）。" + "換模型時 MTP 回到關閉；打開「MTP 推測解碼」時，`current_llm()` 第一次會順便載入草稿模型（[[agent/model_backends/simple_backend.py|def load_drafter(self):]]）。", "en": "`load_llm()` makes a `SimpleBackend`, which loads the model onto this machine ({{files}}). Right after transformers is picked again over vLLM (Docker), " "`current_llm()` first stops vLLM's container, and only then loads the model. On CUDA, cuBLAS's workspaces go before the memory is handed back: " "cuBLAS keeps one from the first matmul on, carved out of the segment transformers reserved for the weights, and while it lives none of that segment " - "can go back ([[harness/backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]]). Switching models turns MTP back off; the first time " - "MTP speculative decoding is switched on, `current_llm()` also loads the drafter ([[harness/backends/simple_backend.py|def load_drafter(self):]])."}, + "can go back ([[agent/model_backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]]). Switching models turns MTP back off; the first time " + "MTP speculative decoding is switched on, `current_llm()` also loads the drafter ([[agent/model_backends/simple_backend.py|def load_drafter(self):]])."}, ), "vllm-docker": ( {"zh": "`current_llm()` 先確定 Docker 裡的 vLLM 伺服器已經啟動，沒有就啟動它，拿到它的網址（{{backends}}）；`load_llm()` 再為它建一個 `VLLMBackend`。" @@ -906,59 +966,115 @@ def build(out: Path) -> list[Path]: "language: the errors that the harness and the tools send the model are worded in it for the whole turn. `tools` are the tools " "the model is offered: `run_python` only when Let the model write code is on and Docker is ready ({{execute}})."}, Code("app/page.py", "def current_settings("), - {"zh": "`LiveRun` 建立 `Run`，把 `self.events.append` 當成 `emit` 傳進去，連同附件和這段對話的資料夾，然後開背景 thread。" + {"zh": "`harness_for()` 決定這一輪由誰跑：`SimpleHarness` 跑在記憶體裡的模型上，或是 `PiHarness`；pi 問這個網頁的模型時，" + "`PiHarness` 也拿到那個模型和側欄的解碼設定（`PageModel`）。兩種都帶著這段對話的歷史、附件和資料夾，設定在這時候就固定下來。", + "en": "`harness_for()` decides what runs the turn: a `SimpleHarness` on the model in memory, or a `PiHarness`, which, when pi asks this page's model, " + "gets that model and the sidebar's decoding too (`PageModel`). Either gets the conversation so far, the attachments and the conversation's folder, " + "and its settings are fixed now."}, + Code("app/page.py", "def harness_for("), + {"zh": "`LiveRun` 拿 `self.events.append` 當成 `emit` 建出這個 harness，然後開背景 thread。" "thread 跑完 `run.run()` 之後才在 `finally` 裡放開鎖，所以就算 agent 出錯，鎖也一定會還回去。", - "en": "`LiveRun` creates the `Run`, passing `self.events.append` as its `emit`, along with the attachments and the conversation's folder, " - "and starts the background thread. " + "en": "`LiveRun` makes that harness with `self.events.append` as its `emit`, and starts the background thread. " "The thread releases the lock in a `finally` only after `run.run()` returns, so the lock always comes back, even when the agent fails."}, Code("app/page.py", "class LiveRun:", caption={"zh": "`daemon=True`：關掉 server 時不用等 agent 跑完。", "en": "`daemon=True`: stopping the server does not wait for the agent to finish."}), )), Section("prepare", "harness", - {"zh": "Run 準備：訊息、工具、system prompt", "en": "Preparing a run: messages, tools, system prompt"}, - {"zh": "`self.messages` 是 agent 的全部記憶，之後每一步都把它整份重新 render。", - "en": "`self.messages` is all the agent remembers; every step renders the whole of it again."}, ( - {"zh": "`Run.__init__` 把之前的對話歷史接上這次的問題，存成 `self.messages`（[[harness/agent.py|self.messages = [*history,]]）。" - "之後的工具呼叫和工具結果都 append 在後面；模型本身不記得任何事，每一步看到的都是這份 list 重新 render 出來的字串。" - "訊息用的是 Hugging Face 的通用格式（`role`、`content`、`tool_calls`），兩個家族都吃得下，格式差異交給各自的 chat template；" - "附了檔案時，這則訊息的 `content` 是一串 part（{{media}}）。`self.tools` 是這一輪模型拿得到的工具，" - "`self.files` 是這段對話的資料夾。`_stop` 和 `_decided` 是 UI thread 用來通知 agent 的兩個同步工具。", - "en": "`Run.__init__` appends this question to the conversation so far and keeps it as `self.messages` ([[harness/agent.py|self.messages = [*history,]]). " - "Tool calls and tool results are appended after it; the model itself remembers nothing, and what it sees in each step is this " - "list rendered into a string again. The messages use Hugging Face's common format (`role`, `content`, `tool_calls`), which " - "both families accept; each family's chat template takes care of its own format. With files attached, that message's `content` " - "is a list of parts ({{media}}). `self.tools` are the tools the model gets this turn, and `self.files` is the conversation's folder. " - "`_stop` and `_decided` are the two synchronization tools the UI thread signals the agent with."}, - Code("harness/agent.py", "def __init__(self, llm: Backend, history"), - {"zh": "`run()` 一開始拿出這個模型的 family，決定這一輪模型看得到什麼：收集 `self.tools` 裡每個工具的 `schema()` 給 chat template" - "（關掉「使用工具」時一個都沒有），" - "另外留一份「工具名稱 → 參數 schema」給 parser（Qwen 的參數值是純文字，要靠 schema 才知道型別）；" - "system prompt 也分有工具（[[harness/agent.py|SYSTEM_WITH_TOOLS = ]]）和沒工具兩個版本。" - "第一個事件 `run_start` 帶著模型和 family，畫面就知道這一輪要用哪個模型的格式來畫。", - "en": "`run()` starts by taking the model's family and deciding what the model gets to see this turn: it collects the `schema()` " - "of every tool in `self.tools` for the chat template (none with Use tools off), and keeps a map from tool name to parameters schema for the parser (Qwen's " - "argument values are plain text, and only the schema says their types); the system prompt comes in two versions, with tools " - "([[harness/agent.py|SYSTEM_WITH_TOOLS = ]]) and without. The first event, `run_start`, carries the model and the family, so the page " - "knows whose format to draw the turn in."}, - Code("harness/agent.py", "def run(self):", end='stop_reason, final = "max_steps", ""', - caption={"zh": "`stop_reason` 的預設值是 `max_steps`：迴圈跑完所有步數都沒有 break，停止原因就是步數上限。", - "en": "`stop_reason` starts out as `max_steps`: when the loop uses up its steps without a break, the step limit is why it stopped."}), - {"zh": "工具的 schema 就是一份 JSON Schema，包成 Hugging Face chat template 認得的 function 格式：", - "en": "A tool's schema is a JSON Schema, wrapped in the function format that Hugging Face chat templates know:"}, - Code("harness/tools.py", "def schema(self) -> dict:"), + {"zh": "準備一輪：訊息、工具、system prompt", "en": "Preparing a run: messages, tools, system prompt"}, + {"zh": "harness 準備這一輪模型看得到的東西：對話、工具和 system prompt。", + "en": "The harness gathers what the model will see this turn: the conversation, the tools and the system prompt."}, ( + Tabs({ + "simple": ( + {"zh": "`SimpleHarness.__init__` 把之前的對話歷史接上這次的問題，存成 `self.messages`（[[agent/harnesses/simple_harness.py|self.messages = [*history,]]）。" + "之後的工具呼叫和工具結果都 append 在後面；模型本身不記得任何事，每一步看到的都是這份 list 重新 render 出來的字串。" + "訊息用的是 Hugging Face 的通用格式（`role`、`content`、`tool_calls`），兩個家族都吃得下，格式差異交給各自的 chat template；" + "附了檔案時，這則訊息的 `content` 是一串 part（{{media}}）。`self.tools` 是這一輪模型拿得到的工具，" + "`self.files` 是這段對話的資料夾。`_stop` 和 `_decided` 是 UI thread 用來通知 agent 的兩個同步工具。", + "en": "`SimpleHarness.__init__` appends this question to the conversation so far and keeps it as `self.messages` ([[agent/harnesses/simple_harness.py|self.messages = [*history,]]). " + "Tool calls and tool results are appended after it; the model itself remembers nothing, and what it sees in each step is this " + "list rendered into a string again. The messages use Hugging Face's common format (`role`, `content`, `tool_calls`), which " + "both families accept; each family's chat template takes care of its own format. With files attached, that message's `content` " + "is a list of parts ({{media}}). `self.tools` are the tools the model gets this turn, and `self.files` is the conversation's folder. " + "`_stop` and `_decided` are the two synchronization tools the UI thread signals the agent with."}, + Code("agent/harnesses/simple_harness.py", "def __init__(self, llm: Backend, history"), + {"zh": "`run()` 一開始拿出這個模型的 family，決定這一輪模型看得到什麼：收集 `self.tools` 裡每個工具的 `schema()` 給 chat template" + "（關掉「使用工具」時一個都沒有），" + "另外留一份「工具名稱 → 參數 schema」給 parser（Qwen 的參數值是純文字，要靠 schema 才知道型別）；" + "system prompt 也分有工具（[[agent/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]]）和沒工具兩個版本。" + "第一個事件 `run_start` 帶著模型和 family，畫面就知道這一輪要用哪個模型的格式來畫；它也說這一輪是哪個 harness 跑的（`simple`，或 pi 的 `pi`）。", + "en": "`run()` starts by taking the model's family and deciding what the model gets to see this turn: it collects the `schema()` " + "of every tool in `self.tools` for the chat template (none with Use tools off), and keeps a map from tool name to parameters schema for the parser (Qwen's " + "argument values are plain text, and only the schema says their types); the system prompt comes in two versions, with tools " + "([[agent/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]]) and without. The first event, `run_start`, carries the model and the family, so the page " + "knows whose format to draw the turn in; it also says which harness ran the turn (`simple`, or pi's `pi`)."}, + Code("agent/harnesses/simple_harness.py", "def run(self):", end='stop_reason, final = "max_steps", ""', + caption={"zh": "`stop_reason` 的預設值是 `max_steps`：迴圈跑完所有步數都沒有 break，停止原因就是步數上限。", + "en": "`stop_reason` starts out as `max_steps`: when the loop uses up its steps without a break, the step limit is why it stopped."}), + {"zh": "工具的 schema 就是一份 JSON Schema，包成 Hugging Face chat template 認得的 function 格式：", + "en": "A tool's schema is a JSON Schema, wrapped in the function format that Hugging Face chat templates know:"}, + Code("agent/tools.py", "def schema(self) -> dict:"), + ), + "pi": ( + {"zh": "`PiHarness.run()` 啟動 pi，先選好模型和思考等級、關掉 pi 自己的自動重試（重試會變成另一個 step，所以請求失敗就當成錯誤顯示），再把問題當成 prompt 送出；" + "每一個指令都等到 pi 回應才送下一個，因為 pi 是同時處理指令的，和 `set_model` 一起送出的 prompt 會送到原本的模型。", + "en": "`PiHarness.run()` starts pi, picks the model and the thinking level and turns pi's own retries off (a retry would be a step of its own, so a " + "failed request is shown as an error instead), then sends the question as a prompt, each command only after pi has answered the last, since pi " + "handles commands concurrently and a prompt sent beside `set_model` would go to the old model."}, + Code("agent/harnesses/pi_harness.py", "self._pi = Process(self._args()", end="self._loop()"), + {"zh": "模型看得到什麼是 pi 的事：它自己的 system prompt、它的工具（側欄選模型看得到哪幾個），和它存在這段對話資料夾裡的 session；下一個問題用同一個 session 啟動 pi，" + "模型就讀得到前面的全部。網頁啟動 pi 時不載入你自己的 extension、skill、prompt 範本和 AGENTS.md（[[agent/harnesses/pi_harness.py|FLAGS = ]]），畫出來的就是 pi 本身的迴圈；" + "唯一多載入的是網頁的 extension `pi_extension.ts`（{{context}}、{{check}}）。問題附上的檔案存在 pi 的工作資料夾裡，問題寫出它們的名字，讀得懂圖片的模型也直接拿到圖片" + "（[[agent/harnesses/pi_harness.py|def _message(self)]]）。`run_start` 和內建 harness 的一樣，pi 問這個網頁的模型時帶著它的 family，問 pi 自己的模型時沒有。", + "en": "What the model gets to see is pi's: its own system prompt, its tools (of which the sidebar picks the ones the model sees) and the session it keeps in " + "the conversation's folder, on which the next question starts pi again, so the model reads all of it. The page starts pi without your own extensions, " + "skills, prompt templates and AGENTS.md ([[agent/harnesses/pi_harness.py|FLAGS = ]]), so what it draws is pi's own loop; the one thing it adds is its " + "extension, `pi_extension.ts` ({{context}}, {{check}}). Files attached to the question are saved in pi's working folder and named in the question, and a " + "model that reads images gets them too ([[agent/harnesses/pi_harness.py|def _message(self)]]). `run_start` is as the built-in harness's; it carries the " + "family when pi asks this page's model, and none when pi asks one of its own."}, + ), + }), )), Section("context", "context", {"zh": "脈絡：把整段對話 render 成一個字串", "en": "Context: rendering the whole conversation into one string"}, {"zh": "chat template 把 system、工具、對話接成一長串；模型這一步能看到的就只有它。", "en": "The chat template joins the system prompt, the tools and the conversation into one long string; it is all the model can see in this step."}, ( - {"zh": "每一圈從 `for step in range(1, s.max_steps + 1)` 開始，側欄的「步數上限」就是這個 range 的上界。第一件事是 render：", - "en": "Each step starts at `for step in range(1, s.max_steps + 1)`; the sidebar's Max steps is the top of that range. The first thing is to render:"}, - Code("harness/agent.py", "for step in range(1, s.max_steps + 1):", end="roles=_role_counts([system, *self.messages])", - caption={"zh": "`step_start` 讓畫面開一張新的 step 卡片；`context` 帶著模型實際讀到的完整字串送出。", - "en": "`step_start` has the page open a new step card; `context` goes out with the whole string the model really reads."}), - {"zh": "`render()` 呼叫 tokenizer 自帶的 chat template，模型在 vLLM 上跑時也是在這裡 render（{{files}}）。格式是模型自己的，harness 不用知道：", - "en": "`render()` calls the chat template that comes with the tokenizer, here even when the model runs on vLLM ({{files}}). The format is the model's own, " - "and the harness does not need to know it:"}, + Tabs({ + "simple": ( + {"zh": "每一圈從 `for step in range(1, s.max_steps + 1)` 開始，側欄的「步數上限」就是這個 range 的上界。第一件事是 render：", + "en": "Each step starts at `for step in range(1, s.max_steps + 1)`; the sidebar's Max steps is the top of that range. The first thing is to render:"}, + Code("agent/harnesses/simple_harness.py", "for step in range(1, s.max_steps + 1):", end="roles=_role_counts([system, *self.messages])", + caption={"zh": "`step_start` 讓畫面開一張新的 step 卡片；`context` 帶著模型實際讀到的完整字串送出。", + "en": "`step_start` has the page open a new step card; `context` goes out with the whole string the model really reads."}), + ), + "pi-page": ( + {"zh": "pi 開始一個 turn，把請求送給 provider `agent-loop`，也就是 `model_endpoint.py`：pi 的 system prompt、工具和對話，寫成 OpenAI 的 JSON。" + "和問 pi 自己的模型時一樣，extension 先回報這個請求，「脈絡」分頁把它收在脈絡下面。端點把它換回 `SimpleHarness` 用的那種對話" + "（[[agent/harnesses/model_endpoint.py|def conversation(]]）：工具呼叫的參數變回 dict，思考放進 family 的欄位，工具結果寫上它的工具名稱；" + "再和 `SimpleHarness` 一樣 render、encode，送出 `context`。這一步新加入的部分，從模型上一則訊息為止的對話算起（[[agent/harnesses/model_endpoint.py|def seen_before(]]）。", + "en": "pi starts a turn and sends its request to the provider `agent-loop`, which is `model_endpoint.py`: pi's system prompt, its tools and the " + "conversation, as OpenAI's JSON. As on pi's own models, the extension reports the request first, and the Context tab keeps it under the context. " + "The endpoint turns it back into the conversation `SimpleHarness` keeps ([[agent/harnesses/model_endpoint.py|def conversation(]]): a tool call's " + "arguments a dict again, the thinking under the family's key, a tool result with its tool's name. Then, as `SimpleHarness` does, it renders and " + "encodes it and sends `context`; what is new in the step is counted from the conversation as of the model's last message " + "([[agent/harnesses/model_endpoint.py|def seen_before(]])."}, + Code("agent/harnesses/model_endpoint.py", "messages = conversation(body, llm, serving.folder)", end='emit("context",'), + ), + "pi-provider": ( + {"zh": "網頁啟動的每個 pi 都載入一個小 extension，`pi_extension.ts`。pi 每次要向模型送出請求前，它把請求的內容當成一則通知送出；" + "在 RPC 模式下，通知是 stdout 上的一筆 `extension_ui_request`，`PiHarness` 把它變成 `request` 事件（[[agent/harnesses/pi_harness.py|def _request(self, payload)]]）：" + "step 卡片的脈絡段和「脈絡」分頁畫出 pi 的 system prompt、工具和對話，寫成 JSON，這一步新加的部分（上一步的回覆和觀察）另外標出來。" + "provider 的 API 怎麼把這個請求寫成模型讀的字串，pi 和 provider 都不說，所以這一段下面講的 chat template，在這裡看不到。", + "en": "Every pi the page starts loads one small extension, `pi_extension.ts`. Before each request pi sends the model, it sends what the request holds as " + "a notification; in RPC mode a notification is an `extension_ui_request` on stdout, which `PiHarness` turns into a `request` event " + "([[agent/harnesses/pi_harness.py|def _request(self, payload)]]): the step card's context section and the Context tab show pi's system prompt, its " + "tools and the conversation as JSON, with what this step added (the last step's reply and observation) marked. How the provider's API writes the " + "request out as the string its model reads, neither pi nor the provider says, so the chat template the rest of this section is about is not to be seen here."}, + Code("agent/harnesses/pi_extension.ts", 'pi.on("before_provider_request"', end="});"), + ), + }), + {"zh": "`render()` 呼叫 tokenizer 自帶的 chat template，模型在 vLLM 上跑時、pi 問這個網頁的模型時也都是在這裡 render（{{files}}）。格式是模型自己的，harness 不用知道：", + "en": "`render()` calls the chat template that comes with the tokenizer, here even when the model runs on vLLM, and when pi asks this page's model ({{files}}). " + "The format is the model's own, and the harness does not need to know it:"}, Tabs({ "gemma4": ( {"zh": "`<|turn>system` 開頭；開了思考模式，system turn 最前面多一個 `<|think|>`；" @@ -979,28 +1095,30 @@ def build(out: Path) -> list[Path]: "followed by `` (thinking mode on) or an empty `` (off)."}, ), }), - Code("harness/backends/simple_backend.py", "def render(self, messages"), - {"zh": "接著 `encode()` 把字串變成模型真正讀的 token（[[harness/agent.py|encoded = self.llm.encode(]]）。只有文字時，就是 tokenizer 切出來的 token；" + Code("agent/model_backends/simple_backend.py", "def render(self, messages"), + {"zh": "接著 `encode()` 把字串變成模型真正讀的 token（[[agent/harnesses/simple_harness.py|encoded = self.llm.encode(]]）。只有文字時，就是 tokenizer 切出來的 token；" "對話裡有圖片、聲音或影片時，chat template 在它們的位置只寫了一個佔位 token，`encode()` 把它展開成那個檔案佔的幾十到幾千個 token" "（{{media}}），所以 `context` 送出的是展開後的字串。`n_tokens` 是它的長度，畫面拿它和脈絡上限比，顯示用掉幾 %。" "`added_from` 是這一步新加入的內容從哪裡開始：先算 render 出來的字串和 `prev_sequence` 共同開頭的長度（{{record}}會說 `prev_sequence` 是什麼），" "再加上媒體展開多出來的長度：媒體都在使用者訊息裡，在新加入的內容前面。第 2 步以後，新加入的正好是工具結果。", - "en": "Next `encode()` turns the string into the tokens the model really reads ([[harness/agent.py|encoded = self.llm.encode(]]). With text " + "en": "Next `encode()` turns the string into the tokens the model really reads ([[agent/harnesses/simple_harness.py|encoded = self.llm.encode(]]). With text " "alone, they are the tokenizer's tokens; when the conversation has images, sound or video, the chat template wrote a single " "placeholder token where each goes, and `encode()` expands it into the tens to thousands of tokens that file takes ({{media}}), " "so `context` goes out with the expanded string. `n_tokens` is its length, which the page compares with the context limit to " "show how much of it is used. `added_from` is where what is new in this step begins: the length of the start that the rendered " "string shares with `prev_sequence` ({{record}} says what `prev_sequence` is), plus however much longer the media made it, since " "the media are all in user messages, before what is new. From step 2 on, what is new is exactly the tool result."}, - Code("harness/agent.py", "def common_prefix(", + Code("agent/harnesses/simple_harness.py", "def common_prefix(", caption={"zh": "逐字元比較，回傳兩個字串共同開頭的長度。", "en": "Compares character by character and returns the length of the start the two strings share."}), {"zh": "畫面上的「脈絡」分頁把 `context` 事件裡的字串原樣印出來（[[app/render.py|def context_html(]]），後面接著模型這一步寫的輸出；" "`added_from` 之後的部分標成「這一步新加入」（第 1 步沒有上一步，從最新一則使用者訊息開始標），family 列出的特殊 token 換成可以滑過看說明的深色標籤；" - "一長串同一個特殊 token，例如一張圖片展開成的幾百個 `<|image|>`，畫成一個標籤加上個數。", + "一長串同一個特殊 token，例如一張圖片展開成的幾百個 `<|image|>`，畫成一個標籤加上個數。pi 跑的 step 還有它送出的請求：" + "問 pi 自己的模型時畫在脈絡的位置，問這個網頁的模型時收在脈絡下面（[[app/render.py|def request_html(]]）。", "en": "The page's Context tab prints the string in the `context` event as it is ([[app/render.py|def context_html(]]), followed by what " "the model writes in this step; it marks everything after `added_from` as new in this step (in step 1, which has no step before it, " "from the newest user message on), and turns the special tokens the family lists into dark chips you can hover over for a note; " - "a long run of one special token, like the hundreds of `<|image|>` an image expands into, is one chip with the count."}, + "a long run of one special token, like the hundreds of `<|image|>` an image expands into, is one chip with the count. A step pi ran also has the request " + "it sent, in place of the context on pi's own models, and folded under it on this page's model ([[app/render.py|def request_html(]])."}, )), Section("media", "context", {"zh": "圖片、聲音和影片：變成脈絡裡的 token", "en": "Images, sound and video: tokens in the context"}, @@ -1016,31 +1134,31 @@ def build(out: Path) -> list[Path]: "data files such as CSV and Excel once Let the model write code is on."}, Tabs({ "transformers": ( - Code("harness/backends/simple_backend.py", "def _modalities(self)"), + Code("agent/model_backends/simple_backend.py", "def _modalities(self)"), ), "vllm": ( {"zh": "在 vLLM 伺服器上，是這些再扣掉伺服器不收的：載入時，backend 對 checkpoint 讀得懂的每一種媒體，各送一個只生成一個 token 的請求問伺服器，" - "模型讀得懂的就是伺服器收的那幾種（[[harness/backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]]）。只有 `/inference/v1/generate` 收媒體，所以沒加 `--enable-scale-out` 的伺服器一種都不收。" + "模型讀得懂的就是伺服器收的那幾種（[[agent/model_backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]]）。只有 `/inference/v1/generate` 收媒體，所以沒加 `--enable-scale-out` 的伺服器一種都不收。" "不收的那幾種是給 `run_python` 的一般檔案；模型分頁列出伺服器不收哪幾種、為什麼（[[app/render.py|def refused_html(]]），輸入框上方也說哪些不能附上。", "en": "On a vLLM server, it is those less what the server does not take: at load, the backend asks the server with a one-token request for each kind of " - "media the checkpoint reads, and the model reads those it takes ([[harness/backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]]). Only `/inference/v1/generate` takes media, so a server started without " + "media the checkpoint reads, and the model reads those it takes ([[agent/model_backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]]). Only `/inference/v1/generate` takes media, so a server started without " "`--enable-scale-out` takes none. A kind it refuses is a plain file for `run_python`; the Model tab lists the kinds the server refused and why " "([[app/render.py|def refused_html(]]), and above the input box the page says which cannot be attached."}, - Code("harness/backends/vllm_backend.py", "def _probe("), + Code("agent/model_backends/vllm_backend.py", "def _probe("), ), }), - {"zh": "`start_run()` 把每個檔案存進這段對話的資料夾，名字只留一般字元，因為模型可能要在程式裡打出它（[[harness/media.py|def save(]]）。" + {"zh": "`start_run()` 把每個檔案存進這段對話的資料夾，名字只留一般字元，因為模型可能要在程式裡打出它（[[agent/media.py|def save(]]）。" "使用者訊息裡，模型讀得懂的檔案各是一個 part，排在文字前面；其他檔案模型看不到內容，只在文字後面多一段說明：" "`run_python` 在 `/data` 底下找得到它們，CSV、TSV、純文字和 Markdown 檔再附上第一行（CSV 的話就是欄位名稱；JSON 不附，它的第一行多半只是 `[`）。少了這一行，小模型寫程式時會亂猜欄位，" "例如不存在的 `Date`，然後出錯；多給幾列資料也不行，Gemma 會把那幾列直接抄進程式，只算它們，不去讀檔案。" "這段說明是寫給模型看的，所以和 system prompt 一樣用英文。", "en": "`start_run()` saves each file in the conversation's folder, keeping its name to plain characters, since the model may have to " - "type it into code ([[harness/media.py|def save(]]). In the user's message, each file the model reads is a part of its own, before the " + "type it into code ([[agent/media.py|def save(]]). In the user's message, each file the model reads is a part of its own, before the " "text; the model cannot see into other files, which get a note after the text: `run_python` finds them under `/data`, and a CSV, " "TSV, plain-text or Markdown file comes with its first line (for a CSV, the column names; not a JSON file, whose first line is mostly a `[`). Without it a small model guesses the columns " "in its code, a `Date` that is not there, say, and fails; a few rows of data are too many, as Gemma copies them into its code and " "works out the answer from them alone instead of reading the file. The note is for the model, so it is in English, like the system prompt."}, - Code("harness/media.py", "def user_content("), + Code("agent/media.py", "def user_content("), {"zh": "chat template 在每個 part 的位置寫一個佔位 token，模型的 processor 再把每個佔位 token 展開成那個檔案佔的 token：", "en": "The chat template writes a placeholder token where each part goes, and the model's processor expands each placeholder into the tokens that file takes:"}, Tabs({ @@ -1064,16 +1182,16 @@ def build(out: Path) -> list[Path]: Tabs({ "transformers": ( {"zh": "`encode()` 把對話裡所有媒體依出現的順序交給 processor，由它展開：", "en": "`encode()` hands the processor all the media in the conversation, in the order they come, to expand:"}, - Code("harness/backends/simple_backend.py", "def encode(self, prompt", + Code("agent/model_backends/simple_backend.py", "def encode(self, prompt", caption={"zh": "`media_tokens` 數每種媒體佔了幾個 token，step 卡片的脈絡段會寫出來。", "en": "`media_tokens` counts how many tokens each kind of media takes; the step card's context section shows it."}), {"zh": "processor 要的是解碼好的資料，由 `media.py` 準備：圖片轉成 RGB，照片照 EXIF 轉正；聲音用 PyAV 解碼成單聲道，" "取樣率改成模型的 feature extractor 要的 16 kHz；影片平均取最多 64 張畫格、縮小到長邊 768 像素，" - "再附上取完之後的畫格率，processor 寫出的時間才對（[[harness/media.py|def load_video(]]）。每個檔案只解碼一次，之後每一步都重複使用。", + "再附上取完之後的畫格率，processor 寫出的時間才對（[[agent/media.py|def load_video(]]）。每個檔案只解碼一次，之後每一步都重複使用。", "en": "The processor wants decoded data, which `media.py` prepares: images become RGB, and photos are turned upright by their EXIF; " "sound is decoded with PyAV into mono at the 16 kHz the model's feature extractor expects; a video gives at most 64 frames taken " "evenly, shrunk to 768 pixels on the longer side, with the frame rate of what was kept, so the times the processor writes are " - "right ([[harness/media.py|def load_video(]]). Each file is decoded once and reused in every step after."}, + "right ([[agent/media.py|def load_video(]]). Each file is decoded once and reused in every step after."}, {"zh": "prefill 時，編碼器的輸出換掉這些佔位 token 的 embedding。圖片和影片先跑一次視覺編碼器再交給 prefill，和 `generate()` 的做法一樣；" "直接交給 forward 的話，transformers 5.20 的 Gemma 4 會漏掉影片的畫格數。Qwen3.5 的位置編碼（M-RoPE）要知道圖片的長寬，" "所以除了像素以外的輸入都留著；它也會把上一個 prompt 的位置偏移記在模型上，每次 prefill 前要清掉。" @@ -1084,17 +1202,17 @@ def build(out: Path) -> list[Path]: "so every input but the pixels stays; it also keeps the last prompt's position offset on the model, which is cleared before " "every prefill. After the prefill, the media are in the KV cache, and decoding is the same as for text. " "`tests/test_real_model.py` checks that with an image, a video and a recording, too, greedy output matches `generate()`."}, - Code("harness/backends/simple_backend.py", "def _encoded_media("), + Code("agent/model_backends/simple_backend.py", "def _encoded_media("), ), "vllm": ( {"zh": "檔案跟著 chat template 寫的佔位 token 一起送到伺服器（圖片轉正存成 PNG、聲音解碼成 WAV、影片送原檔，" - "[[harness/backends/vllm_backend.py|def content_part(]]），由伺服器展開每個佔位 token，編碼器也在伺服器的 prefill 裡跑。" + "[[agent/model_backends/vllm_backend.py|def content_part(]]），由伺服器展開每個佔位 token，編碼器也在伺服器的 prefill 裡跑。" "`encode()` 先用一個只生成一個 token 的請求問伺服器怎麼展開，所以脈絡分頁上是伺服器讀到的 prompt；生成時伺服器若展開得不一樣，就當成錯誤。", "en": "The files go to the server next to the placeholders the chat template wrote (an image upright as PNG, sound decoded to WAV, a video as " - "the file, [[harness/backends/vllm_backend.py|def content_part(]]), the server expands each placeholder, and the encoders run in its prefill. " + "the file, [[agent/model_backends/vllm_backend.py|def content_part(]]), the server expands each placeholder, and the encoders run in its prefill. " "`encode()` asks the server how first, with a request that generates one token, so the Context tab shows the prompt as the server reads it; " "a different expansion at generation time is an error."}, - Code("harness/backends/vllm_backend.py", "def encode(self, prompt"), + Code("agent/model_backends/vllm_backend.py", "def encode(self, prompt"), ), }), )), @@ -1102,78 +1220,106 @@ def build(out: Path) -> list[Path]: {"zh": "模型：prefill，然後一次一個 token", "en": "Model: prefill, then one token at a time"}, {"zh": "一次 forward 讀完整段 prompt，之後每生成一個 token 再 forward 一次。", "en": "One forward pass reads the whole prompt; after that, each new token takes another forward pass."}, ( - {"zh": "harness 用一個 for 迴圈消費 `generate()` 產生的事件：`prefill` 和每個 `token` 都加上 step 編號轉送出去，" - "最後的 `done` 留下來，改名成 `model_end` 送出。", - "en": "The harness consumes the events that `generate()` yields in a for loop: `prefill` and each `token` are passed on with the step " - "number, and the final `done` is kept and sent on as `model_end`."}, - Code("harness/agent.py", "done = None", end='self.emit("model_end"', - caption={"zh": "傳進去的 `self._stop.is_set` 是一個函式：模型每生成一個 token 前都會呼叫它，問「要停了嗎？」。", - "en": "The `self._stop.is_set` passed in is a function: the model calls it before every token to ask whether to stop."}), + Tabs({ + "simple": ( + {"zh": "harness 用一個 for 迴圈消費 `generate()` 產生的事件：`prefill` 和每個 `token` 都加上 step 編號轉送出去，" + "最後的 `done` 留下來，改名成 `model_end` 送出。", + "en": "The harness consumes the events that `generate()` yields in a for loop: `prefill` and each `token` are passed on with the step " + "number, and the final `done` is kept and sent on as `model_end`."}, + Code("agent/harnesses/simple_harness.py", "done = None", end='self.emit("model_end"', + caption={"zh": "傳進去的 `self._stop.is_set` 是一個函式：模型每生成一個 token 前都會呼叫它，問「要停了嗎？」。", + "en": "The `self._stop.is_set` passed in is a function: the model calls it before every token to ask whether to stop."}), + ), + "pi-page": ( + {"zh": "端點用同樣的方式消費 `generate()` 的事件，每一個都送進 pi 的那一步。pi 先開始一個 turn 才送出請求，兩件事從不同的路到頁面，" + "所以請求先等頁面開好那一步（[[agent/harnesses/pi_harness.py|def _serving_step(self)]]）。思考和回覆的每個 token 也一邊生成一邊傳回 pi，" + "分別是 `reasoning_content` 和 `content`。模型每生成一個 token 前都問一次要不要停：這一輪被停止了，或是 pi 已經不收了（它被中止時）。", + "en": "The endpoint consumes `generate()`'s events in the same way, sending each into pi's step. pi starts a turn and then sends its request, and the two " + "reach the page by different ways, so the request first waits until the page has opened that step ([[agent/harnesses/pi_harness.py|def _serving_step(self)]]). " + "Each token of the thinking and of the reply also goes back to pi as it comes, as `reasoning_content` and `content`. Before every token the model asks " + "whether to stop: whether the turn is being stopped, or pi has stopped listening (as it does when it is aborted)."}, + Code("agent/harnesses/model_endpoint.py", "for ev in llm.generate(encoded, serving.sampling"), + ), + "pi-provider": ( + {"zh": "這裡沒有 backend：provider 的 API 把文字一段一段串流回 pi，pi 把每一段當成 `message_update` 紀錄送出，`PiHarness` 轉成 `delta`，帶著它是哪一種 channel" + "（[[agent/harnesses/pi_harness.py|def _update(self, e: dict)]]）；step 卡片照 channel 畫出這些文字，沒有 token 的分界（[[app/render.py|def streamed_html(]]）。" + "訊息結束時，`model_end` 帶著 provider 報的 token 數、第一段輸出花了多久，和照 pi 的價目估的費用（[[agent/harnesses/pi_harness.py|def _message_end(self, message: dict)]]）。" + "下面的頁籤講的是這個網頁的模型。", + "en": "No backend runs here: the provider's API streams text back to pi, pi sends each piece on as a `message_update` record, and `PiHarness` turns it into " + "`delta`, with its channel ([[agent/harnesses/pi_harness.py|def _update(self, e: dict)]]); the step card draws the pieces by channel, without token " + "boundaries ([[app/render.py|def streamed_html(]]). When the message ends, `model_end` carries the token counts the provider reported, how long the " + "first output took and the cost at pi's prices ([[agent/harnesses/pi_harness.py|def _message_end(self, message: dict)]]). The tabs below are about " + "this page's model."}, + Code("agent/harnesses/pi_harness.py", "def _update(self, e: dict):"), + ), + }), + {"zh": "這個網頁的模型怎麼生成（內建 harness 和 pi 問這個網頁的模型時都一樣），看是用什麼跑它：", + "en": "How the page's model generates (for the built-in harness and for pi on this page's model alike) depends on what runs it:"}, Tabs({ "transformers": ( {"zh": "**prefill**：`encode()` 已經把整個 prompt 轉成 token id，一次 forward 算完；有媒體時，這次 forward 也把編碼器的輸出放進它們的位置（{{media}}）。" "`use_cache=True` 留下每一層的狀態（一般 attention 層是 key / value，" "Qwen3.5 的 Gated DeltaNet 層是一個固定大小的狀態），之後每個新 token 只需要算自己那一個位置；" - "`logits_to_keep=1` 只取最後一個位置的 logits，因為只需要「下一個 token」的分布（[[harness/backends/simple_backend.py|def _forward(]]）。" + "`logits_to_keep=1` 只取最後一個位置的 logits，因為只需要「下一個 token」的分布（[[agent/model_backends/simple_backend.py|def _forward(]]）。" "prefill 的時間主要看 prompt 有多長，模型分頁的表格裡有每一步的 prefill ms。", "en": "**prefill**: the whole prompt, already token ids from `encode()`, goes through one forward pass, which also puts the encoders' " "output in place of any media ({{media}}). `use_cache=True` keeps each layer's state " "(the keys and values in ordinary attention layers, a fixed-size state in Qwen3.5's Gated DeltaNet layers), so each new token " "needs only its own position computed; `logits_to_keep=1` returns the logits for the last position only, since only the " - "distribution of the next token is needed ([[harness/backends/simple_backend.py|def _forward(]]). Prefill time depends mostly on the prompt's length; " + "distribution of the next token is needed ([[agent/model_backends/simple_backend.py|def _forward(]]). Prefill time depends mostly on the prompt's length; " "the table in the Model tab has each step's prefill ms."}, - Code("harness/backends/simple_backend.py", "def generate(self, prompt", end='yield {"type": "prefill"', + Code("agent/model_backends/simple_backend.py", "def generate(self, prompt", end='yield {"type": "prefill"', caption={"zh": "prompt 如果以 family 的 `thought_prefix` 結尾（Gemma 的 `<|channel>thought`、Qwen 的 ``），模型一開始就在思考裡。", "en": "If the prompt ends with the family's `thought_prefix` (`<|channel>thought` for Gemma, `` for Qwen), the model starts out inside its thinking."}), {"zh": "**解碼迴圈**從 `tokens` 一個一個拿抽好的 token：平常由 `_one_at_a_time()` 產生，打開 MTP 時改由 `_drafted()`（{{mtp}}），" "迴圈本身不用知道是哪一種。每一圈：檢查停止 → 拿到一個 token → " "判斷它是不是特殊 token、有沒有切換 channel：哪些 token 切到思考、工具呼叫或回覆，由 family 的 `channel_switch` 決定" - "（Gemma：[[harness/families/gemma4.py|channel_switch={]]；Qwen：[[harness/families/qwen3_5.py|channel_switch={]]）→ " + "（Gemma：[[agent/model_families/gemma4.py|channel_switch={]]；Qwen：[[agent/model_families/qwen3_5.py|channel_switch={]]）→ " "把目前為止的所有 token 整段 decode 再取差值，這樣被拆成好幾個 byte token 的中文字會等到湊齊才輸出 → 送出 `token` 事件。", "en": "**The decoding loop** takes sampled tokens from `tokens` one at a time: they normally come from `_one_at_a_time()`, and from " "`_drafted()` with MTP on ({{mtp}}); the loop itself does not need to know which. Each time around: check for stop → get a token → " "work out whether it is a special token and whether it switches channel, where the family's `channel_switch` says which tokens " - "switch to thinking, a tool call or the reply (Gemma: [[harness/families/gemma4.py|channel_switch={]]; Qwen: " - "[[harness/families/qwen3_5.py|channel_switch={]]) → decode all the tokens so far and take the difference, so that a Chinese character " + "switch to thinking, a tool call or the reply (Gemma: [[agent/model_families/gemma4.py|channel_switch={]]; Qwen: " + "[[agent/model_families/qwen3_5.py|channel_switch={]]) → decode all the tokens so far and take the difference, so that a Chinese character " "split across several byte tokens comes out only once it is whole → send a `token` event."}, - Code("harness/backends/simple_backend.py", "decode_start = time.perf_counter()", end="**details,"), + Code("agent/model_backends/simple_backend.py", "decode_start = time.perf_counter()", end="**details,"), {"zh": "生成到停止 token，或到了 `max_new_tokens`（側欄的「每步 token 上限」），就跳出迴圈。", "en": "A stop token, or reaching `max_new_tokens` (the sidebar's Max tokens per step), ends the loop."}, - Code("harness/backends/simple_backend.py", "if token_id in self.eos_ids:", end="if len(generated) == sampling.max_new_tokens:"), + Code("agent/model_backends/simple_backend.py", "if token_id in self.eos_ids:", end="if len(generated) == sampling.max_new_tokens:"), {"zh": "還沒停，迴圈就再跟 `tokens` 要下一個。`_one_at_a_time()` 這時才把剛抽到的 token 餵回模型，帶著快取算出下一個位置的 logits；" "它是一個 generator，迴圈跳出之後就不會再多跑一次 forward。", "en": "If it has not stopped, the loop asks `tokens` for the next one. Only then does `_one_at_a_time()` feed the token it just sampled " "back into the model, with the cache, for the next position's logits; it is a generator, so once the loop ends, no extra forward pass runs."}, - Code("harness/backends/simple_backend.py", "def _one_at_a_time("), + Code("agent/model_backends/simple_backend.py", "def _one_at_a_time("), {"zh": "最後的 `done` 告訴 harness 為什麼停（`finish`：`stop_token`、`max_new_tokens` 或 `user_stopped`）、停在哪個 token、" "生成速度，以及整段原始輸出 `raw`（保留特殊 token，因為 harness 要靠它們解析）。", "en": "The final `done` tells the harness why it stopped (`finish`: `stop_token`, `max_new_tokens` or `user_stopped`), at which token, " "how fast it generated, and the whole raw output, `raw` (special tokens included, because the harness parses by them)."}, - Code("harness/backends/simple_backend.py", "decode_s = time.perf_counter() - decode_start", end='"raw": self.tokenizer.decode(generated'), + Code("agent/model_backends/simple_backend.py", "decode_s = time.perf_counter() - decode_start", end='"raw": self.tokenizer.decode(generated'), ), "vllm": ( {"zh": "`VLLMBackend.generate()` 送出一樣的事件，內容來自伺服器用 server-sent events 一段一段送回的 token（每個 token 帶了什麼，見{{sampling}}）。" "伺服器第一次回 token 時送出 `prefill`，所以它的時間是等到第一個 token 的時間，包含網路：", "en": "`VLLMBackend.generate()` yields the same events, from the tokens the server streams back as server-sent events (what each token comes with is in " "{{sampling}}). `prefill` goes out once the first token is back, so its time is the time to the first token, the network included:"}, - Code("harness/backends/vllm_backend.py", "def generate(self, prompt", end='yield {"type": "prefill"'), + Code("agent/model_backends/vllm_backend.py", "def generate(self, prompt", end='yield {"type": "prefill"'), {"zh": "vLLM 會留著和上一個 prompt 相同開頭的 KV cache（prefix caching），所以第 2 步開始只讀和上一步不一樣的部分：" "chat template 重新寫過的模型回合，加上新的觀察；`SimpleBackend` 每一步都從頭讀完整個 prompt。", "en": "vLLM keeps the KV cache of what a prompt shares with the last one (prefix caching), so from step 2 on it reads only what differs from the last step: " "the model's turn as the chat template rewrote it, and the new observation; `SimpleBackend` reads the whole prompt from the start every step."}, {"zh": "拿到 token 之後和 `SimpleBackend` 一樣：family 的 `channel_switch` 決定切到哪個 channel，整段 decode 再取差值，送出 `token` 事件" - "（[[harness/backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]]）。什麼時候停由伺服器決定：請求帶著停止 token（`stop_token_ids`）" + "（[[agent/model_backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]]）。什麼時候停由伺服器決定：請求帶著停止 token（`stop_token_ids`）" "和 `max_tokens`，也就是側欄的「每步 token 上限」，超過伺服器的脈絡上限就再縮短（{{sampling}}）。按下停止時，網頁關掉這條串流，伺服器就不再生成" - "（[[harness/backends/vllm_backend.py|tokens.close()]]）。", + "（[[agent/model_backends/vllm_backend.py|tokens.close()]]）。", "en": "From the token on it is as in `SimpleBackend`: the family's `channel_switch` says which channel, decoding all the tokens so far gives the text, and a " - "`token` event goes out ([[harness/backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]]). The server decides when to stop: the request " + "`token` event goes out ([[agent/model_backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]]). The server decides when to stop: the request " "carries the stop tokens (`stop_token_ids`) and `max_tokens`, the sidebar's Max tokens per step, cut to what fits in the server's context ({{sampling}}). " - "Pressing stop closes the stream, which tells the server to stop generating ([[harness/backends/vllm_backend.py|tokens.close()]])."}, + "Pressing stop closes the stream, which tells the server to stop generating ([[agent/model_backends/vllm_backend.py|tokens.close()]])."}, {"zh": "最後的 `done` 和 `SimpleBackend` 的一樣：伺服器的 `finish_reason` 是 `stop` 就是 `stop_token`、`length` 就是 `max_new_tokens`，其他都當成錯誤；" "伺服器推測解碼時還帶著它的統計（{{mtp}}）。", "en": "The final `done` is the same as `SimpleBackend`'s: the server's `finish_reason` `stop` becomes `stop_token` and `length` `max_new_tokens`, and anything " "else is an error; when the server decodes speculatively, it also carries the server's counts ({{mtp}})."}, - Code("harness/backends/vllm_backend.py", "if finish is None:", end='"raw": self.tokenizer.decode(generated'), + Code("agent/model_backends/vllm_backend.py", "if finish is None:", end='"raw": self.tokenizer.decode(generated'), {"zh": "greedy 的輸出不一定和 `generate()` 逐 token 相同：vLLM 會把請求合成一批跑，也有自己的 kernel。和 `generate()` 完全一樣，是 `SimpleBackend` 才有的保證。", "en": "Greedy output need not match `generate()` token for token: vLLM batches requests and has kernels of its own. Matching `generate()` exactly is " "`SimpleBackend`'s promise only."}, @@ -1198,31 +1344,31 @@ def build(out: Path) -> list[Path]: "`generate()`'s argmax does (in bfloat16, logits tie more often than one would think). Otherwise, in order: logits ÷ T → keep only the top k " "(top-k; 0 = no limit) → softmax again → add up from the most likely down until the cumulative probability reaches top-p " "(nucleus, keeping at least one) → renormalize → draw one with `torch.multinomial`. None of this depends on the model; both families share it."}, - Code("harness/backends/simple_backend.py", "def _sample(self, logits"), + Code("agent/model_backends/simple_backend.py", "def _sample(self, logits"), ), "vllm": ( {"zh": "抽樣在伺服器上做。側欄的每個設定都照送，其他取樣設定都設成不起作用，免得伺服器從 checkpoint 帶來的預設值改變分布：", "en": "The server does the sampling. Every sidebar setting is sent, and every other sampling setting is set to do nothing, so that defaults the server " "took from the checkpoint cannot change the distribution:"}, - Code("harness/backends/vllm_backend.py", "def _sampling_params("), + Code("agent/model_backends/vllm_backend.py", "def _sampling_params("), {"zh": "一般的 `vllm serve` 只有 `/v1/completions`：每個 token 帶著它的 log 機率，" "以及那個位置機率最高的 5 個 token 和它們的 log 機率，key 寫成 `token_id:`；排名只有在 token 落在前 5 名裡時才算得出來。" "vLLM 預設回傳的是模型原本的分布（`--logprobs-mode raw_logprobs`），和 `SimpleBackend` 的 `p` 意思一樣。", "en": "A plain `vllm serve` has only `/v1/completions`: each token comes with its log probability " "and the 5 most likely tokens at its position with theirs, keyed `token_id:`, so the rank is known only when the token is among those 5. " "By default vLLM sends the model's own distribution (`--logprobs-mode raw_logprobs`), which means the same as `SimpleBackend`'s `p`."}, - Code("harness/backends/vllm_backend.py", "def _details("), + Code("agent/model_backends/vllm_backend.py", "def _details("), {"zh": "可是伺服器也可能用別的模式啟動，而沒有哪條路會說是哪一種，所以載入時 backend 先問一個 token：temperature 1、top-k 2、" - "要最可能的 5 個（[[harness/backends/vllm_backend.py|def logprobs_mode(]]）。processed 模式下，被 top-k 剃掉的 3 個會回 -9999；" + "要最可能的 5 個（[[agent/model_backends/vllm_backend.py|def logprobs_mode(]]）。processed 模式下，被 top-k 剃掉的 3 個會回 -9999；" "log 機率加起來最多是 1，logits 不會。`processed_logprobs` 是 token 被抽出來的那個分布，套過 temperature、top-k、top-p，" "所以選中的 token 的就是取樣時的機率，送成 `p_sampled`，不送 `p` 和前 5 名；列出來的有幾個被剃掉時，也知道剩下幾個。logits 不是機率，就不當成機率送。" "排名每種模式都一樣：temperature 不改順序，top-k、top-p 留下的是最可能的那幾個。", "en": "Yet a server may run another mode, and no route says which, so at load the backend asks for one token at temperature 1 with top-k 2 and the 5 " - "most likely tokens ([[harness/backends/vllm_backend.py|def logprobs_mode(]]). In a processed mode, the 3 that top-k cut come back as -9999; log probabilities " + "most likely tokens ([[agent/model_backends/vllm_backend.py|def logprobs_mode(]]). In a processed mode, the 3 that top-k cut come back as -9999; log probabilities " "add up to at most 1, and logits do not. `processed_logprobs` is the distribution the token was drawn from, after temperature, top-k and top-p, " "so the chosen token's is its sampling probability, sent as `p_sampled`, and no `p` or top 5 is sent; when some of those listed were cut, how many " "were left is known too. Logits are no probabilities, and none is sent as one. The rank is the same in every mode: temperature keeps the order, and top-k and top-p keep the most likely tokens."}, - Code("harness/backends/vllm_backend.py", "def _token_details("), + Code("agent/model_backends/vllm_backend.py", "def _token_details("), {"zh": "伺服器加上 `--enable-scale-out` 時，還有一條 `/inference/v1/generate`：一樣送 token id 進去、token id 出來，但說得更多。" "載入時 backend 用幾個只生成一個 token 的請求問伺服器能做什麼：有沒有這條路、有沒有送 sampling mask、收哪些媒體（{{media}}）。這條路帶來的：", "en": "Started with `--enable-scale-out`, the server also has `/inference/v1/generate`: token ids in and token ids out as well, but it tells more. At load " @@ -1239,14 +1385,14 @@ def build(out: Path) -> list[Path]: {"zh": "**`--return-sampling-mask`**：每個位置 top-k、top-p 留下了哪些 token，由此知道剩下幾個候選。vLLM 只在 `--logprobs-mode processed_logprobs` " "下開它，所以有 mask 的伺服器送回的是抽樣分布：看得到取樣時的機率和剩下幾個候選，看不到模型給的機率、前 5 名，也看不到哪幾個被剃掉。" "這時它也不接受 temperature 0 和 top-k 0（mask 的大小要靠 top-k 限制），所以側欄的滑桿從 0.05 和 1 開始。" - "沒有 `--enable-scale-out` 的伺服器看不到 mask，所以載入時也要一個 greedy 的 token，被拒絕就知道它開了 mask（[[harness/backends/vllm_backend.py|def _refuses_greedy(]]）。" - "greedy 不用問伺服器：只剩一個候選，抽中的機率是 1（[[harness/backends/vllm_backend.py|def _greedy(]]）。", + "沒有 `--enable-scale-out` 的伺服器看不到 mask，所以載入時也要一個 greedy 的 token，被拒絕就知道它開了 mask（[[agent/model_backends/vllm_backend.py|def _refuses_greedy(]]）。" + "greedy 不用問伺服器：只剩一個候選，抽中的機率是 1（[[agent/model_backends/vllm_backend.py|def _greedy(]]）。", "en": "**`--return-sampling-mask`**: the tokens top-k and top-p left at each position, from which comes how many candidates were left. vLLM allows it " "only with `--logprobs-mode processed_logprobs`, so a server with masks sends the sampling distribution: the sampling probability and the " "candidates left show, and the model probability, the top 5 and which of them were cut do not. Such a server also refuses temperature 0 and " "top-k 0 (top-k bounds the mask), so the sidebar's sliders start at 0.05 and 1. A server without `--enable-scale-out` shows no mask, so at " - "load it is also asked for a greedy token, and a refusal says it has masks on ([[harness/backends/vllm_backend.py|def _refuses_greedy(]]). Greedy decoding " - "needs no server: one candidate left, drawn with probability 1 ([[harness/backends/vllm_backend.py|def _greedy(]])."}, + "load it is also asked for a greedy token, and a refusal says it has masks on ([[agent/model_backends/vllm_backend.py|def _refuses_greedy(]]). Greedy decoding " + "needs no server: one candidate left, drawn with probability 1 ([[agent/model_backends/vllm_backend.py|def _greedy(]])."}, ), {"zh": "vLLM (Docker) 的伺服器加了 `--enable-scale-out`、用預設的 `raw_logprobs`，所以看得到每個 token 的排名、模型給的機率和前 5 名。", "en": "vLLM (Docker) starts its server with `--enable-scale-out` and the default `raw_logprobs`, so every token's rank, the model probability and the top 5 show."}, @@ -1255,13 +1401,13 @@ def build(out: Path) -> list[Path]: {"zh": "浮窗裡的數字：`p` 是被選中的 token 在原始分布的機率；`rank` 是它的名次；" "`kept` 是剃完還剩幾個候選；`p_sampled` 是它在剃完、重新正規化後的分布裡的機率。" "step 卡片上 token 的底色深淺就是用 `p` 算的（[[app/render.py|def heat(p: float)]]）：越深，代表模型原本越不看好它，卻被抽中了。" - "這些數字列在 `backend.py` 的 `TOKEN_DETAILS`（[[harness/backends/backend.py|TOKEN_DETAILS = (]]）：transformers 全都知道，vLLM 伺服器知道哪些看它怎麼啟動，" + "這些數字列在 `backend.py` 的 `TOKEN_DETAILS`（[[agent/model_backends/backend.py|TOKEN_DETAILS = (]]）：transformers 全都知道，vLLM 伺服器知道哪些看它怎麼啟動，" "不知道的就不畫，不會用別的數字代替（{{backends}}）。", "en": "The numbers in the popover: `p` is the chosen token's probability in the model's own distribution; `rank` " "is its place; `kept` is how many candidates were left after the cuts; `p_sampled` is its probability in the cut and " "renormalized distribution. The shade behind a token in a step card comes from `p` ([[app/render.py|def heat(p: float)]]): " "the darker it is, the less the model expected that token, and yet it was drawn. These numbers are listed in `backend.py`'s " - "`TOKEN_DETAILS` ([[harness/backends/backend.py|TOKEN_DETAILS = (]]): transformers knows them all, a vLLM server what it was started to tell, and what " + "`TOKEN_DETAILS` ([[agent/model_backends/backend.py|TOKEN_DETAILS = (]]): transformers knows them all, a vLLM server what it was started to tell, and what " "a backend does not know is left out, never drawn from some other number ({{backends}})."}, )), Section("mtp", "model", @@ -1278,16 +1424,16 @@ def build(out: Path) -> list[Path]: "all in one forward pass. That is speculative decoding; MTP (multi-token prediction) is the guesser Google trained for Gemma 4."}, Tabs({ "transformers": ( - {"zh": "猜的是草稿模型 `google/gemma-4-E2B-it-assistant`（[[harness/families/__init__.py|DRAFTERS = {]]）：約 78M 參數、4 層、隱藏層維度 256，" + {"zh": "猜的是草稿模型 `google/gemma-4-E2B-it-assistant`（[[agent/model_families/__init__.py|DRAFTERS = {]]）：約 78M 參數、4 層、隱藏層維度 256，" "主模型是 5.1B。它沒有自己的 KV cache：Gemma 4 E2B 的 35 層裡，後 20 層本來就不自己算 key / value，而是共用前面兩層" "（最後一個滑動視窗層和最後一個 full attention 層）算好的，草稿模型也讀這一份。第一次打開 MTP 時才載入，" "每次猜幾個 token 寫在它的 generation config（`num_assistant_tokens` = 6）。", - "en": "The guesses come from the drafter `google/gemma-4-E2B-it-assistant` ([[harness/families/__init__.py|DRAFTERS = {]]): about 78M " + "en": "The guesses come from the drafter `google/gemma-4-E2B-it-assistant` ([[agent/model_families/__init__.py|DRAFTERS = {]]): about 78M " "parameters, 4 layers, hidden size 256, next to the model's 5.1B. It has no KV cache of its own: of Gemma 4 E2B's 35 layers, " "the last 20 do not compute their own keys and values but share those of two earlier layers (the last sliding window layer " "and the last full attention layer), and the drafter reads the same ones. It loads the first time MTP is switched on, and how " "many tokens it guesses at a time is in its generation config (`num_assistant_tokens` = 6)."}, - Code("harness/backends/simple_backend.py", "def load_drafter(self):"), + Code("agent/model_backends/simple_backend.py", "def load_drafter(self):"), {"zh": "**猜**：輸入是「上一個 token 的 embedding」接上「主模型在上一個位置最後一層的 hidden state」，輸出下一個 token 的 logits，" "取機率最高的當作猜測。接著把猜到的 token 和草稿模型自己的 hidden state 當作下一次的輸入，再猜一個，最多 6 個。" "6 次都站在同一個位置，因為它看到的 key / value 只到主模型真正讀過的地方。", @@ -1295,7 +1441,7 @@ def build(out: Path) -> list[Path]: "is the logits for the next token, and the most likely one is the guess. The guessed token and the drafter's own hidden state " "are then the input for another guess, up to 6. All 6 sit at the same position, because the keys and values it sees go only " "as far as the model has really read."}, - Code("harness/backends/simple_backend.py", "def _draft("), + Code("agent/model_backends/simple_backend.py", "def _draft("), {"zh": "**驗證**：主模型對「剛確定的 token + 6 個猜測」做一次 forward，`logits_to_keep=7` 拿回 7 個位置的分布：" "第 i 個位置的分布，就是讀到第 i 個猜測之前時，下一個 token 的分布。從第一個猜測開始依序交給 `_sample()` 檢查：" "收下就是一個不用另外 forward 的 token；第一個被退回的猜測，由主模型自己在那個位置選的 token 取代，後面的猜測全部作廢；" @@ -1305,7 +1451,7 @@ def build(out: Path) -> list[Path]: "from the first guess, `_sample()` checks them in order: a guess it keeps is a token without a forward pass of its own; the " "first guess it turns down is replaced by the model's own pick at that position, and the guesses after it are dropped; when " "all 6 hold, the last position's distribution gives one more token. So one forward pass produces between 1 and 7 tokens."}, - Code("harness/backends/simple_backend.py", "def _drafted("), + Code("agent/model_backends/simple_backend.py", "def _drafted("), {"zh": "作廢的猜測已經寫進了 KV cache，要用 `past.crop()` 拿掉；滑動視窗層平常會馬上丟掉太舊的 key / value，" "`activate_past_recording()` 讓它先留著，等 `crop()` 之後再縮回去。給草稿模型的那份 key / value 也同樣切掉作廢的部分。", "en": "The dropped guesses are already in the KV cache, and `past.crop()` takes them out; sliding window layers normally drop keys and " @@ -1320,7 +1466,7 @@ def build(out: Path) -> list[Path]: "so when two tokens are almost equally likely it may now and then pick the other). When sampling, a guess is kept with its " "probability p after temperature, top-k and top-p; if it is not kept, it is taken out of the candidates and the token is drawn " "from the rest. Every token still comes out with probability p: the drafter decides how many forward passes it takes, not what gets written."}, - Code("harness/backends/simple_backend.py", "at = (cand_ids == draft)", end="pick = int(torch.multinomial(weights, 1))"), + Code("agent/model_backends/simple_backend.py", "at: torch.Tensor | list = (cand_ids == draft)", end="pick = int(torch.multinomial(weights, 1))"), {"zh": "畫面上：", "en": "On the page:"}, ( {"zh": "step 卡片裡被收下的猜測有底線；一串底線加上後面那個沒有底線的 token，就是主模型同一次 forward 的產出。", @@ -1341,12 +1487,12 @@ def build(out: Path) -> list[Path]: {"zh": "推測解碼是伺服器啟動時決定的（`--speculative-config`），不是每個請求各自決定：", "en": "Speculative decoding is decided when the server starts (`--speculative-config`), not request by request:"}, ( - {"zh": "vLLM (Docker) 在 GPU 上用 `families.VLLM_SPECULATIVE` 裡這個模型的設定（[[harness/families/__init__.py|VLLM_SPECULATIVE = {]]）：" + {"zh": "vLLM (Docker) 在 GPU 上用 `model_families.VLLM_SPECULATIVE` 裡這個模型的設定（[[agent/model_families/__init__.py|VLLM_SPECULATIVE = {]]）：" "Gemma 4 用它的草稿模型 `google/gemma-4-E2B-it-assistant`，Qwen3.5 用它自己 checkpoint 裡的 MTP 層（transformers 用不了）；預設開，切換開關就用另一種方式重新啟動伺服器" - "（[[harness/backends/vllm_docker.py|def speculative(model_id]]）。vLLM 0.32 的 CPU 版 image 兩個模型都不能推測解碼，所以在 CPU 上開關不能按。", - "en": "vLLM (Docker) on a GPU takes the model's setting in `families.VLLM_SPECULATIVE` ([[harness/families/__init__.py|VLLM_SPECULATIVE = {]]): " + "（[[agent/model_backends/vllm_docker.py|def speculative(model_id]]）。vLLM 0.32 的 CPU 版 image 兩個模型都不能推測解碼，所以在 CPU 上開關不能按。", + "en": "vLLM (Docker) on a GPU takes the model's setting in `model_families.VLLM_SPECULATIVE` ([[agent/model_families/__init__.py|VLLM_SPECULATIVE = {]]): " "Gemma 4 with its drafter, `google/gemma-4-E2B-it-assistant`, Qwen3.5 with the MTP layer in its own checkpoint, which transformers cannot use. It is on by default, and the switch " - "starts the server again the other way ([[harness/backends/vllm_docker.py|def speculative(model_id]]). vLLM 0.32's CPU image cannot decode " + "starts the server again the other way ([[agent/model_backends/vllm_docker.py|def speculative(model_id]]). vLLM 0.32's CPU image cannot decode " "either model speculatively, so on a CPU the switch is disabled."}, {"zh": "`VLLM_URL` 的伺服器由啟動它的人決定，側欄的 MTP 開關不能按。", "en": "A server at `VLLM_URL` decides as whoever started it did, and the sidebar's MTP switch is disabled."}, @@ -1363,7 +1509,7 @@ def build(out: Path) -> list[Path]: "can the page mark which were guesses; otherwise it shows the counts alone. When they do, only the guesses up to the stop token count as kept: vLLM " "counts them before it cuts the step at the stop token, so the guesses after it would count too. That is known only once the step ends, so the " "underlines appear then ([[app/render.py|def with_drafts(]]), and what a turned-down guess was, the server does not say."}, - Code("harness/backends/vllm_backend.py", "def drafted_tokens("), + Code("agent/model_backends/vllm_backend.py", "def drafted_tokens("), ), }), )), @@ -1374,32 +1520,32 @@ def build(out: Path) -> list[Path]: "en": "The harness and the page know only a `Backend`: mainly three methods, render, encode and generate, and the events it yields. The sections above tell " "transformers here and a vLLM server apart in their tabs; this one is about the interface they share, and where a vLLM server comes from."}, ( {"zh": "前面幾段的 prefill、解碼迴圈、抽樣和 MTP，transformers 頁籤裡是 `simple_backend.py` 的 `SimpleBackend`，vLLM 頁籤裡是 `vllm_backend.py` 的 `VLLMBackend`。" - "可是 harness 和畫面從來不直接找它們，只透過 `Backend` 用模型（[[harness/backends/backend.py|class Backend(Protocol):]]）：`render()` 把對話套進 chat template，`encode()` 把 prompt 變成模型讀的 token（媒體展開），" + "可是 harness 和畫面從來不直接找它們，只透過 `Backend` 用模型（[[agent/model_backends/backend.py|class Backend(Protocol):]]）：`render()` 把對話套進 chat template，`encode()` 把 prompt 變成模型讀的 token（媒體展開），" "`generate()` 依序送出 `prefill`、每個 token 一個 `token`、最後的 `done`；再加上畫面會讀的幾個屬性：family、模型讀得懂什麼、脈絡上限、模型分頁用的 `info`。" "誰有這些，誰就能跑模型。", "en": "The prefill, the decoding loop, the sampling and MTP in the sections above are `SimpleBackend`'s in `simple_backend.py` in their transformers tabs, and " "`VLLMBackend`'s in `vllm_backend.py` in their vLLM tabs. Yet the harness and the page never ask for either; they use the model only through a `Backend` " - "([[harness/backends/backend.py|class Backend(Protocol):]]): `render()` puts the conversation through " + "([[agent/model_backends/backend.py|class Backend(Protocol):]]): `render()` puts the conversation through " "the chat template, `encode()` turns the prompt into the tokens the model reads (media expanded), and `generate()` yields `prefill`, then a `token` " "for each token, then `done`; and the page reads a few attributes: the family, what the model reads, the context limit and the Model tab's `info`. " "Whatever has these can run the model."}, - Code("harness/backends/backend.py", "class Backend(Protocol):", + Code("agent/model_backends/backend.py", "class Backend(Protocol):", caption={"zh": "`Protocol` 只是一個型別：`SimpleBackend` 不必繼承它，有這些方法和屬性就算數。測試檢查 `SimpleBackend`、`VLLMBackend` 和測試用的 `FakeLLM` 都是 `Backend`。", "en": "A `Protocol` is only a type: `SimpleBackend` need not inherit from it, and having these methods and attributes is enough. " "The tests check that `SimpleBackend`, `VLLMBackend` and the tests' `FakeLLM` are each a `Backend`."}), {"zh": "畫面只在一個地方決定用哪一個：`load_llm()` 拿到伺服器的網址（`VLLM_URL`，或畫面在 Docker 裡啟動的伺服器）就用 vLLM，沒有就在這台電腦上跑" "（[[app/page.py|llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id)]]）。" - "之後 `current_llm()`、側欄和 `Run`（{{prepare}}）拿到的都只是一個 `Backend`；它放在 `st.cache_resource` 裡，換模型或換「用什麼跑模型」才換掉（{{submit}}）。", + "之後 `current_llm()`、側欄和 `SimpleHarness`（{{prepare}}）拿到的都只是一個 `Backend`；它放在 `st.cache_resource` 裡，換模型或換「用什麼跑模型」才換掉（{{submit}}）。", "en": "The page decides which one in a single place: given a server's URL (`VLLM_URL`, or that of the server the page started in Docker), `load_llm()` " "uses vLLM, and without one runs the model here ([[app/page.py|llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id)]]). " - "From then on `current_llm()`, the sidebar and `Run` ({{prepare}}) get just a `Backend`; it stays in `st.cache_resource` until the model, or what " + "From then on `current_llm()`, the sidebar and `SimpleHarness` ({{prepare}}) get just a `Backend`; it stays in `st.cache_resource` until the model, or what " "runs it, changes ({{submit}})."}, {"zh": "兩種 backend 對模型知道的不一樣多，差別全在 `token` 事件裡。`TOKEN_FIELDS` 每個 backend 都要送：位置、token 的 id 和 piece、它加上的文字、" "是不是特殊 token、正在寫思考、工具呼叫還是回覆。`TOKEN_DETAILS` 則是知道才送：機率、排名、取樣時的機率、剩下的候選、前 5 名，以及 MTP 草稿猜了什麼。", "en": "The two backends know different amounts about the model, and the difference is all in the `token` event. Every backend sends `TOKEN_FIELDS`: " "the place, the token's id and piece, the text it adds, whether it is special, and whether it is thinking, a tool call or the reply. `TOKEN_DETAILS` " "it sends only when it knows them: the probability, the rank, the sampling probability, the candidates left, the top 5, and what the MTP drafter guessed."}, - Code("harness/backends/backend.py", "TOKEN_FIELDS = (", end="TOKEN_DETAILS = ("), + Code("agent/model_backends/backend.py", "TOKEN_FIELDS = (", end="TOKEN_DETAILS = ("), {"zh": "畫面拿到什麼就畫什麼（[[app/render.py|def known(]]）：沒回報的數字整列不畫，沒有機率就沒有熱度底色，沒說哪些候選被剃掉，前 5 名就不會變灰" "（[[app/render.py|def candidates_html(]]）。畫面寧可少畫，也不拿別的數字湊。", "en": "The page draws what it gets ([[app/render.py|def known(]]): a number nobody reported is not a row, a token without a probability gets no heat, and when " @@ -1432,19 +1578,19 @@ def build(out: Path) -> list[Path]: Tabs({ "vllm-docker": ( {"zh": "沒設 `VLLM_URL` 時，側欄「用什麼跑模型」選 vLLM (Docker)，" - "`harness/backends/vllm_docker.py` 就照 `docker-compose/compose.yaml` 用 Docker Compose 啟動它（[[harness/backends/vllm_docker.py|def start(model_id]]）。" + "`agent/model_backends/vllm_docker.py` 就照 `docker-compose/compose.yaml` 用 Docker Compose 啟動它（[[agent/model_backends/vllm_docker.py|def start(model_id]]）。" "權重先下載到這台電腦的 `models/`，容器唯讀掛上、自己不下載，所以伺服器載入的正是 `REVISIONS` 固定的那個 commit（{{files}}）。" - "Docker 有 NVIDIA 的 runtime 時，再疊上 `compose.gpu.yaml` 改在 GPU 上跑（KV cache 和 CPU 上一樣是 2 GiB，不是 vLLM 預設的 GPU 記憶體的 92%），否則在 CPU 上跑（[[harness/backends/vllm_docker.py|def device()]]），" + "Docker 有 NVIDIA 的 runtime 時，再疊上 `compose.gpu.yaml` 改在 GPU 上跑（KV cache 和 CPU 上一樣是 2 GiB，不是 vLLM 預設的 GPU 記憶體的 92%），否則在 CPU 上跑（[[agent/model_backends/vllm_docker.py|def device()]]），" "Mac 也一樣，在 Docker 的 Linux 虛擬機裡；CPU 版的 image 由 `docker-compose/vllm-cpu/` 建，補上 vLLM 0.32.0 在 CPU 上缺的兩個修正。換到 vLLM 之前，畫面先放掉這台電腦上的模型（[[app/page.py|def release_model():]]）；" - "換回 transformers 或結束這個程式時，停掉容器（[[harness/backends/vllm_docker.py|def stop():]]、[[harness/backends/vllm_docker.py|def _stop_at_exit():]]）。", + "換回 transformers 或結束這個程式時，停掉容器（[[agent/model_backends/vllm_docker.py|def stop():]]、[[agent/model_backends/vllm_docker.py|def _stop_at_exit():]]）。", "en": "Without `VLLM_URL`, when the sidebar's What runs the model picks vLLM (Docker), " - "`harness/backends/vllm_docker.py` starts it with Docker Compose from `docker-compose/compose.yaml` ([[harness/backends/vllm_docker.py|def start(model_id]]). " + "`agent/model_backends/vllm_docker.py` starts it with Docker Compose from `docker-compose/compose.yaml` ([[agent/model_backends/vllm_docker.py|def start(model_id]]). " "The weights are downloaded into this machine's `models/` first, and the container mounts that read-only and downloads nothing itself, so the " "server loads exactly the commit `REVISIONS` pins ({{files}}). Where Docker has NVIDIA's runtime, `compose.gpu.yaml` goes on top and the server runs on the GPU, with a 2 GiB KV cache as on the CPU rather than vLLM's 92% of the GPU's memory; " - "elsewhere it runs on the CPU ([[harness/backends/vllm_docker.py|def device()]]), a Mac's included, in Docker's Linux VM, from an image " + "elsewhere it runs on the CPU ([[agent/model_backends/vllm_docker.py|def device()]]), a Mac's included, in Docker's Linux VM, from an image " "`docker-compose/vllm-cpu/` builds with two fixes vLLM 0.32.0 lacks on a CPU. Before vLLM takes over, the page " "lets go of the model on this machine ([[app/page.py|def release_model():]]); picking transformers again, or quitting the app, stops the container " - "([[harness/backends/vllm_docker.py|def stop():]], [[harness/backends/vllm_docker.py|def _stop_at_exit():]])."}, + "([[agent/model_backends/vllm_docker.py|def stop():]], [[agent/model_backends/vllm_docker.py|def _stop_at_exit():]])."}, ), "vllm-url": ( {"zh": "任何一台 vLLM 伺服器，在這台電腦上或別台都可以，用 `vllm serve <模型> --revision <它在 REVISIONS 的 commit>` 啟動，再用 `VLLM_URL` 告訴網頁它在哪裡。" @@ -1473,34 +1619,53 @@ def build(out: Path) -> list[Path]: )), Section("parse", "harness", {"zh": "Harness：把輸出拆成思考、工具呼叫、回覆", "en": "Harness: splitting the output into thinking, tool calls and the reply"}, - {"zh": "模型只會吐文字；哪段是工具呼叫，要 harness 用 family 的 parser 讀出來。", - "en": "The model only writes text; which part is a tool call, the harness reads with the family's parser."}, ( - {"zh": "chat template 有時會在 prompt 結尾先打開思考（family 的 `thought_prefix`）；這段在 prompt 裡、不在 `raw` 裡，" - "所以先補回來，再交給 family 的 `parse`，`s.lang` 是它寫錯誤訊息用的語言。關掉「使用工具」時，就算模型寫了工具呼叫也當作沒有。", - "en": "The chat template sometimes opens the thinking at the end of the prompt (the family's `thought_prefix`); that part is in the " - "prompt, not in `raw`, so it is put back before the output goes to the family's `parse`, with `s.lang` as the language for its " - "error messages. With Use tools off, a tool call the model writes anyway counts as none."}, - Code("harness/agent.py", "prefix = family.thought_prefix if", end='self.emit("parsed"'), - {"zh": "兩個 parser 回傳一樣的結構：`thinking`、`tool_calls`（每個有 `name`、`arguments`、`error`）、`content`，所以後面的 harness 不用管是哪個模型。", - "en": "Both parsers return the same structure, `thinking`, `tool_calls` (each with `name`, `arguments` and `error`) and `content`, " + {"zh": "模型只會吐文字；哪段是工具呼叫，要有人讀出來：內建 harness 和模型端點用 family 的 parser，provider 的 API 則自己讀好再傳回來。", + "en": "The model only writes text; which part is a tool call, someone has to read out: the built-in harness and the model endpoint with the family's parser, a provider's API before it sends the reply back."}, ( + Tabs({ + "simple": ( + {"zh": "chat template 有時會在 prompt 結尾先打開思考（family 的 `thought_prefix`）；這段在 prompt 裡、不在 `raw` 裡，" + "所以先補回來，再交給 family 的 `parse`，`s.lang` 是它寫錯誤訊息用的語言。關掉「使用工具」時，就算模型寫了工具呼叫也當作沒有。", + "en": "The chat template sometimes opens the thinking at the end of the prompt (the family's `thought_prefix`); that part is in the " + "prompt, not in `raw`, so it is put back before the output goes to the family's `parse`, with `s.lang` as the language for its " + "error messages. With Use tools off, a tool call the model writes anyway counts as none."}, + Code("agent/harnesses/simple_harness.py", "prefix = family.thought_prefix if", end='self.emit("parsed"'), + ), + "pi-page": ( + {"zh": "端點用同一個 family 的 parser 讀模型的輸出，chat template 先打開的思考也一樣先補回來；讀得出來的呼叫以 `tool_calls` 傳回 pi，由 pi 去執行。" + "讀不出來的呼叫不送，pi 的這一輪就停在那裡：不像內建 harness（{{check}}），沒有人告訴模型哪裡寫錯。pi 收到的訊息結束時，`PiHarness` 把它讀到的思考、回覆和工具呼叫送成 `parsed`。", + "en": "The endpoint reads the model's output with the same family's parser, putting back first the thinking the chat template opened; the calls it can read " + "go back to pi as `tool_calls`, for pi to run. A call it cannot read is not sent, and pi's turn ends there: unlike the built-in harness ({{check}}), " + "nothing tells the model what it wrote wrong. Once the message pi got ends, `PiHarness` sends the thinking, the reply and the calls in it as `parsed`."}, + Code("agent/harnesses/model_endpoint.py", "prefix = family.thought_prefix if", end="send(finish=finish)"), + ), + "pi-provider": ( + {"zh": "provider 的 API 把工具呼叫和文字分開傳回來，已經是 JSON；模型自己的格式留在 provider 那邊。pi 從結束的訊息裡讀出它們，`PiHarness` 連同思考和回覆送成 `parsed`。", + "en": "A provider's API sends the tool calls apart from the text, already as JSON; the model's own format stays at the provider. pi reads them out of the " + "finished message, and `PiHarness` sends them on as `parsed`, with the thinking and the reply."}, + Code("agent/harnesses/pi_harness.py", 'blocks = message.get("content") or []', end='self.emit("parsed"'), + ), + }), + {"zh": "內建 harness 和 pi 問這個網頁的模型時，讀輸出的都是 family 的 parser。兩個 parser 回傳一樣的結構：`thinking`、`tool_calls`（每個有 `name`、`arguments`、`error`）、`content`，所以後面的 harness 不用管是哪個模型。", + "en": "For the built-in harness and for pi on this page's model, the family's parser reads the output. Both parsers return the same structure, " + "`thinking`, `tool_calls` (each with `name`, `arguments` and `error`) and `content`, " "so the rest of the harness does not care which model it was."}, Tabs({ "gemma4": ( {"zh": "Gemma 4 的 parser 分三步：(1) 切出 `<|channel>thought` 到 `` 之間的思考；(2) 用正規表達式找出每一個 `<|tool_call>call:名稱{…}`" - "（[[harness/families/gemma4.py|CALL_RE = re.compile]]），參數交給小 parser；(3) 有開頭、沒有結尾的工具呼叫（例如被 token 上限截斷）記成格式錯誤。" + "（[[agent/model_families/gemma4.py|CALL_RE = re.compile]]），參數交給小 parser；(3) 有開頭、沒有結尾的工具呼叫（例如被 token 上限截斷）記成格式錯誤。" "剩下的文字去掉停止 token，就是給使用者的回覆。", "en": "Gemma 4's parser works in three steps: (1) cut out the thinking between `<|channel>thought` and ``; (2) find every " - "`<|tool_call>call:name{…}` with a regular expression ([[harness/families/gemma4.py|CALL_RE = re.compile]]) and hand the " + "`<|tool_call>call:name{…}` with a regular expression ([[agent/model_families/gemma4.py|CALL_RE = re.compile]]) and hand the " "arguments to a small parser; (3) record a tool call that opens but never closes (cut off at the token limit, say) as a format " "error. The text left over, without stop tokens, is the reply for the user."}, - Code("harness/families/gemma4.py", "def parse_output("), + Code("agent/model_families/gemma4.py", "def parse_output("), {"zh": "Gemma 的參數不是 JSON：key 不加引號，字串用 `<|\"|>` 這個特殊 token 包起來，例如 `{timezone:<|\"|>Asia/Taipei<|\"|>}`。" "`_Parser` 是遞迴下降 parser：看到 `<|\"|>` 就一路讀到下一個 `<|\"|>`，`{` 讀物件、`[` 讀陣列、`\"` 讀 JSON 字串，" "其他的當作數字、`true` / `false` 或 `null`，都不是就當成字串。", "en": "Gemma's arguments are not JSON: keys have no quotes, and strings are wrapped in the special token `<|\"|>`, as in " "`{timezone:<|\"|>Asia/Taipei<|\"|>}`. `_Parser` is a recursive descent parser: at `<|\"|>` it reads up to the next `<|\"|>`, " "at `{` an object, at `[` an array, at `\"` a JSON string, and anything else is a number, `true` / `false` or `null`, or else a string."}, - Code("harness/families/gemma4.py", "def value(self):"), + Code("agent/model_families/gemma4.py", "def value(self):"), ), "qwen3_5": ( {"zh": "Qwen3.5 的 parser：思考在 `` 之前；每個 `` 區塊裡找 `` 和一段段 ``。" @@ -1510,8 +1675,8 @@ def build(out: Path) -> list[Path]: "`` sections. Every value is plain text, so its type comes from the tool's schema: a string stays as it is, " "a number, array or object is read as JSON, and a boolean is whether it says `true` or `false`; text that does not convert " "stays text, and the schema check that follows reports the wrong type to the model."}, - Code("harness/families/qwen3_5.py", "def parse_output("), - Code("harness/families/qwen3_5.py", "def convert(value: str, kind"), + Code("agent/model_families/qwen3_5.py", "def parse_output("), + Code("agent/model_families/qwen3_5.py", "def convert(value: str, kind"), ), }), )), @@ -1519,48 +1684,85 @@ def build(out: Path) -> list[Path]: {"zh": "要不要停：由 harness 決定", "en": "Whether to stop: the harness decides"}, {"zh": "模型只是停止生成；這一輪要不要結束，是 harness 看完輸出才決定的。", "en": "The model only stops generating; whether the turn ends is up to the harness, once it has read the output."}, ( - {"zh": "解析完先檢查兩種馬上結束的情況：你按了停止；或這一步沒有任何工具呼叫。沒有工具呼叫，回覆就是最終答案，停止原因是 `no_tool_call`；" - "如果是被 token 上限截斷而沒有工具呼叫，停止原因記成 `max_new_tokens`。兩種情況都把回覆存成 assistant 訊息，下一輪會當成歷史。", - "en": "After parsing, it first checks the two cases that end the turn at once: you pressed stop, or this step has no tool call. Without " - "a tool call, the reply is the final answer and the stop reason is `no_tool_call`; if the generation was cut off at the token " - "limit without a tool call, the stop reason is `max_new_tokens`. Either way, the reply is kept as an assistant message, part of " - "the history for the next turn."}, - {"zh": "這裡看的是「解析出來有沒有工具呼叫」，不是「停在哪個 token」。Gemma 停在 `<|tool_response>` 就代表要用工具，" - "但 Qwen 不管回答完或要用工具都停在 `<|im_end|>`；用解析結果判斷，兩個家族才能共用同一段程式。", - "en": "What counts here is whether parsing found a tool call, not which token it stopped at. Gemma stopping at `<|tool_response>` does " - "mean it wants a tool, but Qwen stops at `<|im_end|>` both after an answer and for a tool; deciding by the parse lets both " - "families share this code."}, - Code("harness/agent.py", 'if done["finish"] == "user_stopped":', end="if not calls:"), - {"zh": "其他停止條件分散在迴圈各處：", "en": "The other ways to stop are spread around the loop:"}, - ( - {"zh": "for 迴圈跑完所有步數都沒 break → `max_steps`（{{prepare}}設定的預設值）。", - "en": "The for loop uses up its steps without a break → `max_steps` (the default set in {{prepare}})."}, - {"zh": "等待確認時被停止或逾時 → `user_stopped` 或 `approval_timeout`（[[harness/agent.py|stop_reason = \"approval_timeout\" if self._timed_out]]）。", - "en": "Stopped or timed out while waiting for a confirmation → `user_stopped` or `approval_timeout` " - "([[harness/agent.py|stop_reason = \"approval_timeout\" if self._timed_out]])."}, - {"zh": "連續 `MAX_FORMAT_ERRORS`（3）次寫出無法解析的工具呼叫 → `format_errors`。", - "en": "`MAX_FORMAT_ERRORS` (3) tool calls in a row that cannot be parsed → `format_errors`."}, - {"zh": "backend 或 harness 本身丟出任何例外 → `error`，錯誤訊息會顯示在畫面上，而不是讓 thread 默默死掉；backend 的錯誤（例如 vLLM 伺服器的）兩種語言都有，畫面顯示目前語言的那一個。" - "工具丟的例外不會結束這一輪，而是變成觀察（{{execute}}）。", - "en": "Any exception from the backend or the harness itself → `error`; the message shows on the page instead of the thread dying silently, and a backend's error " - "(a vLLM server's, say) comes in both languages, of which the page shows the one it is in. A tool's exception does not end the turn; it " - "becomes an observation ({{execute}})."}, - ), - Code("harness/agent.py", "if format_errors >= MAX_FORMAT_ERRORS:", end='self.emit("run_end"', - caption={"zh": "不管怎麼停，最後一定送出 `run_end`，帶著停止原因、最終回覆和整份訊息列表。", - "en": "However it stops, `run_end` always goes out last, with the stop reason, the final reply and the whole message list."}), + Tabs({ + "simple": ( + {"zh": "解析完先檢查兩種馬上結束的情況：你按了停止；或這一步沒有任何工具呼叫。沒有工具呼叫，回覆就是最終答案，停止原因是 `no_tool_call`；" + "如果是被 token 上限截斷而沒有工具呼叫，停止原因記成 `max_new_tokens`。兩種情況都把回覆存成 assistant 訊息，下一輪會當成歷史。", + "en": "After parsing, it first checks the two cases that end the turn at once: you pressed stop, or this step has no tool call. Without " + "a tool call, the reply is the final answer and the stop reason is `no_tool_call`; if the generation was cut off at the token " + "limit without a tool call, the stop reason is `max_new_tokens`. Either way, the reply is kept as an assistant message, part of " + "the history for the next turn."}, + {"zh": "這裡看的是「解析出來有沒有工具呼叫」，不是「停在哪個 token」。Gemma 停在 `<|tool_response>` 就代表要用工具，" + "但 Qwen 不管回答完或要用工具都停在 `<|im_end|>`；用解析結果判斷，兩個家族才能共用同一段程式。", + "en": "What counts here is whether parsing found a tool call, not which token it stopped at. Gemma stopping at `<|tool_response>` does " + "mean it wants a tool, but Qwen stops at `<|im_end|>` both after an answer and for a tool; deciding by the parse lets both " + "families share this code."}, + Code("agent/harnesses/simple_harness.py", 'if done["finish"] == "user_stopped":', end="if not calls:"), + {"zh": "其他停止條件分散在迴圈各處：", "en": "The other ways to stop are spread around the loop:"}, + ( + {"zh": "for 迴圈跑完所有步數都沒 break → `max_steps`（{{prepare}}設定的預設值）。", + "en": "The for loop uses up its steps without a break → `max_steps` (the default set in {{prepare}})."}, + {"zh": "等待確認時被停止或逾時 → `user_stopped` 或 `approval_timeout`（[[agent/harnesses/simple_harness.py|stop_reason = \"approval_timeout\" if self._timed_out]]）。", + "en": "Stopped or timed out while waiting for a confirmation → `user_stopped` or `approval_timeout` " + "([[agent/harnesses/simple_harness.py|stop_reason = \"approval_timeout\" if self._timed_out]])."}, + {"zh": "連續 `MAX_FORMAT_ERRORS`（3）次寫出無法解析的工具呼叫 → `format_errors`。", + "en": "`MAX_FORMAT_ERRORS` (3) tool calls in a row that cannot be parsed → `format_errors`."}, + {"zh": "backend 或 harness 本身丟出任何例外 → `error`，錯誤訊息會顯示在畫面上，而不是讓 thread 默默死掉；backend 的錯誤（例如 vLLM 伺服器的）兩種語言都有，畫面顯示目前語言的那一個。" + "工具丟的例外不會結束這一輪，而是變成觀察（{{execute}}）。", + "en": "Any exception from the backend or the harness itself → `error`; the message shows on the page instead of the thread dying silently, and a backend's error " + "(a vLLM server's, say) comes in both languages, of which the page shows the one it is in. A tool's exception does not end the turn; it " + "becomes an observation ({{execute}})."}, + ), + Code("agent/harnesses/simple_harness.py", "if format_errors >= MAX_FORMAT_ERRORS:", end='self.emit("run_end"', + caption={"zh": "不管怎麼停，最後一定送出 `run_end`，帶著停止原因、最終回覆和整份訊息列表。", + "en": "However it stops, `run_end` always goes out last, with the stop reason, the final reply and the whole message list."}), + ), + "pi": ( + {"zh": "迴圈什麼時候結束由 pi 決定：一則沒有工具呼叫的回覆就結束它，pi 再用 `agent_settled` 說它沒有別的事要做了。`PiHarness` 只守著頁面的上限：" + "pi 自己沒有步數上限，所以 pi 開始超過「步數上限」的那個 turn 時，`PiHarness` 請 pi 中止，不再畫下一步；按停止或確認逾時也會中止它。", + "en": "pi decides when its loop ends: a reply without a tool call ends it, and pi then says with `agent_settled` that it has nothing more to do. `PiHarness` " + "only keeps the page's limits: pi has no step limit of its own, so when pi starts the turn past Max steps, `PiHarness` asks pi to abort and draws no " + "further step; pressing stop or an unanswered confirmation aborts it too."}, + Code("agent/harnesses/pi_harness.py", "def _turn_start(self):"), + {"zh": "停止原因照為什麼停來定：被中止就是中止的原因，pi 回報請求失敗就是 `error`（pi 的自動重試關掉了，失敗不會變成另一個 step），回覆被 token 上限截斷就是 `max_new_tokens`，其他都是 `no_tool_call`。", + "en": "The stop reason follows why it stopped: an abort's reason, `error` when pi reports a failed request (pi's retries are off, so a failure does not become " + "another step), `max_new_tokens` for a reply cut off at the token limit, and `no_tool_call` otherwise."}, + Code("agent/harnesses/pi_harness.py", "def _stop_reason(self) -> str:"), + ), + }), )), Section("record", "harness", {"zh": "把工具呼叫寫回對話", "en": "Writing the tool call back into the conversation"}, {"zh": "工具呼叫先變成一則 assistant 訊息，下一步由 chat template 寫回模型自己的格式。", "en": "A tool call first becomes an assistant message, which the chat template writes back in the model's own format in the next step."}, ( - {"zh": "有工具呼叫時，harness 給每個呼叫一個 id（`call_{step}_{i}`），寫成一則 assistant 訊息，" - "格式和 OpenAI / Hugging Face 的 `tool_calls` 一樣；下一步 chat template 會把它寫回 Gemma 的 `<|tool_call>…` 或 Qwen 的 `…`。", - "en": "With tool calls, the harness gives each one an id (`call_{step}_{i}`) and writes them into an assistant message in the format " - "of OpenAI's and Hugging Face's `tool_calls`; in the next step the chat template writes it back as Gemma's " - "`<|tool_call>…` or Qwen's `…`."}, - {"zh": "兩個家族在這裡有兩點不同，都由 family 決定：模型在工具呼叫前寫的文字要不要留（`text_with_calls`），以及思考內容存在哪個欄位，模板才讀得到（`reasoning_key`）。", - "en": "The two families differ in two ways here, both decided by the family: whether the text a model writes before a tool call is kept " + Tabs({ + "simple": ( + {"zh": "有工具呼叫時，harness 給每個呼叫一個 id（`call_{step}_{i}`），寫成一則 assistant 訊息，" + "格式和 OpenAI / Hugging Face 的 `tool_calls` 一樣；下一步 chat template 會把它寫回 Gemma 的 `<|tool_call>…` 或 Qwen 的 `…`。", + "en": "With tool calls, the harness gives each one an id (`call_{step}_{i}`) and writes them into an assistant message in the format " + "of OpenAI's and Hugging Face's `tool_calls`; in the next step the chat template writes it back as Gemma's " + "`<|tool_call>…` or Qwen's `…`."}, + {"zh": "接著先 render 一次「加上這則 assistant 訊息、但還沒有工具結果」的字串，存成 `prev_sequence`；`generation_prompt=False`，" + "因為這時還輪不到模型寫。下一步的 prompt 和它比，多出來的就是工具結果。直接拿「上一步 prompt + 模型輸出」來比會不準，" + "因為模板重新 render 時會改寫模型那一段（排序參數、換掉思考的標記）。", + "en": "Then it renders the conversation with this assistant message but without the tool results yet, and keeps that as " + "`prev_sequence`; `generation_prompt=False`, since it is not the model's turn to write yet. Compared with it, what the next " + "step's prompt has in addition is the tool result. Comparing with the last prompt plus the model's output would be off, " + "because the template rewrites the model's part when it renders it again (it sorts the arguments and replaces the thinking markup)."}, + Code("agent/harnesses/simple_harness.py", 'ids = [f"call_{step}_{i}"', end="prev_sequence = self.llm.render("), + ), + "pi": ( + {"zh": "pi 自己把這一步寫進它的 session：模型那則訊息（連同工具呼叫），接著每個工具結果，存在這段對話資料夾裡的 session 檔，下一個請求就帶著這一切。" + "pi 問這個網頁的模型時，端點把請求換回對話（{{context}}），chat template 再把工具呼叫寫回模型自己的格式，和下面說的一樣；" + "問 pi 自己的模型時，請求裡的工具呼叫是 JSON，provider 怎麼寫給模型就看不到了。", + "en": "pi writes the step into its session itself: the model's message, tool calls included, then each tool result, in the session file in the " + "conversation's folder, and its next request carries all of it. On this page's model, the endpoint turns the request back into the conversation " + "({{context}}), and the chat template writes the calls back in the model's own format, as below; on pi's own models, the calls in the request " + "are JSON, and how the provider writes them for its model is not to be seen."}, + ), + }), + {"zh": "chat template 把工具呼叫寫回模型自己的格式時（內建 harness 和端點都一樣），兩個家族在這裡有兩點不同，都由 family 決定：模型在工具呼叫前寫的文字要不要留（`text_with_calls`），以及思考內容存在哪個欄位，模板才讀得到（`reasoning_key`）。", + "en": "Where the chat template writes the calls back in the model's own format (for the built-in harness and for the endpoint alike), the two families differ in two ways, both decided by the family: whether the text a model writes before a tool call is kept " "(`text_with_calls`), and the field the thinking goes in, for the template to read it from (`reasoning_key`)."}, Tabs({ "gemma4": ( @@ -1573,47 +1775,59 @@ def build(out: Path) -> list[Path]: "en": "Qwen's template keeps the text before a tool call, and so does the harness; the thinking goes in `reasoning_content`."}, ), }), - {"zh": "接著先 render 一次「加上這則 assistant 訊息、但還沒有工具結果」的字串，存成 `prev_sequence`；`generation_prompt=False`，" - "因為這時還輪不到模型寫。下一步的 prompt 和它比，多出來的就是工具結果。直接拿「上一步 prompt + 模型輸出」來比會不準，" - "因為模板重新 render 時會改寫模型那一段（排序參數、換掉思考的標記）。", - "en": "Then it renders the conversation with this assistant message but without the tool results yet, and keeps that as " - "`prev_sequence`; `generation_prompt=False`, since it is not the model's turn to write yet. Compared with it, what the next " - "step's prompt has in addition is the tool result. Comparing with the last prompt plus the model's output would be off, " - "because the template rewrites the model's part when it renders it again (it sorts the arguments and replaces the thinking markup)."}, - Code("harness/agent.py", 'ids = [f"call_{step}_{i}"', end="prev_sequence = self.llm.render("), )), Section("check", "harness", {"zh": "驗證與權限：模型從不直接碰環境", "en": "Validation and permission: the model never touches the environment"}, {"zh": "每個呼叫要過三關：格式和 schema、重複檢查、（開著時）你的確認。", "en": "Each call has three checks to pass: format and schema, repeats, and (when it is on) your confirmation."}, ( - {"zh": "每個呼叫交給 `_handle_call()`。第一關是驗證：解析失敗、沒有這個工具（這一輪沒提供的也算，例如沒打開「讓模型寫程式」時的 `run_python`）、" - "參數不符合 schema，都不會執行。", - "en": "Each call goes to `_handle_call()`. The first check is validation: a call that failed to parse, names no tool offered this turn " - "(`run_python` without Let the model write code, say) or has arguments that do not fit the schema never runs."}, - Code("harness/agent.py", "def _handle_call(", end="args, errors = tool.validate(args, lang)"), - {"zh": "`Tool.validate()` 檢查必要參數有沒有給、有沒有多出不認得的參數、型別和 enum 對不對，然後補上預設值。" - "錯誤訊息是完整的句子，用這一輪開始時頁面的語言寫，因為它會原封不動變成模型下一步讀到的觀察。", - "en": "`Tool.validate()` checks that the required arguments are there, that there are no unknown ones, and that the types and enums " - "are right, then fills in the defaults. Each error is a full sentence in the language the page was in when the turn started, " - "because it becomes, word for word, the observation the model reads in the next step."}, - Code("harness/tools.py", "def validate(self, args: dict"), - {"zh": "第二關記錄重複：把「工具名稱 + 排序後的參數 JSON」當成 key，跟這一輪之前的呼叫比；一模一樣就在 step 卡片標出" - "「迴圈可能在打轉」。這只是提醒，不會擋。然後送出 `tool_call` 事件；有錯誤就直接把錯誤包成觀察回報，不進環境。", - "en": "The second check records repeats: the tool name plus the arguments as sorted JSON is the key, compared with the earlier calls " - "in this turn; an exact repeat is flagged in the step card as the loop maybe going in circles. It is only a warning and blocks " - "nothing. Then the `tool_call` event goes out; with errors, the errors go straight back as the observation, without reaching the environment."}, - Code("harness/agent.py", 'key = (call["name"], json.dumps', end='{"error": "; ".join(errors)}'), - {"zh": "第三關是你。開著「執行工具前先問我」時，harness 送出 `approval_request`，然後停在 `_wait_for_decision()` 等。" - "拒絕、停止、逾時都會變成一則錯誤觀察，模型下一步只看得到這段文字。", - "en": "The third check is you. With Ask me before running a tool on, the harness sends `approval_request` and waits in " - "`_wait_for_decision()`. A rejection, a stop or a timeout each becomes an error observation, and that text is all the model sees in the next step."}, - Code("harness/agent.py", "if self.settings.approve_tools:", end='return self._observe(step, cid, call["name"], {"error": error}'), - {"zh": "等待用的是 `threading.Condition`：每 0.5 秒醒來一次，直到你按下按鈕（`decide()` 會 `notify_all`）、按了停止，" - "或超過 `APPROVAL_TIMEOUT`（[[harness/agent.py|APPROVAL_TIMEOUT = ]]，5 分鐘）。逾時就當作拒絕並停止整輪，免得一個關掉的分頁把模型佔住。", - "en": "The wait uses a `threading.Condition`: it wakes every 0.5 s until you press a button (`decide()` calls `notify_all`), press stop, " - "or `APPROVAL_TIMEOUT` passes ([[harness/agent.py|APPROVAL_TIMEOUT = ]], 5 minutes). A timeout counts as a rejection and stops the " - "whole turn, so a closed tab cannot hold on to the model."}, - Code("harness/agent.py", "def _wait_for_decision("), + Tabs({ + "simple": ( + {"zh": "每個呼叫交給 `_handle_call()`。第一關是驗證：解析失敗、沒有這個工具（這一輪沒提供的也算，例如沒打開「讓模型寫程式」時的 `run_python`）、" + "參數不符合 schema，都不會執行。", + "en": "Each call goes to `_handle_call()`. The first check is validation: a call that failed to parse, names no tool offered this turn " + "(`run_python` without Let the model write code, say) or has arguments that do not fit the schema never runs."}, + Code("agent/harnesses/simple_harness.py", "def _handle_call(", end="args, errors = tool.validate(args, lang)"), + {"zh": "`Tool.validate()` 檢查必要參數有沒有給、有沒有多出不認得的參數、型別和 enum 對不對，然後補上預設值。" + "錯誤訊息是完整的句子，用這一輪開始時頁面的語言寫，因為它會原封不動變成模型下一步讀到的觀察。", + "en": "`Tool.validate()` checks that the required arguments are there, that there are no unknown ones, and that the types and enums " + "are right, then fills in the defaults. Each error is a full sentence in the language the page was in when the turn started, " + "because it becomes, word for word, the observation the model reads in the next step."}, + Code("agent/tools.py", "def validate(self, args: dict"), + {"zh": "第二關記錄重複：把「工具名稱 + 排序後的參數 JSON」當成 key，跟這一輪之前的呼叫比；一模一樣就在 step 卡片標出" + "「迴圈可能在打轉」。這只是提醒，不會擋。然後送出 `tool_call` 事件；有錯誤就直接把錯誤包成觀察回報，不進環境。", + "en": "The second check records repeats: the tool name plus the arguments as sorted JSON is the key, compared with the earlier calls " + "in this turn; an exact repeat is flagged in the step card as the loop maybe going in circles. It is only a warning and blocks " + "nothing. Then the `tool_call` event goes out; with errors, the errors go straight back as the observation, without reaching the environment."}, + Code("agent/harnesses/simple_harness.py", 'key = (call["name"], json.dumps', end='{"error": "; ".join(errors)}'), + {"zh": "第三關是你。開著「執行工具前先問我」時，harness 送出 `approval_request`，然後停在 `_wait_for_decision()` 等。" + "拒絕、停止、逾時都會變成一則錯誤觀察，模型下一步只看得到這段文字。", + "en": "The third check is you. With Ask me before running a tool on, the harness sends `approval_request` and waits in " + "`_wait_for_decision()`. A rejection, a stop or a timeout each becomes an error observation, and that text is all the model sees in the next step."}, + Code("agent/harnesses/simple_harness.py", "if self.settings.approve_tools:", end='return self._observe(step, cid, call["name"], {"error": error}'), + {"zh": "等待用的是 `threading.Condition`：每 0.5 秒醒來一次，直到你按下按鈕（`decide()` 會 `notify_all`）、按了停止，" + "或超過 `APPROVAL_TIMEOUT`（[[agent/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]]，5 分鐘）。逾時就當作拒絕並停止整輪，免得一個關掉的分頁把模型佔住。", + "en": "The wait uses a `threading.Condition`: it wakes every 0.5 s until you press a button (`decide()` calls `notify_all`), press stop, " + "or `APPROVAL_TIMEOUT` passes ([[agent/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]], 5 minutes). A timeout counts as a rejection and stops the " + "whole turn, so a closed tab cannot hold on to the model."}, + Code("agent/harnesses/simple_harness.py", "def _wait_for_decision("), + ), + "pi": ( + {"zh": "pi 執行工具前自己用工具的 schema 檢查參數；不符合的呼叫不會執行，錯誤就是模型讀到的工具結果。重複的呼叫，`PiHarness` 用和內建 harness 一樣的方式標出來" + "（[[agent/harnesses/pi_harness.py|def _tool_start(self]]）。", + "en": "pi checks a call's arguments against the tool's schema itself before running it; a call that does not fit does not run, and the error is the result " + "the model reads. `PiHarness` flags repeats as the built-in harness does ([[agent/harnesses/pi_harness.py|def _tool_start(self]])."}, + {"zh": "pi 執行工具前不會問人。打開「執行工具前先問我」時，網頁的 extension 在 pi 每次要執行工具前開一個確認對話框。" + "在 RPC 模式下，這個對話框是 stdout 上的一筆 `extension_ui_request`，`PiHarness` 把它變成 `approval_request`（[[agent/harnesses/pi_harness.py|def _dialog(self, rec: dict)]]），" + "你按的「允許」或「拒絕」再從 stdin 送回去（[[agent/harnesses/pi_harness.py|def _decided(self]]）；被拒絕的呼叫不執行，模型讀到的結果就是拒絕的原因，用這一輪開始時頁面的語言寫。" + "沒有人回答的話，和內建 harness 一樣，`APPROVAL_TIMEOUT` 之後停掉 pi。", + "en": "pi does not ask anyone before it runs a tool. With Ask me before running a tool on, the page's extension opens a confirm dialog every time pi is " + "about to run a tool. In RPC mode the dialog is an `extension_ui_request` on stdout, which `PiHarness` turns into an `approval_request` " + "([[agent/harnesses/pi_harness.py|def _dialog(self, rec: dict)]]), and your Allow or Reject goes back on stdin ([[agent/harnesses/pi_harness.py|def _decided(self]]); " + "a rejected call does not run, and the result the model reads is the refusal, in the language the page was in when the turn started. Unanswered, " + "the dialog stops pi after `APPROVAL_TIMEOUT`, as with the built-in harness."}, + Code("agent/harnesses/pi_extension.ts", 'pi.on("tool_call"', end="return undefined;"), + ), + }), {"zh": "畫面這邊，等待中的呼叫會在聊天區顯示「允許／拒絕」按鈕，按下去就呼叫 `run.run.decide()`；" "像 `run_python` 的程式碼這種好幾行的參數，另外用程式碼區塊顯示，看得清楚再決定。" "「先問我」寫在 harness 的程式裡，而不是寫在 prompt 裡請模型遵守，所以模型沒有辦法繞過。", @@ -1624,83 +1838,108 @@ def build(out: Path) -> list[Path]: )), Section("execute", "env", {"zh": "環境：真的去執行工具", "en": "Environment: really running the tool"}, - {"zh": "通過三關才會呼叫 Python 函式；錯誤也會變成觀察，不會讓迴圈當掉。", - "en": "Only after the three checks is the Python function called; errors become observations too, and never crash the loop."}, ( - {"zh": "送出 `tool_start` 後呼叫工具的 Python 函式；`takes_files` 的工具（`run_python`）另外拿到這段對話的資料夾。" - "工具自己丟的 `ToolError` 是可以預期的錯誤（時區打錯、找不到頁面、程式出錯），" - "訊息用這一輪的語言直接給模型看；其他例外也會被接住，變成 `{\"error\": …}`。", - "en": "After `tool_start` goes out, the tool's Python function is called; a tool that `takes_files` (`run_python`) also gets the " - "conversation's folder. A `ToolError` from the tool is an expected failure (a mistyped timezone, a page that does not exist, " - "code that fails), and its message goes to the model as it is, in the turn's language; any other exception is caught too and " - "becomes `{\"error\": …}`."}, - Code("harness/agent.py", 'self.emit("tool_start"', end="return self._observe(step, cid, call[\"name\"], result, ok=ok"), - {"zh": "五個工具各自連到不同的環境：", "en": "The five tools reach different environments:"}, - ( - {"zh": "`get_current_time`：讀系統時鐘，用 IANA 時區名稱換算（[[harness/tools.py|def get_current_time(]]）。", - "en": "`get_current_time`: reads the system clock and converts with an IANA timezone name ([[harness/tools.py|def get_current_time(]])."}, - {"zh": "`calculator`：在 AST 白名單上計算，從不呼叫 `eval()`。只允許數字、四則運算、次方和少數數學函式；" - "算次方前先估計位數，`round` 的位數參數也有上限（它在內部也會算次方），避免 `10**10**10` 這種算式把 thread 卡死。", - "en": "`calculator`: computes over an AST allowlist and never calls `eval()`. Only numbers, arithmetic, powers and a few math functions " - "are allowed; before a power it estimates the number of digits, and it caps the digits argument of `round`, which computes a power " - "inside, so an expression like `10**10**10` cannot hang the thread."}, - {"zh": "`search_wikipedia` / `read_wikipedia`：用 httpx 呼叫 Wikipedia API（[[harness/tools.py|def search_wikipedia(]]），" - "仿照 ReAct 論文在 HotpotQA 上給模型的 Wikipedia 動作（search、lookup、finish），這裡是 search 和 read 兩個。", - "en": "`search_wikipedia` / `read_wikipedia`: call the Wikipedia API with httpx ([[harness/tools.py|def search_wikipedia(]]), modeled " - "on the Wikipedia actions the ReAct paper gave its model on HotpotQA (search, lookup and finish), here as search and read."}, - {"zh": "`run_python`：在 Docker 容器裡跑模型寫的程式，把印出來的東西和畫出來的圖傳回來（[[harness/tools.py|def run_python(]]）。", - "en": "`run_python`: runs the code the model writes in a Docker container and returns what it prints and the charts it draws ([[harness/tools.py|def run_python(]])."}, - ), - Code("harness/tools.py", "def _eval(node):", - caption={"zh": "遞迴走過語法樹，遇到白名單以外的節點就丟 `ToolError`。", - "en": "Walks the syntax tree recursively and raises `ToolError` at any node outside the allowlist."}), - {"zh": "`calculator` 只算得了一行算式；`run_python` 則讓模型寫完整的程式，所以環境要關得更緊。每次呼叫都從 `harness/sandbox/Dockerfile` 建的 image" - "（Python 3.12 加上 numpy、pandas，讀 Excel 的 openpyxl、印表格的 tabulate，還有畫圖的 matplotlib 和一套中文字型）開一個新容器：沒有網路，檔案系統唯讀、只有 `/tmp` 能寫，記憶體 512 MB、一顆 CPU、最多 64 個 process，" - "拿掉所有 Linux capability，用你自己的使用者執行、絕不用 root（[[harness/tools.py|def sandbox_user(]]：在 Linux 上，掛進容器的資料夾保留原本的權限，換成別的使用者可能進不去）；" - "這段對話的資料夾唯讀掛在 `/data`。程式從 stdin 送進去，跑完容器就刪掉。", - "en": "`calculator` can only work out one expression; `run_python` lets the model write whole programs, so its environment is shut " - "tighter. Every call starts a new container from the image `harness/sandbox/Dockerfile` builds (Python 3.12 with numpy and pandas, openpyxl to read Excel, tabulate to print tables, and matplotlib to draw charts, with a font for Chinese labels): no " - "network, a read-only file system where only `/tmp` can be written, 512 MB of memory, one CPU, at most 64 processes, no Linux " - "capabilities, and your own user, never root ([[harness/tools.py|def sandbox_user(]]: on Linux a folder mounted in the container keeps its " - "permissions, which might not let another user in); the conversation's folder is mounted read-only at `/data`. The code goes in on stdin, " - "and the container is deleted when it is done."}, - Code("harness/tools.py", "def sandbox_command("), - {"zh": "容器寫不進任何頁面讀得到的地方，所以圖跟著印出來的東西從 stdout 出來：image 裡 matplotlib 的 backend（`harness/sandbox/agent_loop_figures.py`）" - "在 `plt.show()` 或程式結束時把每張圖轉成 PNG，程式結束前接在印出來的東西後面：先一行 `run_python` 用 `AGENT_LOOP_FIGURES` 傳進去的記號，再每行一張 base64 的 PNG。" - "`run_python` 把它們從模型要讀的輸出切掉，存進這段對話的資料夾，叫 `figure-1.png`、`figure-2.png`…（[[harness/tools.py|def save_figures(]]），" - "回傳一個 `ToolResult`：模型讀到的是 `{\"stdout\": …, \"figures\": [\"/data/figure-1.png\"]}`，圖本身只給頁面，畫在聊天區和這一步的卡片上。", - "en": "The container cannot write anywhere the page can read, so the charts come out on stdout with what the code prints: matplotlib's " - "backend in the image (`harness/sandbox/agent_loop_figures.py`) turns each figure into a PNG at `plt.show()` or when the program ends, and as it " - "exits writes them after the printed text: a line with the marker `run_python` passes in `AGENT_LOOP_FIGURES`, then one base64 PNG per line. " - "`run_python` cuts them off what the model reads, saves them in the conversation's folder as `figure-1.png`, `figure-2.png`… " - "([[harness/tools.py|def save_figures(]]) and returns a `ToolResult`: the model reads `{\"stdout\": …, \"figures\": [\"/data/figure-1.png\"]}`, " - "and the images themselves are for the page alone, drawn in the chat and on this step's card."}, - {"zh": "跑超過 30 秒就把容器停掉；結束代碼不是 0（例如丟出例外）、記憶體用太多或什麼都沒印也沒畫，都當成 `ToolError`，模型下一步讀到 traceback 或提醒，可以改了再跑。" - "Docker 不是必要的：只有側欄打開「讓模型寫程式」時才會提供 `run_python`，頁面也是那時候才去找 Docker（[[harness/tools.py|def sandbox_problem(]]），" - "沒裝、沒啟動、還沒建 image，或 image 比 `harness/sandbox/Dockerfile` 舊（看它的 `agent-loop-sandbox.version` label），就在側欄說明怎麼做，這一輪也不把 `run_python` 給模型。", - "en": "Code that runs past 30 seconds has its container stopped; a nonzero exit code (an exception, say), too much memory or neither output nor a chart is a " - "`ToolError`, so the model reads the traceback or a reminder in the next step and can fix the code and run it again. Docker is " - "not required: `run_python` is offered only when Let the model write code is on in the sidebar, and only then does the page look " - "for Docker ([[harness/tools.py|def sandbox_problem(]]); if it is not installed, not running, or the image is not built or is older than " - "`harness/sandbox/Dockerfile` (going by its `agent-loop-sandbox.version` label), the sidebar says " - "what to do, and the turn goes on without `run_python`."}, + {"zh": "通過檢查才會執行工具；錯誤也會變成觀察，不會讓迴圈當掉。", + "en": "Only after the checks does the tool run; its errors become observations too, and never crash the loop."}, ( + Tabs({ + "simple": ( + {"zh": "送出 `tool_start` 後呼叫工具的 Python 函式；`takes_files` 的工具（`run_python`）另外拿到這段對話的資料夾。" + "工具自己丟的 `ToolError` 是可以預期的錯誤（時區打錯、找不到頁面、程式出錯），" + "訊息用這一輪的語言直接給模型看；其他例外也會被接住，變成 `{\"error\": …}`。", + "en": "After `tool_start` goes out, the tool's Python function is called; a tool that `takes_files` (`run_python`) also gets the " + "conversation's folder. A `ToolError` from the tool is an expected failure (a mistyped timezone, a page that does not exist, " + "code that fails), and its message goes to the model as it is, in the turn's language; any other exception is caught too and " + "becomes `{\"error\": …}`."}, + Code("agent/harnesses/simple_harness.py", 'self.emit("tool_start"', end="return self._observe(step, cid, call[\"name\"], result, ok=ok"), + {"zh": "五個工具各自連到不同的環境：", "en": "The five tools reach different environments:"}, + ( + {"zh": "`get_current_time`：讀系統時鐘，用 IANA 時區名稱換算（[[agent/tools.py|def get_current_time(]]）。", + "en": "`get_current_time`: reads the system clock and converts with an IANA timezone name ([[agent/tools.py|def get_current_time(]])."}, + {"zh": "`calculator`：在 AST 白名單上計算，從不呼叫 `eval()`。只允許數字、四則運算、次方和少數數學函式；" + "算次方前先估計位數，`round` 的位數參數也有上限（它在內部也會算次方），避免 `10**10**10` 這種算式把 thread 卡死。", + "en": "`calculator`: computes over an AST allowlist and never calls `eval()`. Only numbers, arithmetic, powers and a few math functions " + "are allowed; before a power it estimates the number of digits, and it caps the digits argument of `round`, which computes a power " + "inside, so an expression like `10**10**10` cannot hang the thread."}, + {"zh": "`search_wikipedia` / `read_wikipedia`：用 httpx 呼叫 Wikipedia API（[[agent/tools.py|def search_wikipedia(]]），" + "仿照 ReAct 論文在 HotpotQA 上給模型的 Wikipedia 動作（search、lookup、finish），這裡是 search 和 read 兩個。", + "en": "`search_wikipedia` / `read_wikipedia`: call the Wikipedia API with httpx ([[agent/tools.py|def search_wikipedia(]]), modeled " + "on the Wikipedia actions the ReAct paper gave its model on HotpotQA (search, lookup and finish), here as search and read."}, + {"zh": "`run_python`：在 Docker 容器裡跑模型寫的程式，把印出來的東西和畫出來的圖傳回來（[[agent/tools.py|def run_python(]]）。", + "en": "`run_python`: runs the code the model writes in a Docker container and returns what it prints and the charts it draws ([[agent/tools.py|def run_python(]])."}, + ), + Code("agent/tools.py", "def _eval(node):", + caption={"zh": "遞迴走過語法樹，遇到白名單以外的節點就丟 `ToolError`。", + "en": "Walks the syntax tree recursively and raises `ToolError` at any node outside the allowlist."}), + {"zh": "`calculator` 只算得了一行算式；`run_python` 則讓模型寫完整的程式，所以環境要關得更緊。每次呼叫都從 `agent/sandbox/Dockerfile` 建的 image" + "（Python 3.12 加上 numpy、pandas，讀 Excel 的 openpyxl、印表格的 tabulate，還有畫圖的 matplotlib 和一套中文字型）開一個新容器：沒有網路，檔案系統唯讀、只有 `/tmp` 能寫，記憶體 512 MB、一顆 CPU、最多 64 個 process，" + "拿掉所有 Linux capability，用你自己的使用者執行、絕不用 root（[[agent/tools.py|def sandbox_user(]]：在 Linux 上，掛進容器的資料夾保留原本的權限，換成別的使用者可能進不去）；" + "這段對話的資料夾唯讀掛在 `/data`。程式從 stdin 送進去，跑完容器就刪掉。", + "en": "`calculator` can only work out one expression; `run_python` lets the model write whole programs, so its environment is shut " + "tighter. Every call starts a new container from the image `agent/sandbox/Dockerfile` builds (Python 3.12 with numpy and pandas, openpyxl to read Excel, tabulate to print tables, and matplotlib to draw charts, with a font for Chinese labels): no " + "network, a read-only file system where only `/tmp` can be written, 512 MB of memory, one CPU, at most 64 processes, no Linux " + "capabilities, and your own user, never root ([[agent/tools.py|def sandbox_user(]]: on Linux a folder mounted in the container keeps its " + "permissions, which might not let another user in); the conversation's folder is mounted read-only at `/data`. The code goes in on stdin, " + "and the container is deleted when it is done."}, + Code("agent/tools.py", "def sandbox_command("), + {"zh": "容器寫不進任何頁面讀得到的地方，所以圖跟著印出來的東西從 stdout 出來：image 裡 matplotlib 的 backend（`agent/sandbox/agent_loop_figures.py`）" + "在 `plt.show()` 或程式結束時把每張圖轉成 PNG，程式結束前接在印出來的東西後面：先一行 `run_python` 用 `AGENT_LOOP_FIGURES` 傳進去的記號，再每行一張 base64 的 PNG。" + "`run_python` 把它們從模型要讀的輸出切掉，存進這段對話的資料夾，叫 `figure-1.png`、`figure-2.png`…（[[agent/tools.py|def save_figures(]]），" + "回傳一個 `ToolResult`：模型讀到的是 `{\"stdout\": …, \"figures\": [\"/data/figure-1.png\"]}`，圖本身只給頁面，畫在聊天區和這一步的卡片上。", + "en": "The container cannot write anywhere the page can read, so the charts come out on stdout with what the code prints: matplotlib's " + "backend in the image (`agent/sandbox/agent_loop_figures.py`) turns each figure into a PNG at `plt.show()` or when the program ends, and as it " + "exits writes them after the printed text: a line with the marker `run_python` passes in `AGENT_LOOP_FIGURES`, then one base64 PNG per line. " + "`run_python` cuts them off what the model reads, saves them in the conversation's folder as `figure-1.png`, `figure-2.png`… " + "([[agent/tools.py|def save_figures(]]) and returns a `ToolResult`: the model reads `{\"stdout\": …, \"figures\": [\"/data/figure-1.png\"]}`, " + "and the images themselves are for the page alone, drawn in the chat and on this step's card."}, + {"zh": "跑超過 30 秒就把容器停掉；結束代碼不是 0（例如丟出例外）、記憶體用太多或什麼都沒印也沒畫，都當成 `ToolError`，模型下一步讀到 traceback 或提醒，可以改了再跑。" + "Docker 不是必要的：只有側欄打開「讓模型寫程式」時才會提供 `run_python`，頁面也是那時候才去找 Docker（[[agent/tools.py|def sandbox_problem(]]），" + "沒裝、沒啟動、還沒建 image，或 image 比 `agent/sandbox/Dockerfile` 舊（看它的 `agent-loop-sandbox.version` label），就在側欄說明怎麼做，這一輪也不把 `run_python` 給模型。", + "en": "Code that runs past 30 seconds has its container stopped; a nonzero exit code (an exception, say), too much memory or neither output nor a chart is a " + "`ToolError`, so the model reads the traceback or a reminder in the next step and can fix the code and run it again. Docker is " + "not required: `run_python` is offered only when Let the model write code is on in the sidebar, and only then does the page look " + "for Docker ([[agent/tools.py|def sandbox_problem(]]); if it is not installed, not running, or the image is not built or is older than " + "`agent/sandbox/Dockerfile` (going by its `agent-loop-sandbox.version` label), the sidebar says " + "what to do, and the turn goes on without `run_python`."}, + ), + "pi": ( + {"zh": "pi 執行它自己的工具：read、bash、edit、write、grep、find、ls，側欄選模型看得到哪幾個（預設 read、bash、edit、write，和 pi 一樣）。" + "它們在這台電腦上、這段對話的資料夾裡執行，那也是 pi 的工作資料夾，沒有沙箱：不像 `run_python` 在沒有網路、唯讀的 Docker 容器裡，bash 碰得到你碰得到的一切。" + "pi 開始執行一個工具時，`PiHarness` 送出 `tool_call` 和 `tool_start`（要先問你時，等你允許才送 `tool_start`）；工具結束時送出 `tool_result`，帶著 pi 的工具傳回的文字。", + "en": "pi runs its own tools: read, bash, edit, write, grep, find and ls, of which the sidebar picks the ones the model sees (read, bash, edit and write by " + "default, as pi offers). They run on this computer, in the conversation's folder, which is pi's working folder, without a sandbox: unlike `run_python` " + "in a Docker container with no network and read-only files, bash reaches whatever you can. When pi starts a tool, `PiHarness` sends `tool_call` and " + "`tool_start` (with asking first, `tool_start` only once you allow it); when the tool ends, `tool_result`, with the text pi's tool returned."}, + Code("agent/harnesses/pi_harness.py", "def _tool_start(self"), + ), + }), )), Section("observe", "context", {"zh": "觀察接回脈絡，進下一圈", "en": "The observation goes back into the context for the next step"}, {"zh": "工具結果變成一則 `tool` 訊息；下一圈 render 時，它就出現在模型的工具呼叫後面。", "en": "The tool result becomes a `tool` message; when the next step renders, it appears after the model's tool call."}, ( - {"zh": "工具結果（或錯誤）在 `_observe()` 裡轉成 JSON 字串（帶著圖的 `ToolResult` 先拆開：模型只讀它的資料，圖只放進事件給頁面），超過 `MAX_RESULT_CHARS`（2000 字元）就截斷並加上 `…(truncated)`，" - "免得一個長網頁塞爆脈絡；送出 `tool_result` 事件，再把字串回傳。", - "en": "In `_observe()` the tool result (or error) becomes a JSON string (a `ToolResult` with charts is split first: the model reads only " - "its data, and the images go only into the event, for the page), cut at `MAX_RESULT_CHARS` (2000 characters) with " - "`…(truncated)` added, so one long web page cannot flood the context; the `tool_result` event goes out, and the string is returned."}, - Code("harness/agent.py", "def _observe("), - {"zh": "回傳的字串變成一則 `role: tool` 的訊息 append 到 `self.messages`。同時更新連續格式錯誤的次數：這次是解析錯誤就加一，正常就歸零。", - "en": "The returned string is appended to `self.messages` as a `role: tool` message. The count of format errors in a row is updated at " - "the same time: one more for a parse error, back to zero otherwise."}, - Code("harness/agent.py", "for cid, call in zip(ids, calls):", end="if format_errors >= MAX_FORMAT_ERRORS:"), - {"zh": "然後回到 for 迴圈開頭，下一圈 render 時 chat template 把工具結果寫進脈絡，這就是 step 卡片上「上一步的觀察接回脈絡」那一段：", - "en": "Then it is back to the top of the for loop, and when the next step renders, the chat template writes the tool result into the " - "context; this is what the step card shows as the last step's observation, back in the context:"}, + Tabs({ + "simple": ( + {"zh": "工具結果（或錯誤）在 `_observe()` 裡轉成 JSON 字串（帶著圖的 `ToolResult` 先拆開：模型只讀它的資料，圖只放進事件給頁面），超過 `MAX_RESULT_CHARS`（2000 字元）就截斷並加上 `…(truncated)`，" + "免得一個長網頁塞爆脈絡；送出 `tool_result` 事件，再把字串回傳。", + "en": "In `_observe()` the tool result (or error) becomes a JSON string (a `ToolResult` with charts is split first: the model reads only " + "its data, and the images go only into the event, for the page), cut at `MAX_RESULT_CHARS` (2000 characters) with " + "`…(truncated)` added, so one long web page cannot flood the context; the `tool_result` event goes out, and the string is returned."}, + Code("agent/harnesses/simple_harness.py", "def _observe("), + {"zh": "回傳的字串變成一則 `role: tool` 的訊息 append 到 `self.messages`。同時更新連續格式錯誤的次數：這次是解析錯誤就加一，正常就歸零。", + "en": "The returned string is appended to `self.messages` as a `role: tool` message. The count of format errors in a row is updated at " + "the same time: one more for a parse error, back to zero otherwise."}, + Code("agent/harnesses/simple_harness.py", "for cid, call in zip(ids, calls):", end="if format_errors >= MAX_FORMAT_ERRORS:"), + ), + "pi": ( + {"zh": "pi 把工具的結果接在它的呼叫後面，開始下一個 turn；`PiHarness` 在工具結束時送出 `tool_result`，帶著 pi 的工具傳回的文字（[[agent/harnesses/pi_harness.py|def _tool_end(self]]）。" + "下一個請求就帶著它，step 的請求把它標成新加入的部分。pi 問這個網頁的模型時，端點把它換成一則帶著工具名稱的 `tool` 訊息，交給 chat template 寫進脈絡，和下面說的一樣。", + "en": "pi puts the tool's result after its call and starts the next turn; `PiHarness` sends `tool_result` when the tool ends, with the text pi's tool " + "returned ([[agent/harnesses/pi_harness.py|def _tool_end(self]]). The next request carries it, and the step's request marks it as new. On this page's " + "model, the endpoint makes it a `tool` message with its tool's name, which the chat template writes into the context, as below."}, + ), + }), + {"zh": "然後回到迴圈開頭，下一圈 render 時（內建 harness 和 pi 問這個網頁的模型時都一樣）chat template 把工具結果寫進脈絡，這就是 step 卡片上「上一步的觀察接回脈絡」那一段：", + "en": "Then it is back to the top of the loop, and when the next step renders (for the built-in harness and for pi on this page's model alike), the chat " + "template writes the tool result into the context; this is what the step card shows as the last step's observation, back in the context:"}, Tabs({ "gemma4": ( {"zh": "`<|tool_response>response:名稱{value:<|\"|>…<|\"|>}`，緊接在模型的工具呼叫後面，" @@ -1732,14 +1971,14 @@ def build(out: Path) -> list[Path]: Code("app/page.py", "@st.fragment(run_every=", end="viz_panel(llm)"), {"zh": "每次重畫，`LiveRun.data()` 複製一份目前的事件列表交給 `render.build()`，按順序折成一個個 step。" "`run_start` 告訴它這一輪是哪個模型、哪個 family，之後畫特殊 token、停止 token 和使用者訊息的位置都問這個 family；" - "`step_start` 開一個新的 step，`context`、`prefill`、`token`、`parsed` 填進去，`model_end` 帶來的推測解碼結果標出哪些 token 是猜中的" + "`step_start` 開一個新的 step，`context`、`prefill`、`token`、`parsed` 填進去（pi 跑的一輪還有 pi 送出的請求 `request`；pi 問它自己的模型時，沒有 family 也沒有前三個，換成串流進來的文字 `delta`），`model_end` 帶來的推測解碼結果標出哪些 token 是猜中的" "（伺服器到這時才說，{{mtp}}），工具相關的事件依 `call_id` 歸到各自的呼叫。" "之後所有 HTML 都是這個結構（和頁面語言）的純函式，所以執行到一半的任何時刻都畫得出來；" "切換語言也只是用另一種語言重畫同一份事件，跑完的對話也會跟著換。", "en": "On every redraw, `LiveRun.data()` copies the event list so far and hands it to `render.build()`, which folds it into steps in " "order. `run_start` says which model and family the turn uses, and from then on the special tokens, the stop tokens and where " "the user's message starts are all asked of that family; `step_start` opens a new step, `context`, `prefill`, `token` and " - "`parsed` fill it in, `model_end` marks the tokens that were guesses when a server says so only then ({{mtp}}), and the tool events go to their calls by `call_id`. All the HTML after that is a pure function of this " + "`parsed` fill it in (a turn pi ran also has the request pi sent, `request`, and when pi asks one of its own models, no family and none of the first three, but `delta`, the text as it streams), `model_end` marks the tokens that were guesses when a server says so only then ({{mtp}}), and the tool events go to their calls by `call_id`. All the HTML after that is a pure function of this " "structure (and of the page's language), so it can be drawn at any moment of a run; switching the language just redraws the " "same events in the other one, finished turns included."}, Code("app/render.py", "def build(events: list[dict])"), @@ -1751,62 +1990,85 @@ def build(out: Path) -> list[Path]: "en": "When a turn ends, `absorb_finished()` keeps the full message list from `run_end` as `history`. The next question is appended " "to it, so the agent remembers what was said before, earlier tool results included."}, Code("app/page.py", "def absorb_finished("), - {"zh": "停止按鈕呼叫 `Run.stop()`：設定 `_stop` 並叫醒正在等確認的 thread。解碼迴圈每個 token 前都會檢查 `_stop`，" - "所以按下去之後最多再生成一個 token 就停。", - "en": "The stop button calls `Run.stop()`: it sets `_stop` and wakes a thread that is waiting for a confirmation. The decoding loop " - "checks `_stop` before every token, so after the press at most one more token is generated."}, - Code("harness/agent.py", "def stop(self):", end="def decide(self, call_id"), + Tabs({ + "simple": ( + {"zh": "停止按鈕呼叫 `SimpleHarness.stop()`：設定 `_stop` 並叫醒正在等確認的 thread。解碼迴圈每個 token 前都會檢查 `_stop`，" + "所以按下去之後最多再生成一個 token 就停。", + "en": "The stop button calls `SimpleHarness.stop()`: it sets `_stop` and wakes a thread that is waiting for a confirmation. The decoding loop " + "checks `_stop` before every token, so after the press at most one more token is generated."}, + Code("agent/harnesses/simple_harness.py", "def stop(self):", end="def decide(self, call_id"), + ), + "pi": ( + {"zh": "停止按鈕呼叫 `PiHarness.stop()`，它把停止放進 `run()` 讀 pi 紀錄的那個佇列；`run()` 接著送給 pi `abort`，等著的確認一律當成被停止，" + "還在等自己那一步的請求也叫醒（[[agent/harnesses/pi_harness.py|def _abort(self, reason: str)]]）。pi 停下模型的請求和正在執行的工具；" + "pi 問這個網頁的模型時，端點在下一個 token 前就停，不屬於任何一步的請求則直接拒絕，不再 prefill。" + "pi 超過 `STOP_GRACE`（10 秒）還沒停下，就直接結束它。", + "en": "The stop button calls `PiHarness.stop()`, which puts the stop in the queue `run()` reads pi's records from; `run()` then sends pi `abort`, " + "takes every confirmation still waiting as stopped, and wakes a request still waiting for its step ([[agent/harnesses/pi_harness.py|def _abort(self, reason: str)]]). " + "pi stops the request to the model and the tool that runs; on this page's model, the endpoint stops before the next token, and turns away a request " + "that belongs to no step instead of prefilling it. A pi that has not settled after `STOP_GRACE` (10 s) is killed."}, + Code("agent/harnesses/pi_harness.py", "def _abort(self, reason: str):"), + ), + }), )), ] # (event, file, text on the line that sends it, what it carries, what the page does with it) EVENTS = [ - ("run_start", "harness/agent.py", '"run_start", run_id=', - {"zh": "模型和 family、這一輪的設定、可用的工具", "en": "the model and family, the turn's settings, the available tools"}, + ("run_start", "agent/harnesses/simple_harness.py", '"run_start", run_id=', + {"zh": "哪個 harness 跑的、模型和 family（pi 沒有）、這一輪的設定、可用的工具", "en": "which harness ran it, the model and family (none from pi), the turn's settings, the available tools"}, {"zh": "用哪個模型的格式畫這一輪；步數上限、解碼說明", "en": "whose format to draw the turn in; the step limit, the decoding notes"}), - ("step_start", "harness/agent.py", 'self.emit("step_start"', + ("step_start", "agent/harnesses/simple_harness.py", 'self.emit("step_start"', {"zh": "step 編號", "en": "the step number"}, {"zh": "開一張新的 step 卡片；迴圈圖亮起「脈絡」", "en": "opens a new step card; Context lights up in the loop diagram"}), - ("context", "harness/agent.py", '"context",', + ("request", "agent/harnesses/pi_harness.py", 'self.emit("request"', + {"zh": "pi 送給模型的請求，寫成 JSON（圖片只留類型）、這一步新加的部分在哪、各角色訊息數；只有 pi 送", + "en": "the request pi sends the model, as JSON (an image as its type alone), where this step's new part is, the number of messages per role; only from pi"}, + {"zh": "pi 問 provider 的模型時，step 卡片的脈絡段；「脈絡」分頁", "en": "the step card's context section when pi asks a provider's model; the Context tab"}), + ("context", "agent/harnesses/simple_harness.py", '"context",', {"zh": "模型讀到的完整 prompt（媒體已展開）、token 數、`added_from`、每種媒體佔的 token 數、脈絡上限、各角色訊息數", "en": "the whole prompt as the model reads it (media expanded), its token count, `added_from`, the tokens each kind of media takes, " "the context limit, the number of messages per role"}, {"zh": "step 卡片的脈絡段、「脈絡」分頁", "en": "the step card's context section, the Context tab"}), - ("prefill", "harness/backends/simple_backend.py", 'yield {"type": "prefill"', + ("prefill", "agent/model_backends/simple_backend.py", 'yield {"type": "prefill"', {"zh": "prompt 的 token 數、prefill 耗時", "en": "the prompt's token count, the prefill time"}, {"zh": "迴圈圖「模型」格、模型分頁的 prefill ms", "en": "the Model box in the loop diagram, prefill ms in the Model tab"}), - ("token", "harness/backends/simple_backend.py", '"type": "token",', + ("token", "agent/model_backends/simple_backend.py", '"type": "token",', {"zh": "id、piece、文字、channel、是否為特殊 token、機率、排名、前 5 名候選；開 MTP 時還有 `draft`（草稿猜了什麼、有沒有收下）。" "機率以後的欄位要 backend 知道才有：`simple_backend.py` 全都有，vLLM 伺服器有機率和前 5 名，其他看它加了哪些參數", "en": "id, piece, text, channel, whether it is special, probability, rank, the top 5 candidates; with MTP, also `draft` (what the drafter guessed and whether it was kept). " "The fields from the probability on are there only when the backend knows them: `simple_backend.py` knows them all, a vLLM server the probability and the top 5, and more as its flags allow"}, {"zh": "token 的熱度顏色、底線與浮窗、左邊逐字出現的回覆", "en": "the token's heat color, underline and popover; the reply appearing bit by bit on the left"}), - ("model_end", "harness/agent.py", 'self.emit("model_end"', + ("delta", "agent/harnesses/pi_harness.py", 'self.emit("delta"', + {"zh": "模型輸出的一段文字和它的 channel（思考、回覆、工具呼叫）；只有看不到 token 的 harness 送，也就是 pi", + "en": "a piece of the model's output and its channel (thinking, reply, tool call); only from a harness that sees no tokens, which is pi"}, + {"zh": "step 卡片模型段一段段出現的文字、左邊逐字出現的回覆", "en": "the text appearing in the step card's model section, the reply appearing bit by bit on the left"}), + ("model_end", "agent/harnesses/simple_harness.py", 'self.emit("model_end"', {"zh": "停止原因、停止 token、token 數、tok/s、MTP 統計（vLLM 伺服器還可能說哪些 token 是猜中的）、原始輸出", "en": "why it stopped, the stop token, the token count, tok/s, the MTP counts (from a vLLM server, perhaps which tokens were guesses), the raw output"}, {"zh": "「停在 …」、tok/s、MTP 收下幾個、模型分頁的表格", "en": "“stopped at …”, tok/s, how many MTP guesses were kept, the table in the Model tab"}), - ("parsed", "harness/agent.py", 'self.emit("parsed"', + ("parsed", "agent/harnesses/simple_harness.py", 'self.emit("parsed"', {"zh": "思考、回覆、工具呼叫", "en": "the thinking, the reply, the tool calls"}, {"zh": "Harness 段開始出現", "en": "the Harness section appears"}), - ("tool_call", "harness/agent.py", '"tool_call",', + ("tool_call", "agent/harnesses/simple_harness.py", '"tool_call",', {"zh": "名稱、驗證後的參數、錯誤、是否重複、環境", "en": "the name, the validated arguments, errors, whether it repeats, the environment"}, {"zh": "Harness 段的呼叫、驗證結果、重複警告", "en": "the call, the validation result and the repeat warning in the Harness section"}), - ("approval_request", "harness/agent.py", 'self.emit("approval_request"', + ("approval_request", "agent/harnesses/simple_harness.py", 'self.emit("approval_request"', {"zh": "等待確認的呼叫", "en": "the call waiting for a confirmation"}, {"zh": "聊天區的「允許／拒絕」按鈕；迴圈圖閃爍", "en": "the Allow and Reject buttons in the chat; the loop diagram pulses"}), - ("approval", "harness/agent.py", 'self.emit("approval",', + ("approval", "agent/harnesses/simple_harness.py", 'self.emit("approval",', {"zh": "允許、拒絕、逾時或被停止", "en": "allowed, rejected, timed out or stopped"}, {"zh": "Harness 段顯示你的決定", "en": "the Harness section shows your decision"}), - ("tool_start", "harness/agent.py", 'self.emit("tool_start"', + ("tool_start", "agent/harnesses/simple_harness.py", 'self.emit("tool_start"', {"zh": "工具名稱、環境", "en": "the tool name, the environment"}, {"zh": "迴圈圖亮起「環境」", "en": "Environment lights up in the loop diagram"}), - ("tool_result", "harness/agent.py", 'self.emit("tool_result"', + ("tool_result", "agent/harnesses/simple_harness.py", 'self.emit("tool_result"', {"zh": "結果字串、成功與否、是否截斷、耗時、給你看的圖", "en": "the result string, whether it succeeded, whether it was cut, the time it took, the images for you"}, {"zh": "環境段的結果、聊天區的工具標籤；圖畫在兩邊", "en": "the result in the Environment section, the tool chip in the chat; the charts in both"}), - ("error", "harness/agent.py", 'self.emit("error"', + ("error", "agent/harnesses/simple_harness.py", 'self.emit("error"', {"zh": "例外的類型和訊息；backend 的錯誤兩種語言都有（`text`）", "en": "the exception's type and message, and both languages of a backend's error (`text`)"}, {"zh": "聊天區和停止卡片的錯誤訊息", "en": "the error message in the chat and on the stop card"}), - ("run_end", "harness/agent.py", 'self.emit("run_end"', + ("run_end", "agent/harnesses/simple_harness.py", 'self.emit("run_end"', {"zh": "停止原因、最終回覆、整份訊息列表", "en": "the stop reason, the final reply, the whole message list"}, {"zh": "停止卡片；訊息列表成為下一輪的 `history`", "en": "the stop card; the message list becomes the next turn's `history`"}), ] diff --git a/app/page.py b/app/page.py index 2337edd..b70f683 100644 --- a/app/page.py +++ b/app/page.py @@ -3,28 +3,33 @@ Run with: uv run streamlit run streamlit_app.py (which calls main() here on every rerun) -This is the app's side of the project: the harness (harness/) runs the turn, and this page only starts it, draws +This is the app's side of the project: the agent (agent/) runs the turn, and this page only starts it, draws its events (render.py) and passes on the user's clicks. The settings read from the environment below are read once, when the module is first imported. -The agent runs in a background thread (harness.agent.Run) and appends one event per loop +The agent runs in a background thread (agent.harnesses.simple_harness.SimpleHarness) and appends one event per loop stage to a list. While a run is live, a fragment redraws the page from that list every half second. Clicking stop or approve reruns the script but not the thread, so the loop keeps going. -With VLLM_URL set, a vLLM server runs the model instead of this machine (harness/backends/vllm_backend.py), and the +The sidebar can also hand the loop to pi (agent/harnesses/pi_harness.py), a coding agent that runs its own, and the +page draws what pi says it does, with the request pi sends the model. pi asks either the page's own model, which +agent/harnesses/model_endpoint.py serves it on 127.0.0.1, so every token still shows, or a model it has credentials +for (Claude, GPT…), of which only the text shows, and then no model loads here. + +With VLLM_URL set, a vLLM server runs the model instead of this machine (agent/model_backends/vllm_backend.py), and the model menu lists what that server serves, marking those the page does not support; picking one of those says why and offers a GitHub issue asking for it, already filled in. Without it, the sidebar can still pick vLLM: the page then starts a -vLLM server in Docker from docker-compose/compose.yaml (harness/backends/vllm_docker.py). Either way, the model in +vLLM server in Docker from docker-compose/compose.yaml (agent/model_backends/vllm_docker.py). Either way, the model in memory is let go before the next one loads, so switching models or what runs them never holds two. The top bar also links to 運作說明 (How it works), a walk through the loop with the code behind it: a static page that app/guide.py builds and GitHub Pages serves. -The page is in Traditional Chinese or English (harness/i18n.py). It starts in the language last picked on the +The page is in Traditional Chinese or English (agent/i18n.py). It starts in the language last picked on the switch at the top right, kept in a cookie, or else in the browser's language. -A question can come with files (harness/media.py): images, sound and video the model reads, and, when the sidebar +A question can come with files (agent/media.py): images, sound and video the model reads, and, when the sidebar lets the model write code, data files for run_python. Only then does the page look for Docker, which run_python's sandbox needs, and say how to install it when it is missing. The charts run_python draws are shown in the chat and in the step that drew them. @@ -37,23 +42,27 @@ import threading import uuid import weakref +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field from pathlib import Path from urllib.parse import quote, urlencode import streamlit as st +from agent import media, model_families, tools +from agent.harnesses import model_endpoint, pi_harness +from agent.harnesses.harness import Harness +from agent.harnesses.simple_harness import RunSettings, SimpleHarness +from agent.i18n import LANGS, from_locale, pick +from agent.model_backends import vllm_backend, vllm_docker +from agent.model_backends.backend import Backend, Sampling +from agent.model_backends.simple_backend import SimpleBackend, empty_cache +from agent.tools import TOOLS from app import render -from harness import families, media, tools -from harness.agent import Run, RunSettings -from harness.backends import vllm_backend, vllm_docker -from harness.backends.backend import Backend, Sampling -from harness.backends.simple_backend import SimpleBackend, empty_cache -from harness.i18n import LANGS, from_locale, pick -from harness.tools import TOOLS # The model menu; MODEL_ID picks the one selected at first, and may name a model the menu does not list. -DEFAULT_MODEL = os.environ.get("MODEL_ID", next(iter(families.MODELS))) -MODELS = {**families.MODELS, DEFAULT_MODEL: families.MODELS.get(DEFAULT_MODEL, DEFAULT_MODEL)} +DEFAULT_MODEL = os.environ.get("MODEL_ID", next(iter(model_families.MODELS))) +MODELS = {**model_families.MODELS, DEFAULT_MODEL: model_families.MODELS.get(DEFAULT_MODEL, DEFAULT_MODEL)} # Where a vLLM server listens (http://localhost:8000, say); with it set, the server runs the model, not this machine. VLLM_URL = os.environ.get("VLLM_URL") ASSETS = Path(__file__).parent / "assets" @@ -108,7 +117,7 @@ ("Use Python to read this sales data: which month had the highest total revenue, and what was the revenue for the whole year?", "sales.csv"), ], } -EXAMPLE_ICONS = {"image": ":material/image:", "audio": ":material/graphic_eq:", "video": ":material/movie:", "file": ":material/table_chart:", "python": ":material/code:"} +EXAMPLE_ICONS: dict[str | None, str] = {"image": ":material/image:", "audio": ":material/graphic_eq:", "video": ":material/movie:", "file": ":material/table_chart:", "python": ":material/code:"} SAMPLE_FILES = sorted(p.name for p in SAMPLES.iterdir() if p.is_file() and not p.name.startswith(".")) @@ -126,7 +135,7 @@ def model_menu() -> dict[str, str]: return MODELS # A LoRA adapter (a card with a parent) is left out: its root is the adapter's folder on the server. served = (card.get("root") or card["id"] for card in vllm_backend.served_models(VLLM_URL) if not card.get("parent")) - return {model_id: ("⚠ " if supported(model_id) is False else "") + families.MODELS.get(model_id, model_id) for model_id in served} + return {model_id: ("⚠ " if supported(model_id) is False else "") + model_families.MODELS.get(model_id, model_id) for model_id in served} @st.cache_data(show_spinner=False) @@ -141,7 +150,7 @@ def served_family(model_id: str) -> str | None: def supported(model_id: str) -> bool | None: """Whether the page runs a model the menu lists; None when that cannot be told (no way to the Hub, or a folder only on the server, which then says why it does not load).""" - if model_id in families.MODELS: + if model_id in model_families.MODELS: return True try: return served_family(model_id) is not None @@ -237,6 +246,87 @@ def current_llm() -> Backend: return llm +# ---------- pi ---------- + +@dataclass +class PiAgent: + """pi and the model it asks, where the page would otherwise have a Backend: what the chat needs to know of it. The + model is the page's own (`llm`, served to pi by model_endpoint.py) or one of pi's own (`model`, at a provider).""" + model_id: str # the page's model id, or "provider/id", as pi's own model menu names it + version: str + llm: Backend | None = None + model: dict = field(default_factory=dict) # pi's model object for one of its own (pi_harness.probe) + + @property + def modalities(self) -> tuple[str, ...]: + """What pi sends the model besides text: images alone, and those only to a model that reads them.""" + reads = self.llm.modalities if self.llm else self.model.get("input", ()) + return ("image",) if "image" in reads else () + + @property + def info(self) -> dict: + return self.llm.info if self.llm else {} # a provider's model has no server or folder here + + +def pi_picked() -> bool: + return st.session_state.get("harness") == "pi" + + +@st.cache_data(ttl=60, show_spinner=False) +def pi_found() -> dict: + """What pi can run (pi_harness.probe), asked again at most every minute; a PiError is not kept, and is asked again.""" + return pi_harness.probe() + + +def retry_pi(): + pi_found.clear() + + +def pi_settings(who: PiAgent) -> pi_harness.PiSettings: + s = st.session_state + if who.llm: # Thinking mode, which pi sends as its thinking level and the endpoint turns into the template's switch + provider, model, thinking = model_endpoint.PROVIDER, who.llm.model_id, "medium" if s.thinking else "off" + else: + provider, model, thinking = who.model["provider"], who.model["id"], s.pi_thinking + return pi_harness.PiSettings(provider, model, thinking=thinking, tools=tuple(s.pi_tools), approve_tools=s.approve_tools, + max_steps=s.max_steps, lang=s.lang) + + +def pi_picker(busy: bool) -> PiAgent: + """Which model pi asks: the page's own, with the model menu and what runs it as the built-in harness has them, or one + of the models pi has credentials for. pi is started once to ask which (and again after a minute).""" + with st.spinner(tr("問 pi 它能用哪些模型…", "Asking pi which models it can use…")): + found = pi_found() + with st.sidebar: + sources = {"page": tr("這個網頁的模型", "This page's model"), "pi": tr("pi 自己的模型", "pi's own models")} + st.segmented_control( + tr("pi 問的模型", "The model pi asks"), list(sources), format_func=sources.__getitem__, key="pi_source", required=True, + on_change=new_chat, disabled=busy, + help=tr("這個網頁的模型：Gemma 或 Qwen，跑在 transformers 或 vLLM 上，由網頁在 127.0.0.1 上當成一個相容 OpenAI 的端點提供給 pi；" + "每個 token 和模型讀到的字串都看得到。pi 自己的模型：pi 登入過的 provider 的模型（Claude、GPT…），只看得到文字。", + "This page's model: Gemma or Qwen, on transformers or vLLM, which the page serves to pi as an OpenAI-compatible endpoint on 127.0.0.1; " + "every token and the string the model reads show. pi's own models: those of the providers pi is logged in to (Claude, GPT…); only their text shows."), + ) + if st.session_state.pi_source == "page": + model_picker(model_menu(), busy) + return PiAgent(st.session_state.model_id, found["version"], llm=current_llm()) + models = {f'{m["provider"]}/{m["id"]}': m for m in found["models"]} + if not models: + raise pi_harness.PiError( + "pi 還沒有可以用的模型。在終端機執行 `pi`，用 `/login` 登入一個 provider；或在啟動這個 app 之前設好 API key" + "（例如 `ANTHROPIC_API_KEY`、`OPENAI_API_KEY`），或在 pi 的 `models.json` 加上一個相容的端點。然後按「再試一次」。", + "pi has no model it can use yet. Run `pi` in a terminal and log in to a provider with `/login`, or set an API key " + "(such as `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`) before starting this app, or add a compatible endpoint to pi's `models.json`. Then press Try again.", + ) + if st.session_state.get("pi_model") not in models: + st.session_state.pi_model = found["default"] if found["default"] in models else next(iter(models)) + with st.sidebar: + st.selectbox(tr("模型", "Model"), list(models), key="pi_model", on_change=new_chat, disabled=busy, + help=tr("pi 有登入資訊的模型（provider/模型 id）。換模型會開始新的對話。", + "The models pi has credentials for (provider/model id). Switching models starts a new conversation.")) + return PiAgent(st.session_state.pi_model, found["version"], model=models[st.session_state.pi_model]) + + class Slot: """One model, one run at a time, across every browser tab. The run is kept here as well as in session state so another tab can see it and stop it, e.g. after the tab that started it was reloaded.""" @@ -252,15 +342,15 @@ def generation_slot() -> Slot: class LiveRun: - """One user turn: the harness thread and the events it has emitted so far.""" + """One user turn: the harness thread and the events it has emitted so far. `make` makes the harness (simple_harness.SimpleHarness + or pi_harness.PiHarness) given the function it sends its events to.""" - def __init__(self, llm: Backend, history: list[dict], text: str, attachments: list[media.Attachment], files: Path, - settings: RunSettings, slot: Slot): + def __init__(self, make: Callable[[Callable[[dict], None]], Harness], text: str, attachments: list[media.Attachment], slot: Slot): self.text = text self.attachments = attachments self.events: list[dict] = [] self.absorbed = False - self.run = Run(llm, history, text, settings, self.events.append, attachments, files) + self.run = make(self.events.append) self._slot = slot slot.run = self self.thread = threading.Thread(target=self._work, daemon=True) @@ -287,6 +377,15 @@ def data(self) -> dict: def init_state(): for key, value in DEFAULTS.items(): st.session_state.setdefault(key, value) + st.session_state.setdefault("harness", "simple") # what runs the loop: this project's harness, or pi + st.session_state.setdefault("pi_tools", list(pi_harness.DEFAULT_TOOLS)) + st.session_state.setdefault("pi_thinking", "off") + st.session_state.setdefault("pi_source", "page") # the model pi asks: the page's own, or one of pi's + # The sidebar draws only one harness's settings; Streamlit would forget the other's, whose widgets are not drawn, + # unless they are set again here. + for key in [*DEFAULTS, "model_id", "pi_model", "pi_tools", "pi_thinking", "pi_source"]: + if key in st.session_state: + st.session_state[key] = st.session_state[key] st.session_state.setdefault("runs", []) st.session_state.setdefault("history", []) st.session_state.setdefault("view", None) @@ -336,12 +435,15 @@ def sandbox_problem() -> dict | None: def code_allowed() -> bool: - """Whether the sidebar lets the model write code; Docker is looked for only then.""" + """Whether the sidebar lets the model write code: run_python, for which Docker is looked for only then, or with + pi its bash tool.""" + if pi_picked(): + return "bash" in st.session_state.pi_tools return st.session_state.use_tools and st.session_state.python def code_ready() -> bool: - return code_allowed() and sandbox_problem() is None + return code_allowed() and (pi_picked() or sandbox_problem() is None) def offered_tools() -> tuple[str, ...]: @@ -349,15 +451,15 @@ def offered_tools() -> tuple[str, ...]: return tuple(name for name in TOOLS if name != "run_python" or code_ready()) -def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: +def start_run(who: "Backend | PiAgent", text: str, uploads: Sequence[tuple[str, bytes]] = ()) -> bool: """Ask the question in a new run, with `uploads` (file name, contents) saved in the conversation's folder first.""" slot = generation_slot() if not slot.lock.acquire(blocking=False): st.toast(tr("另一個 agent 正在執行（可能在別的分頁），請稍等。", "Another agent is running (maybe in another tab). Please wait.")) return False try: - attachments = [media.save(chat_folder(), name, data, llm.modalities) for name, data in uploads] - run = LiveRun(llm, st.session_state.history, text, attachments, chat_folder(), current_settings(llm), slot) + attachments = [media.save(chat_folder(), name, data, who.modalities) for name, data in uploads] + run = LiveRun(harness_for(who, text, attachments), text, attachments, slot) except Exception: slot.lock.release() raise @@ -366,6 +468,17 @@ def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> return True +def harness_for(who: "Backend | PiAgent", text: str, attachments: list[media.Attachment]) -> Callable[[Callable[[dict], None]], Harness]: + """What runs the turn: pi, or this project's harness on the model in memory, with the sidebar's settings now.""" + history, folder = st.session_state.history, chat_folder() + if isinstance(who, PiAgent): + settings, session = pi_settings(who), st.session_state.chat_id + page = pi_harness.PageModel(who.llm, current_settings(who.llm).sampling) if who.llm else None + return lambda emit: pi_harness.PiHarness(settings, history, text, emit, attachments, folder, session, page) + run_settings = current_settings(who) + return lambda emit: SimpleHarness(who, history, text, run_settings, emit, attachments, folder) + + def other_tab_run() -> LiveRun | None: """A run that is still going but was started by another tab, or by this tab before a reload.""" run = generation_slot().run @@ -438,25 +551,43 @@ def show_run(run_index: int): # ---------- sidebar ---------- +def harness_picker(busy: bool): + """The very top of the sidebar: what runs the loop. It is drawn before anything can fail, so that the other can + always be picked.""" + with st.sidebar: + st.markdown(tr("### Agent Loop 可視化", "### Agent Loop Visualizer")) + names = {"simple": tr("內建 harness", "Built-in harness"), "pi": "pi"} + st.segmented_control( + tr("用什麼跑迴圈", "What runs the loop"), list(names), format_func=names.__getitem__, key="harness", required=True, + on_change=new_chat, disabled=busy, + help=tr("內建 harness：這個專案自己寫的迴圈（`agent/harnesses/simple_harness.py`），模型在這台電腦或 vLLM 上跑，每個 token 都看得到。" + "pi：把迴圈交給 pi 這個 coding agent，網頁畫出它每一步做了什麼，和它送給模型的請求。pi 可以問這個網頁的模型，每個 token 都看得到；" + "或問你在 pi 裡設定好的模型（Claude、GPT…），只看得到文字。pi 的工具在這台電腦上執行，沒有沙箱。換過去會開始新的對話。", + "Built-in harness: this project's own loop (`agent/harnesses/simple_harness.py`), with the model on this computer or on vLLM, and every token in view. " + "pi: hands the loop to pi, a coding agent, and the page draws what it does at each step, with the request it sends the model. " + "pi can ask this page's model, with every token in view, or a model you set up in pi (Claude, GPT…), of which only the text shows. " + "pi's tools run on this computer, without a sandbox. Switching starts a new conversation."), + ) + + def model_picker(menu: dict[str, str], busy: bool): """The top of the sidebar: the model, and, without VLLM_URL, what runs it. It is drawn before the model loads, so that while vLLM starts, or after it failed to, another model or transformers can still be picked.""" with st.sidebar: - st.markdown(tr("### Agent Loop 可視化", "### Agent Loop Visualizer")) about = tr("換模型會開始新的對話，解碼設定也換成新模型的建議值。harness、工具和畫面都不變。", "Switching models starts a new conversation and resets the decoding settings to what the new model recommends. " "The harness, the tools and the page stay the same.") if VLLM_URL: - names = [f.name for f in families.FAMILIES.values()] + names = [f.name for f in model_families.FAMILIES.values()] about += tr(f"\n\n標 ⚠ 的模型，這個網頁還不支援它的家族（目前支援 {'、'.join(names)}）；選了會說明原因，也可以直接到 GitHub 提 issue。", f"\n\nA model marked ⚠ is of a family this page does not support yet (it supports {', '.join(names)}); " "picking it says why, and offers a GitHub issue asking for it.") - st.selectbox(tr("模型", "Model"), list(menu), format_func=menu.get, key="model_id", on_change=switch_model, disabled=busy, help=about) + st.selectbox(tr("模型", "Model"), list(menu), format_func=menu.__getitem__, key="model_id", on_change=switch_model, disabled=busy, help=about) if not VLLM_URL: # The same in both languages: with full-width parentheses the two options no longer fit side by side. engines = {"local": "transformers", "vllm": "vLLM (Docker)"} st.segmented_control( - tr("用什麼跑模型", "What runs the model"), list(engines), format_func=engines.get, key="engine", required=True, + tr("用什麼跑模型", "What runs the model"), list(engines), format_func=engines.__getitem__, key="engine", required=True, on_change=switch_engine, disabled=busy, help=tr("transformers：模型載入這個程式，在這台電腦上跑，每個 token 的細節都看得到。" "vLLM：用專案的 docker-compose/compose.yaml 在 Docker 裡啟動 vLLM 伺服器，有 NVIDIA GPU 就用 GPU，否則用 CPU；" @@ -468,16 +599,21 @@ def model_picker(menu: dict[str, str], busy: bool): ) +def model_caption(llm: Backend) -> str: + """The model in memory and what runs it, under the model menu.""" + info = llm.info + if server := info.get("server"): + where = tr(f" · Docker（{st.session_state.vllm_device.upper()}）", f" · Docker ({st.session_state.vllm_device.upper()})") if docker_vllm() else "" + where += " · MTP" if docker_vllm() and st.session_state.mtp else "" + return f"`{llm.model_id}` · {llm.family.name} · {server['engine']} {server.get('version') or ''} · {server['url']}{where}" + return (f"`{llm.model_id}` · {llm.family.name} · {info['device'].upper()} · {info['dtype']} · " + + tr(f"載入 {info['load_seconds']} s", f"loaded in {info['load_seconds']} s")) + + def sidebar(llm: Backend, live: bool, busy: bool): - info, lang = llm.info, st.session_state.lang + lang = st.session_state.lang with st.sidebar: - if server := info.get("server"): - where = tr(f" · Docker（{st.session_state.vllm_device.upper()}）", f" · Docker ({st.session_state.vllm_device.upper()})") if docker_vllm() else "" - where += " · MTP" if docker_vllm() and st.session_state.mtp else "" - st.caption(f"`{llm.model_id}` · {llm.family.name} · {server['engine']} {server.get('version') or ''} · {server['url']}{where}") - else: - st.caption(f"`{llm.model_id}` · {llm.family.name} · {info['device'].upper()} · {info['dtype']} · " - + tr(f"載入 {info['load_seconds']} s", f"loaded in {info['load_seconds']} s")) + st.caption(model_caption(llm)) st.markdown(tr("##### 迴圈", "##### Loop")) st.toggle(tr("使用工具", "Use tools"), key="use_tools", help=tr("關掉就只是聊天機器人：模型看不到任何工具。", "Off, it is just a chatbot: the model sees no tools.")) @@ -492,7 +628,13 @@ def sidebar(llm: Backend, live: bool, busy: bool): st.toggle(tr("執行工具前先問我", "Ask me before running a tool"), key="approve_tools", help=tr("由 harness 擋下等你確認，而不是寫在 prompt 裡請模型遵守。", "The harness holds the call until you confirm it, instead of a line in the prompt asking the model to check first.")) + decoding_controls(llm, live, busy, tr("harness 的迴圈最多跑幾圈（step），跑滿就停。", "How many times (steps) the harness may go around the loop before it stops.")) + +def decoding_controls(llm: Backend, live: bool, busy: bool, steps_help: str): + """The decoding settings of the model in memory and the loop's limits, for either harness that runs on it.""" + info = llm.info + with st.sidebar: st.markdown(tr("##### 解碼與上限", "##### Decoding and limits")) if (info.get("server") or {}).get("refuses_greedy"): # vLLM refuses greedy decoding and an unlimited top-k once it was started with --return-sampling-mask, so the @@ -511,8 +653,7 @@ def sidebar(llm: Backend, live: bool, busy: bool): st.slider("top-p", 0.05, 1.0, step=0.05, key="top_p") st.slider("top-k", 0, 128, key="top_k", help=tr("0 = 不限。", "0 = no limit.")) st.toggle(tr("MTP 推測解碼", "MTP speculative decoding"), key="mtp", on_change=switch_mtp, disabled=busy or not can_mtp(llm), help=mtp_help(llm)) - st.slider(tr("步數上限", "Max steps"), 1, 100, key="max_steps", - help=tr("harness 的迴圈最多跑幾圈（step），跑滿就停。", "How many times (steps) the harness may go around the loop before it stops.")) + st.slider(tr("步數上限", "Max steps"), 1, 100, key="max_steps", help=steps_help) st.slider(tr("每步 token 上限", "Max tokens per step"), 64, 8192, step=64, key="max_new_tokens", help=tr("一次推論最多生成幾個 token。", "The most tokens one inference may generate.")) st.button(tr("還原成模型建議值", "Reset to the model's defaults"), on_click=reset_settings, args=(info["defaults"],), width="stretch") @@ -520,6 +661,48 @@ def sidebar(llm: Backend, live: bool, busy: bool): st.caption(tr("設定會套用到下一個問題。", "Settings apply to the next question.")) +PI_STEPS_HELP = {"zh": "pi 自己沒有步數上限；迴圈跑滿這麼多步，網頁就請 pi 中止。", + "en": "pi has no step limit of its own; once the loop has gone around this many times, the page asks pi to abort."} + + +def pi_sidebar(who: PiAgent, live: bool, busy: bool): + """pi's settings: its tools, how much its model thinks, asking first, and the step limit the page keeps for it; on + the page's own model, its decoding settings too.""" + lang = st.session_state.lang + with st.sidebar: + if who.llm: + st.caption(f"pi {who.version} · " + model_caption(who.llm)) + else: + st.caption(f"pi {who.version} · `{who.model_id}` · " + tr("在 provider 那邊跑", "runs at the provider")) + st.markdown(tr("##### 迴圈", "##### Loop")) + st.pills(tr("pi 的工具", "pi's tools"), list(pi_harness.TOOLS), selection_mode="multi", key="pi_tools", + help=tr("模型看得到哪些 pi 的工具。全部取消就只是聊天機器人。", "Which of pi's tools the model sees. With none, it is just a chatbot.")) + st.caption(tr("pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：bash 碰得到你碰得到的一切。", + "pi's tools run on this computer, in this conversation's folder, without a sandbox: bash reaches whatever you can.")) + if who.llm: + st.toggle(tr("思考模式", "Thinking mode"), key="thinking", + help=who.llm.family.thinking_help[lang] + tr("（pi 把它當成思考等級送出，模型端點再換成 chat template 的思考開關。）", + " (pi sends it as its thinking level, which the model endpoint turns into the chat template's thinking switch.)")) + else: + levels = pi_harness.THINKING if who.model.get("reasoning") else ("off",) + if st.session_state.pi_thinking not in levels: + st.session_state.pi_thinking = "off" + st.segmented_control(tr("思考", "Thinking"), list(levels), key="pi_thinking", required=True, + help=tr("pi 的思考等級：越高，模型回覆前想得越多。模型不會思考時只有 off。", + "pi's thinking level: the higher, the more the model thinks before it replies. A model that does not think has only off.")) + st.toggle(tr("執行工具前先問我", "Ask me before running a tool"), key="approve_tools", + help=tr("pi 自己不問；網頁讓 pi 載入的 extension（`pi_extension.ts`）在每次執行工具前都先等你確認。", + "pi does not ask by itself; the extension the page loads into pi (`pi_extension.ts`) waits for you before every tool call.")) + if who.llm: + decoding_controls(who.llm, live, busy, PI_STEPS_HELP[lang]) + return + with st.sidebar: + st.markdown(tr("##### 上限", "##### Limits")) + st.slider(tr("步數上限", "Max steps"), 1, 100, key="max_steps", help=PI_STEPS_HELP[lang]) + st.button(tr("新對話", "New chat"), on_click=new_chat, disabled=live, width="stretch") + st.caption(tr("設定會套用到下一個問題。", "Settings apply to the next question.")) + + def can_mtp(llm: Backend) -> bool: """Whether the MTP switch does anything: with transformers, for a model with a drafter; on vLLM in Docker, on a GPU and for a model vLLM drafts for. A server at VLLM_URL decides for itself.""" @@ -536,8 +719,8 @@ def mtp_help(llm: Backend) -> str: "vLLM 0.32's CPU image cannot decode either model speculatively: Qwen3.5 needs a Triton kernel without a CPU version, and Gemma 4's " "drafter sets up CUDA graphs. So vLLM (Docker) runs without MTP on a CPU; on an NVIDIA GPU it can.") if not (config := vllm_docker.speculative(llm.model_id, "gpu")): - return tr("vLLM 推測解碼這個模型的方式還沒有設定（`families.VLLM_SPECULATIVE`）。", - "How vLLM decodes this model speculatively is not set up (`families.VLLM_SPECULATIVE`).") + return tr("vLLM 推測解碼這個模型的方式還沒有設定（`model_families.VLLM_SPECULATIVE`）。", + "How vLLM decodes this model speculatively is not set up (`model_families.VLLM_SPECULATIVE`).") n, drafter = config["num_speculative_tokens"], config.get("model") how = (tr(f"用草稿模型 {drafter}", f"with the drafter {drafter}") if drafter else tr("用它自己 checkpoint 裡的 MTP 層（transformers 用不了）", "with the MTP layer in its own checkpoint, which transformers cannot use")) @@ -555,7 +738,7 @@ def mtp_help(llm: Backend) -> str: ) if not llm.drafter_id: on_vllm = (tr("在有 NVIDIA GPU 的 vLLM (Docker) 上，它可以推測解碼。", " On vLLM (Docker) with an NVIDIA GPU it can decode speculatively.") - if llm.model_id in families.VLLM_SPECULATIVE and not VLLM_URL else "") + if llm.model_id in model_families.VLLM_SPECULATIVE and not VLLM_URL else "") return tr("transformers 沒有這個模型搭配的 MTP 草稿模型，目前只有 Gemma 4 E2B 有。" + on_vllm, "transformers has no MTP drafter for this model; for now only Gemma 4 E2B has one." + on_vllm) return tr( @@ -569,7 +752,7 @@ def mtp_help(llm: Backend) -> str: # ---------- left: chat ---------- -def chat_panel(llm: Backend, live_run: LiveRun | None, other: LiveRun | None): +def chat_panel(llm: "Backend | PiAgent", live_run: LiveRun | None, other: LiveRun | None): runs = st.session_state.runs busy = live_run is not None or other is not None box = st.container(height=BOX_HEIGHT, border=False, key="chat_box") @@ -629,7 +812,7 @@ def chat_panel(llm: Backend, live_run: LiveRun | None, other: LiveRun | None): sample_picker(sample_row, samples, llm, busy) -def sample_picker(row, samples: list[str], llm: Backend, busy: bool): +def sample_picker(row, samples: list[str], llm: "Backend | PiAgent", busy: bool): """The sample files from assets/examples that this model can use, to ask a question of one's own about: a file picked here goes with the next question typed in the input box, as one attached with the + would.""" readable = llm.modalities # not the model: Streamlit keeps format_func after the run, and with it the model @@ -648,9 +831,10 @@ def uploaded(sent) -> list[tuple[str, bytes]]: return files + ([("recording.wav", audio.getvalue())] if (audio := sent.get("audio")) else []) -def examples(llm: Backend, busy: bool): +def examples(llm: "Backend | PiAgent", busy: bool): """The page language's examples that this model can try: one that attaches a file shows only if the model reads - that kind, and one that needs Python only once the model may write code, disabled until Docker is ready.""" + that kind, and one that needs Python only once the model may write code, disabled until Docker is ready (with pi, + once it has bash).""" lang, hidden = st.session_state.lang, False for i, (question, needs) in enumerate(EXAMPLES[lang]): sample = needs if needs not in (None, "python") else None @@ -660,12 +844,14 @@ def examples(llm: Backend, busy: bool): if kind in ("python", "file") and not code_allowed(): hidden = True continue - problem = sandbox_problem() if kind in ("python", "file") else None + problem = sandbox_problem() if kind in ("python", "file") and not pi_picked() else None note = problem[lang] if problem else tr(f"會附上範例檔案 {sample}", f"Attaches the sample file {sample}") if sample else None if st.button(question, key=f"example-{i}", icon=EXAMPLE_ICONS.get(kind), help=note, disabled=busy or problem is not None): if start_run(llm, question, [(sample, (SAMPLES / sample).read_bytes())] if sample else []): st.rerun() - if hidden: + if hidden and pi_picked(): + st.caption(tr("在側欄給 pi 它的 bash 工具，還有要寫 Python 的範例。", "Give pi its bash tool in the sidebar for examples that need Python.")) + elif hidden: st.caption(tr("打開側欄的「讓模型寫程式」，還有要寫 Python 的範例。", "Turn on Let the model write code in the sidebar for examples that need Python.")) @@ -675,8 +861,8 @@ def attachments_view(attachments: list[media.Attachment]): st.image(a.path, width=240) elif a.kind == "audio": st.audio(a.path, format=media.mime(a.name) or "audio/wav") - elif a.kind == "video" and media.mime(a.name): - st.video(a.path, format=media.mime(a.name)) + elif a.kind == "video" and (mime := media.mime(a.name)): + st.video(a.path, format=mime) elif a.kind == "video": # a video some browsers cannot play: the frames the model reads instead st.image(frame_grid(a.path), width=320, caption=tr("瀏覽器不一定播得了這種影片；這是模型讀到的其中幾張畫格", "Browsers may not play this kind of video; here are some of the frames the model reads")) @@ -730,7 +916,7 @@ def assistant_message(i: int, run: LiveRun): # ---------- right: the loop ---------- -def viz_panel(llm: Backend): +def viz_panel(llm: "Backend | PiAgent"): runs = st.session_state.runs index = st.session_state.view run = runs[index] if runs and index is not None else None @@ -739,7 +925,7 @@ def viz_panel(llm: Backend): # The switcher keeps its own width; in a narrow column the caption wraps under it. with st.container(horizontal=True, vertical_alignment="center"): names = {"loop": tr("迴圈", "Loop"), "context": tr("脈絡", "Context"), "model": tr("模型", "Model")} - view = st.segmented_control(tr("檢視", "View"), list(names), format_func=names.get, key="viz_tab", required=True, label_visibility="collapsed") + view = st.segmented_control(tr("檢視", "View"), list(names), format_func=names.__getitem__, key="viz_tab", required=True, label_visibility="collapsed") if run and len(runs) > 1: st.caption(tr(f"第 {index + 1} 輪：{run.text[:40]}", f"Turn {index + 1}: {run.text[:40]}")) @@ -748,16 +934,19 @@ def viz_panel(llm: Backend): if view == "context": context_panel(index, data) elif view == "model": - model_panel(llm, data) + if isinstance(llm, PiAgent): + pi_model_panel(llm, data) + else: + model_panel(llm, data) else: loop_panel(index, run, data, llm.model_id) def loop_panel(index: int | None, run: LiveRun | None, data: dict | None, model_id: str): - lang = st.session_state.lang - st.html(render.diagram_html(data, st.context.theme.type == "dark", lang, model=model_id)) - if data is None: - st.html(render.INTRO[lang]) + lang, harness = st.session_state.lang, st.session_state.harness + st.html(render.diagram_html(data, st.context.theme.type == "dark", lang, model=model_id, harness=harness)) + if data is None or run is None: + st.html(render.PI_INTRO[st.session_state.pi_source][lang] if harness == "pi" else render.INTRO[lang]) return st.caption(render.run_stats(data, lang)) # A fixed height makes it scroll; style.css shrinks it to the space left under the diagram. @@ -769,7 +958,7 @@ def loop_panel(index: int | None, run: LiveRun | None, data: dict | None, model_ st.html(render.step_html(step, data, lang, live=live_step)) for path in render.images([step]): st.image(path, width=320, caption=Path(path).name) - if step["context"]: + if step["context"] or step["request"]: st.button(tr("看這一步的完整脈絡 →", "See this step's full context →"), key=f"ctx-{index}-{step['n']}", type="tertiary", on_click=show_context, args=(index, step["n"])) if data["end"]: @@ -778,8 +967,13 @@ def loop_panel(index: int | None, run: LiveRun | None, data: dict | None, model_ def context_panel(index: int | None, data: dict | None): lang = st.session_state.lang - steps = [s for s in data["steps"] if s["context"]] if data else [] - if not steps: + steps = [s for s in data["steps"] if s["context"] or s["request"]] if data else [] + if not steps and pi_picked(): + st.info(tr("送出問題後，這裡會顯示 pi 每一步送給模型的請求；pi 問的是這個網頁的模型時，還有模型實際讀到的完整字串。", + "Once you send a question, this shows the request pi sends the model in each step, and, when pi asks this page's model, " + "the whole string the model read.")) + return + if not steps or data is None: st.info(tr("送出問題後，這裡會顯示每一步模型實際讀到的完整字串。", "Once you send a question, this shows the whole string the model read in each step.")) return key = f"ctx_step_{index}" @@ -789,8 +983,18 @@ def context_panel(index: int | None, data: dict | None): tr("步驟", "Step"), [s["n"] for s in steps], key=key, required=True, format_func=lambda n: f"Step {n}" ) step = next((s for s in steps if s["n"] == choice), steps[-1]) - st.caption(render.context_summary(step, lang)) - st.html(render.CONTEXT_LEGEND[lang] + render.context_html(step, data["family"], lang)) + if step["context"]: + st.caption(render.context_summary(step, lang)) + st.html(render.CONTEXT_LEGEND[lang] + render.context_html(step, data["family"], lang)) + if step["request"] and step["context"]: # pi asking the page's model: what pi sent, before the template wrote it out + with st.expander(tr("pi 送出的請求（JSON），chat template 把它寫成上面那個字串", "The request pi sent (JSON), which the chat template wrote out as the string above")): + st.caption(render.request_summary(step, lang)) + st.html(render.REQUEST_LEGEND[lang] + render.request_html(step)) + elif step["request"]: + st.caption(render.request_summary(step, lang)) + st.html(render.REQUEST_LEGEND[lang] + render.request_html(step)) + st.caption(tr("API 那邊怎麼把這個請求寫成模型讀的字串，pi 和 provider 都不說。", + "How the API writes this request out as the string the model reads, neither pi nor the provider says.")) def model_panel(llm: Backend, data: dict | None): @@ -809,6 +1013,27 @@ def model_panel(llm: Backend, data: dict | None): st.html(render.folder_html(llm.info, lang)) +def pi_model_panel(who: PiAgent, data: dict | None): + lang, s = st.session_state.lang, st.session_state + if who.llm: # the page's own model, as the built-in harness shows it, with what pi does with it + llm = who.llm + st.html(render.model_html(llm.info, lang)) + settings = data["settings"] if data else {**DEFAULTS, **{k: s[k] for k in DEFAULTS}, "tools": s.pi_tools, "page_model": True, + "thinking": "medium" if s.thinking else "off"} + st.html(render.knobs_html({**settings, "mtp": None} if VLLM_URL else settings, llm.info["defaults"], lang)) + else: + st.html(render.pi_model_html(who.model, who.version, lang)) + settings = data["settings"] if data else {"thinking": s.pi_thinking, "approve_tools": s.approve_tools, "max_steps": s.max_steps, "tools": s.pi_tools} + st.html(render.pi_settings_html(settings, lang)) + if data and data["steps"]: + st.markdown(tr("##### 每一步的推論", "##### Inference in each step") if who.llm else tr("##### 每一步的請求", "##### The request in each step")) + st.dataframe(render.step_rows(data, lang), hide_index=True, width="stretch") + st.html(render.pi_tools_html(pi_harness.TOOLS, settings["tools"], lang)) + if who.llm: + st.html(render.special_tokens_html(who.llm.info, lang)) + st.html(render.folder_html(who.llm.info, lang)) + + def issue_url(error: vllm_backend.UnsupportedModel) -> str: """A new GitHub issue asking the page to support a model, its title and description filled in in the page's language. It names the model and its model_type, never VLLM_URL, which may be a private address, and a checkpoint @@ -853,20 +1078,35 @@ def agent_page(): init_state() own_live = any(run.live for run in st.session_state.runs) polling = own_live or other_tab_run() is not None - model_picker(model_menu(), polling) - llm = current_llm() - except (vllm_backend.ServerError, vllm_docker.DockerError) as e: # the vLLM server is down, refused or did not start + harness_picker(polling) + if pi_picked(): + llm: Backend | PiAgent = pi_picker(polling) + else: + model_picker(model_menu(), polling) + llm = current_llm() + except (vllm_backend.ServerError, vllm_docker.DockerError, pi_harness.PiError) as e: # vLLM is down, refused or did not start, or pi unsupported = isinstance(e, vllm_backend.UnsupportedModel) st.error(e.text[st.session_state.lang], icon=":material/extension_off:" if unsupported else ":material/cloud_off:") if isinstance(e, vllm_docker.DockerError): # the sidebar still has the menu and the switch back to transformers st.button(tr("再試一次", "Try again"), icon=":material/refresh:", on_click=retry_vllm) - if unsupported: # and the menu has the models the page does support + if isinstance(e, pi_harness.PiError): # and the switch back to the built-in harness + st.button(tr("再試一次", "Try again"), icon=":material/refresh:", on_click=retry_pi) + if isinstance(e, vllm_backend.UnsupportedModel): # and the menu has the models the page does support ask_for_model(e) st.stop() - sidebar(llm, own_live, polling) - # Streamlit keeps the fragment after the run, to run it again on its own; held weakly, the model can still be let go - # (release_model()) the moment another is picked, rather than once the next run has replaced the fragment. - model = weakref.ref(llm) + model: Callable[[], Backend | PiAgent | None] # what the workspace draws, None once it has been let go + if isinstance(llm, PiAgent): + pi_sidebar(llm, own_live, polling) + if llm.llm: # the page's own model, held weakly as below + backend, model_id, version = weakref.ref(llm.llm), llm.model_id, llm.version + model = lambda: PiAgent(model_id, version, llm=b) if (b := backend()) else None + else: # pi's own model: nothing in memory here to let go + model = lambda: llm + else: + sidebar(llm, own_live, polling) + # Streamlit keeps the fragment after the run, to run it again on its own; held weakly, the model can still be let go + # (release_model()) the moment another is picked, rather than once the next run has replaced the fragment. + model = weakref.ref(llm) @st.fragment(run_every=POLL_SECONDS if polling else None) def workspace(): @@ -899,7 +1139,7 @@ def language_switch(): picked = st.context.cookies.get(LANG_COOKIE) st.session_state.lang = picked if picked in LANGS else from_locale(st.context.locale) with st.container(key="lang_switch"): - st.segmented_control("Language / 語言", list(LANGS), format_func=LANGS.get, key="lang", required=True, + st.segmented_control("Language / 語言", list(LANGS), format_func=LANGS.__getitem__, key="lang", required=True, label_visibility="collapsed", on_change=pick_lang) if st.session_state.get("lang_picked"): cookie = f"{LANG_COOKIE}={st.session_state.lang}; path=/; max-age=31536000; SameSite=Lax" # a year diff --git a/app/render.py b/app/render.py index 5b51223..2bd9afd 100644 --- a/app/render.py +++ b/app/render.py @@ -7,15 +7,19 @@ A token shows as much of the model's distribution as the backend reported (backend.TOKEN_DETAILS): simple_backend.SimpleBackend reports all of it, and what a backend leaves out is left out here too, never filled in. +So does a step: a harness that does not run the model itself (pi asking a provider's model, harnesses/pi_harness.py) +sends no context and no tokens, only the request it sent the model's API (request), the text the model streams (delta) +and the token counts its provider reported, and its steps are drawn from those. pi asking the page's own model sends +both: its request, and the context and tokens of the model endpoint (harnesses/model_endpoint.py). """ import base64 import html import json -import re +from collections.abc import Sequence -from harness import families -from harness.i18n import pick +from agent import model_families +from agent.i18n import pick STOP_REASONS = { "zh": { @@ -37,6 +41,21 @@ "error": ("error", "Something went wrong in the code; the message is below."), }, } +# Where pi runs the loop (harnesses/pi_harness.py), the page stops it from outside, so these stops are said otherwise. +PI_STOP_REASONS = { + "zh": { + "max_steps": ("達到步數上限", "pi 自己沒有步數上限：迴圈跑滿設定的步數，網頁就請 pi 中止，不管模型想不想繼續。"), + "user_stopped": ("使用者中斷", "你按了停止，網頁請 pi 中止：模型正在寫的回覆和正在執行的工具都會停下。"), + "approval_timeout": ("等待確認逾時", "一直沒有人按「允許」或「拒絕」（例如分頁已經關掉或重新整理），網頁當作拒絕並請 pi 中止，讓其他分頁可以使用。"), + "error": ("執行錯誤", "pi 或模型的 API 回報了錯誤，訊息在下面。"), + }, + "en": { + "max_steps": ("step limit reached", "pi has no step limit of its own: once the loop has gone around the set number of times, the page asks pi to abort, whether or not the model wants to go on."), + "user_stopped": ("stopped by the user", "You pressed stop, and the page asked pi to abort: the reply the model was writing and the tool that was running both stop."), + "approval_timeout": ("confirmation timed out", "No one pressed Allow or Reject (the tab was closed or reloaded, say), so the page took it as a refusal and asked pi to abort, freeing it for other tabs."), + "error": ("error", "pi or the model's API reported an error; the message is below."), + }, +} LAYER_TYPES = { "zh": {"sliding_attention": "滑動視窗", "full_attention": "full attention", "linear_attention": "線性 attention"}, "en": {"sliding_attention": "sliding window", "full_attention": "full attention", "linear_attention": "linear attention"}, @@ -51,7 +70,7 @@ "en": {"language_model": "language model", "audio_tower": "audio encoder", "vision_tower": "vision encoder", "visual": "vision encoder"}, } # Each kind of file a checkpoint's folder may hold, in the order the Model tab lists them: language -> (what it is, -# what it is for and what reads it). What one holds in a family's checkpoints is the family's (families.FileNote). +# what it is for and what reads it). What one holds in a family's checkpoints is the family's (model_families.FileNote). MODEL_FILES = { "config.json": { "zh": ("架構", "模型由什麼組成：類別、層數、寬度、head 數、詞彙量和脈絡上限。`AutoConfig` 最先讀它：`model_type` 決定 family，這個分頁上方的規格也來自它。"), @@ -149,6 +168,62 @@ """, } +# The same cycle, as pi runs it (harnesses/pi_harness.py): on this page's model ("page"), or on one of pi's ("pi"). +PI_INTRO = { + "page": { + "zh": """ +
+

送出問題後，這裡會一步一步畫出 pi 的 agent 迴圈。迴圈由 pi 跑，模型是這個網頁的：

+
+
脈絡
pi 把它的 system prompt、工具和對話組成一個請求，送到這個網頁在 127.0.0.1 上的模型端點；端點用 chat template 把請求接成一長串 token。請求和那串 token 都在「脈絡」分頁。
+
模型
一次推論，每個 token 一邊生成一邊傳回 pi。每個 token 都是從機率分布抽出來的：顏色越深，代表被選中的 token 機率越低。滑過 token 可以看候選。
+
Harness
端點解析模型寫出的工具呼叫，交給 pi；pi 檢查參數，打開「執行工具前先問我」時，先問你。
+
環境
pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱。結果接回對話，開始下一步。
+
停止
模型不再呼叫工具、達到步數或 token 上限，或是你按下停止。
+
+
+""", + "en": """ +
+

Send a question, and pi's agent loop is drawn here step by step. pi runs the loop, on this page's model:

+
+
Context
pi joins its system prompt, its tools and the conversation into one request to this page's model endpoint on 127.0.0.1, whose chat template joins the request into one long run of tokens. Both the request and those tokens are in the Context tab.
+
Model
One inference, each token going back to pi as it comes. Every token is drawn from a probability distribution: the darker a token, the less likely it was. Hover over a token to see the candidates.
+
Harness
The endpoint parses the tool calls the model wrote and hands them to pi, which checks their arguments; with Ask me before running a tool on, it asks you first.
+
Environment
pi's tools run on this computer, in this conversation's folder, without a sandbox. Their results go back into the conversation, and the next step begins.
+
Stop
The model calls no more tools, a step or token limit is reached, or you press stop.
+
+
+""", + }, + "pi": { + "zh": """ +
+

送出問題後，這裡會一步一步畫出 pi 的 agent 迴圈。迴圈由 pi 跑，網頁只畫它說它做了什麼：

+
+
脈絡
pi 把它的 system prompt、工具和對話組成一個請求，送到模型的 API。這個請求在「脈絡」分頁；API 那邊怎麼把它寫成模型讀的字串，pi 和 provider 都不說。
+
模型
一次 API 請求。回覆是一段一段傳回來的文字，不是 token，所以沒有機率；token 數是 provider 報的。
+
Harness
pi 讀出回覆裡的工具呼叫、檢查參數；打開「執行工具前先問我」時，先問你。
+
環境
pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱。結果接回對話，開始下一步。
+
停止
模型不再呼叫工具、達到步數上限，或是你按下停止。
+
+
+""", + "en": """ +
+

Send a question, and pi's agent loop is drawn here step by step. pi runs the loop; the page draws what pi says it does:

+
+
Context
pi joins its system prompt, its tools and the conversation into one request to the model's API. That request is in the Context tab; how the API writes it out as the string the model reads, neither pi nor the provider says.
+
Model
One request to the API. The reply streams back as text, not tokens, so there are no probabilities; the token counts are the provider's.
+
Harness
pi reads the tool calls in the reply and checks their arguments; with Ask me before running a tool on, it asks you first.
+
Environment
pi's tools run on this computer, in this conversation's folder, without a sandbox. Their results go back into the conversation, and the next step begins.
+
Stop
The model calls no more tools, the step limit is reached, or you press stop.
+
+
+""", + }, +} + CONTEXT_LEGEND = { "zh": """

@@ -210,7 +285,7 @@ def prose_html(text: str) -> str: return "".join(f'{p}' if i % 2 else p for i, p in enumerate(parts)) -def meaning_html(piece: str, family: families.Family, lang: str) -> str: +def meaning_html(piece: str, family: model_families.Family, lang: str) -> str: """A special token's name and what it is for; empty for tokens we have no note on.""" if piece not in family.special_tokens: return "" @@ -218,12 +293,12 @@ def meaning_html(piece: str, family: families.Family, lang: str) -> str: return f"{esc(name)}{prose_html(desc)}" -def stop_meaning(piece: str, family: families.Family, lang: str) -> str: +def stop_meaning(piece: str, family: model_families.Family, lang: str) -> str: """What stopping at this token means, e.g. that the model waits for a tool result.""" return family.stop_tokens.get(piece, {}).get(lang, "") -def chip(piece: str, family: families.Family, lang: str, count: int = 1) -> str: +def chip(piece: str, family: model_families.Family, lang: str, count: int = 1) -> str: """A special token as a dark chip, with ×count for a run of it; hovering it pops up what the token is for.""" meaning = meaning_html(piece, family, lang) tip = f'{meaning}' if meaning else "" @@ -231,7 +306,7 @@ def chip(piece: str, family: families.Family, lang: str, count: int = 1) -> str: return f'{esc(piece)}{times}{tip}' -def segments(text: str, family: families.Family) -> list[tuple[str, int]]: +def segments(text: str, family: model_families.Family) -> list[tuple[str, int]]: """Text split into plain text, (text, 0), and runs of one control token, (token, how many in a row).""" out, last = [], 0 for m in family.special_re.finditer(text): @@ -247,7 +322,7 @@ def segments(text: str, family: families.Family) -> list[tuple[str, int]]: return out -def special_html(text: str, family: families.Family, lang: str, last: int | None = None) -> str: +def special_html(text: str, family: model_families.Family, lang: str, last: int | None = None) -> str: """Escape text and turn the family's control tokens, such as <|turn>, into chips; a run of RUN_MIN or more of one token is one chip with its count. With `last`, only about the last that many characters are drawn, after an ellipsis, a run counting as one token, so a cut never lands inside one.""" @@ -304,17 +379,18 @@ def media_line(counts: dict, lang: str) -> str: # ---------- events -> steps ---------- def build(events: list[dict]) -> dict: - data = {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + data: dict = {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} for ev in events: kind = ev["type"] data["last"], data["t"] = kind, ev["t"] if kind == "run_start": - data["model"], data["family"] = ev["model"], families.get(ev["family"]) - data["settings"] = ev["settings"] + # A harness that does not know how the model writes its turns (pi) sends no family. + data["model"], data["family"] = ev["model"], model_families.get(ev["family"]) if ev.get("family") else None + data["harness"], data["settings"] = ev.get("harness", "simple"), ev["settings"] elif kind == "step_start": data["steps"].append({ "n": ev["step"], "t0": ev["t"], "t1": ev["t"], "context": None, "prefill": None, - "tokens": [], "done": None, "parsed": None, "calls": {}, + "tokens": [], "text": [], "request": None, "done": None, "parsed": None, "calls": {}, }) elif kind == "run_end": data["end"] = ev @@ -325,7 +401,12 @@ def build(events: list[dict]) -> dict: step["t1"] = ev["t"] if kind == "token": step["tokens"].append(ev) - elif kind in ("context", "prefill", "parsed"): + elif kind == "delta": # kept as runs of one channel: [{"channel", "text"}] + if step["text"] and step["text"][-1]["channel"] == ev["channel"]: + step["text"][-1] = {**step["text"][-1], "text": step["text"][-1]["text"] + ev["text"]} + else: + step["text"].append({"channel": ev["channel"], "text": ev["text"]}) + elif kind in ("request", "context", "prefill", "parsed"): step[kind] = ev elif kind == "model_end": step["done"] = ev @@ -357,7 +438,7 @@ def with_drafts(tokens: list[dict], mtp: dict) -> list[dict]: kept, turned_down = set(mtp.get("kept_at", ())), set(mtp.get("turned_down_at", ())) if not kept and not turned_down: return tokens - marks = {**{i: {"piece": None, "accepted": False} for i in turned_down}, **{i: {"accepted": True} for i in kept}} + marks: dict[int, dict] = {**{i: {"piece": None, "accepted": False} for i in turned_down}, **{i: {"accepted": True} for i in kept}} return [{**t, "draft": {"piece": t["piece"], **marks[t["index"]]}} if t["index"] in marks and "draft" not in t else t for t in tokens] @@ -375,10 +456,23 @@ def pending_approvals(data: dict) -> list[dict]: def draft(data: dict) -> str: - """The reply as it streams: content tokens of the step in progress.""" + """The reply as it streams: content tokens of the step in progress, or the content its harness streamed.""" if not data["steps"]: return "" - return "".join(t["text"] for t in data["steps"][-1]["tokens"] if t["channel"] == "content") + step = data["steps"][-1] + return "".join(t["text"] for t in step["tokens"] or step["text"] if t["channel"] == "content") + + +def generated(step: dict) -> int: + """How many tokens the model wrote in a step: those drawn, or the count its provider reported when none are.""" + return len(step["tokens"]) or (step["done"] or {}).get("n_tokens") or 0 + + +def context_tokens(step: dict) -> int | None: + """How many tokens the model read in a step: the context's, or the prompt its provider reported reading.""" + if step["context"]: + return step["context"]["n_tokens"] + return ((step["done"] or {}).get("usage") or {}).get("prompt") def stop_title(stop_reason: str, lang: str) -> str: @@ -395,8 +489,8 @@ def status(data: dict, lang: str) -> str: if last == "prefill": read = step["prefill"]["n_tokens"] return f"Step {n} · " + pick(lang, f"模型讀完 {read:,} 個 token，開始生成", f"the model has read {read:,} tokens and starts generating") - if last == "token": - channel = step["tokens"][-1]["channel"] + if last in ("token", "delta"): + channel = (step["tokens"] or step["text"])[-1]["channel"] if channel == "thought": return f"Step {n} · " + pick(lang, "模型思考中…", "the model is thinking…") if channel == "tool_call": @@ -421,8 +515,8 @@ def footer(data: dict, lang: str) -> str: def run_stats(data: dict, lang: str) -> str: s = data["settings"] steps = data["steps"] - ctx = next((st["context"]["n_tokens"] for st in reversed(steps) if st["context"]), 0) - gen = sum(len(st["tokens"]) for st in steps) + ctx = next((n for st in reversed(steps) if (n := context_tokens(st)) is not None), 0) + gen = sum(generated(st) for st in steps) return f"Step {len(steps)} / {s.get('max_steps', '?')} · " + pick( lang, f"脈絡 {ctx:,} tokens · 共生成 {gen:,} tokens", f"context {ctx:,} tokens · {gen:,} tokens generated" ) + f" · {fmt_ms(data['t'])}" @@ -459,18 +553,25 @@ def trace_html(data: dict, lang: str) -> str: ("env-context", "M102 150 V84", 92, 120, "end", {"zh": "觀察", "en": "observation"}), ("harness-stop", "M462 184 H554", 509, 174, "middle", {"zh": "沒有工具呼叫", "en": "no tool call"}), ] +# What the diagram says otherwise when pi runs the loop: its own harness, its tools on this computer, and text, not +# tokens, from the model's API. +PI_DIAGRAM = { + "harness": {"zh": "pi · 驗證 · 權限", "en": "pi · check · ask"}, + "env": {"zh": "這台電腦", "en": "this computer"}, + "model-harness": {"zh": "輸出文字", "en": "text out"}, +} def loop_state(data: dict | None, lang: str) -> dict: """Which node and edge are active, what each node's caption says, and the step counter.""" - state = {"active": None, "edge": None, "visited": set(), "waiting": False, "captions": {}, + state: dict = {"active": None, "edge": None, "visited": set(), "waiting": False, "captions": {}, "counter": pick(lang, "等待問題", "waiting for a question"), "ended": False} if not data or not data["last"]: return state caps, steps = state["captions"], data["steps"] for s in steps: state["visited"].add("context") - if s["prefill"]: + if s["prefill"] or s["text"]: state["visited"].add("model") if s["done"]: state["visited"].add("harness") @@ -479,14 +580,17 @@ def loop_state(data: dict | None, lang: str) -> dict: step = steps[-1] if steps else None if step: - if step["context"]: - caps["context"] = f"{step['context']['n_tokens']:,} tokens" + if (read := context_tokens(step)) is not None: + caps["context"] = f"{read:,} tokens" if step["prefill"]: caps["model"] = f"prefill {fmt_ms(step['prefill']['ms'])}" if step["tokens"]: caps["model"] = pick(lang, f"已生成 {len(step['tokens'])} tokens", f"{len(step['tokens'])} tokens so far") - if step["done"]: - caps["model"] = f"{step['done']['n_tokens']} tok · {step['done']['tok_per_s']} tok/s" + elif step["text"]: + chars = sum(len(r["text"]) for r in step["text"]) + caps["model"] = pick(lang, f"已收到 {chars:,} 字元", f"{chars:,} characters so far") + if done := step["done"]: + caps["model"] = f"{done['n_tokens']} tok" + (f" · {done['tok_per_s']} tok/s" if done.get("tok_per_s") else "") calls = list(step["calls"].values()) if step["parsed"]: caps["harness"] = pick(lang, "、", ", ").join(c["name"] for c in calls) or pick(lang, "沒有工具呼叫", "no tool call") @@ -504,10 +608,12 @@ def loop_state(data: dict | None, lang: str) -> dict: last = data["last"] started = bool(step and step["calls"] and list(step["calls"].values())[-1]["started"]) state["active"], state["edge"] = { - "step_start": ("context", "env-context" if step and step["n"] > 1 else None), + # pi sends the request as its turn starts, and says nothing more until the model's output comes back. + "step_start": ("context", "context-model" if data["harness"] == "pi" else "env-context" if step and step["n"] > 1 else None), "context": ("context", "context-model"), "prefill": ("model", "context-model"), "token": ("model", None), + "delta": ("model", None), "model_end": ("harness", "model-harness"), "parsed": ("harness", "model-harness"), "tool_call": ("harness", None), @@ -557,15 +663,19 @@ def loop_state(data: dict | None, lang: str) -> dict: """ -def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = None, model: str | None = None) -> str: +def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = None, model: str | None = None, + harness: str | None = None) -> str: """The loop diagram as an : Streamlit's HTML sanitizer removes inline . - `model` names the model when no run says which one it is.""" + `model` and `harness` name the model and what runs the loop when no run says which they are.""" st = loop_state(data, lang) if counter: st["counter"] = counter model = (data or {}).get("model") or model if model: st["captions"].setdefault("model", model.split("/")[-1]) + words_for = PI_DIAGRAM if ((data or {}).get("harness") or harness) == "pi" else {} + if data and data["family"]: # pi asking the page's own model: the tokens come out as ever + words_for = {k: v for k, v in words_for.items() if k != "model-harness"} pal = PALETTES["dark" if dark else "light"] css = DIAGRAM_CSS for key, value in pal.items(): @@ -573,20 +683,22 @@ def diagram_html(data: dict | None, dark: bool, lang: str, counter: str | None = edges = [] for key, path, x, y, anchor, label in EDGES: cls = "edge on" if key == st["edge"] else "edge" + label = words_for.get(key, label) edges.append(f'{label[lang]}') nodes = [] for key, x, y, words in NODES: title, caption = words[lang] - cls = ["node"] + caption = words_for[key][lang] if key in words_for else caption + classes = ["node"] if key in st["visited"]: - cls.append("visited") + classes.append("visited") if key == st["active"]: - cls.append("on") + classes.append("on") if st["waiting"]: - cls.append("waiting") + classes.append("waiting") cx = x + 80 nodes.append( - f'' + f'' f'' f'{title}' f'{esc(st["captions"].get(key, caption))}' @@ -613,12 +725,12 @@ def heat(p: float) -> str: return f"{min(1.0, max(0.0, 1 - p)) * 0.6:.3f}" -def special_note(tok: dict, family: families.Family, lang: str) -> str: +def special_note(tok: dict, family: model_families.Family, lang: str) -> str: meaning = meaning_html(tok["piece"], family, lang) if tok.get("special") else "" return f'{meaning}' if meaning else "" -def draft_note(tok: dict, family: families.Family, lang: str) -> str: +def draft_note(tok: dict, family: model_families.Family, lang: str) -> str: """What MTP did at this position: the drafter's guess was kept, or turned down and replaced.""" draft = tok.get("draft") if not draft: @@ -649,7 +761,7 @@ def draft_note(tok: dict, family: families.Family, lang: str) -> str: ) + "" -def tip_html(tok: dict, family: families.Family, lang: str) -> str: +def tip_html(tok: dict, family: model_families.Family, lang: str) -> str: """A finished token's popover: the token, what it means if it is special, what MTP did there, and the numbers and candidates the backend reported for it (backend.TOKEN_DETAILS); a number it did not report is left out.""" numbers = [ @@ -669,7 +781,7 @@ def tip_html(tok: dict, family: families.Family, lang: str) -> str: ) -def candidates_html(tok: dict, family: families.Family, lang: str) -> str: +def candidates_html(tok: dict, family: model_families.Family, lang: str) -> str: """The most likely tokens at this position, the chosen one highlighted and those top-k / top-p cut grayed out, when the backend reported them; a candidate it did not say was cut or kept is drawn as it is.""" top = tok.get("top") @@ -700,12 +812,12 @@ def known(item: dict, key: str) -> bool: return item.get(key) is not None -def reported(tokens: list[dict], key: str) -> bool: +def reported(tokens: Sequence[dict], key: str) -> bool: """Whether the backend reports `key` for its tokens: some token has it, or there are no tokens yet to tell.""" return not tokens or any(known(t, key) for t in tokens) -def tokens_html(tokens: list[dict], with_tips: bool, family: families.Family, lang: str) -> str: +def tokens_html(tokens: list[dict], with_tips: bool, family: model_families.Family, lang: str) -> str: """Generated tokens with heat. Finished steps get a popover per token; while a step is still generating, only special tokens get one, and it just says what the token is for.""" out = [] @@ -728,7 +840,7 @@ def tokens_html(tokens: list[dict], with_tips: bool, family: families.Family, la return f'

{"".join(out)}
' -def token_legend(s: dict, live: bool, lang: str, tokens: list[dict] = ()) -> str: +def token_legend(s: dict, live: bool, lang: str, tokens: Sequence[dict] = ()) -> str: """How to read the tokens. The heat scale and the candidates are mentioned only if the backend reports them for `tokens` (the run's tokens so far).""" if s.get("temperature", 1) <= 0: @@ -809,6 +921,8 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: + f'

{ctx["n_tokens"]:,} tokens · {pick(lang, f"上限的 {share}", f"{share} of the limit")} · {esc(roles)}{media}

' f'

{label}

{special_html(prompt[start:], family, lang, last=1500)}
' ) + elif step["request"]: # pi asking a provider's model: what pi sent, not what the model read + parts.append(request_section(step, lang)) if step["prefill"]: pf = step["prefill"] @@ -845,6 +959,8 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: f'

{meta}

{token_legend(s, live, lang, [t for st in data["steps"] for t in st["tokens"]])}' f'{tokens_html(step["tokens"], not live, family, lang)}' ) + elif step["text"] or step["done"]: # from a harness that sees the model's text, not its tokens + parts.append(streamed_html(step, lang)) if step["parsed"]: body = [] @@ -860,6 +976,13 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: if c["errors"]: errors = esc(pick(lang, "；", "; ").join(c["errors"])) body.append(f'

✗ {errors} → ' + pick(lang, "把錯誤當作觀察送回模型", "the error goes back to the model as an observation") + "

") + elif family is None: # pi: the API sends the call as JSON already, and pi checks it as it runs the tool + body.append('

' + pick(lang, "API 傳回的已經是 JSON；pi 執行前再用工具的 schema 檢查參數", + "the API returned it as JSON; pi checks the arguments against the tool's schema before running it") + "

") + elif data["harness"] == "pi": # pi asking the page's own model: the endpoint parses, pi checks + name = esc(family.name) + body.append('

✓ ' + pick(lang, f"模型端點從 {name} 格式解析成 JSON 交給 pi；pi 執行前再用工具的 schema 檢查參數", + f"the model endpoint parsed it from the {name} format into JSON for pi; pi checks the arguments against the tool's schema before running it") + "

") else: name = esc(family.name) body.append('

✓ ' + pick(lang, f"從 {name} 格式解析成 JSON，符合工具 schema", @@ -879,8 +1002,9 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: "the confirmation timed out; the harness took it as a refusal and stopped the loop") + "

") elif c["approval"] == "stopped": body.append('

⏹ ' + pick(lang, "執行被停止，這個呼叫沒有執行", "the run was stopped, so this call did not run") + "

") + what = PI_DIAGRAM["harness"][lang] if data["harness"] == "pi" else pick(lang, "解析 · 驗證 · 權限", "parse · check · ask") parts.append( - f'

Harness {pick(lang, "解析 · 驗證 · 權限", "parse · check · ask")}

' + f'

Harness {what}

' f'{"".join(body)}
' ) @@ -906,6 +1030,70 @@ def step_html(step: dict, data: dict, lang: str, live: bool) -> str: return "".join(parts) +def request_section(step: dict, lang: str) -> str: + """The context section of a step whose harness asks a model's API (pi): the request it sent, as JSON, since what + the model read of it is not to be seen; its token count once the provider has said it.""" + req = step["request"] + roles = " · ".join(f"{r} {n}" for r, n in req["roles"].items()) + read = ((step["done"] or {}).get("usage") or {}).get("prompt") + meta = (f"{read:,} tokens · " if read else "") + pick(lang, f"{len(req['text']):,} 字元", f"{len(req['text']):,} characters") + f" · {esc(roles)}" + if req["added_from"] is None: + label = pick(lang, "pi 送給模型 API 的請求：system prompt、工具和對話，寫成 JSON", "The request pi sends the model's API: the system prompt, the tools and the conversation, as JSON") + new = req["text"] + else: + label = pick(lang, "這一步的請求新加的部分：上一步的回覆和觀察", "What is new in this step's request: the last step's reply and observation") + new = req["text"][req["added_from"]:req["added_to"]] + snippet = esc(new) if len(new) <= 1500 else "…" + esc(new[-1500:]) + return ( + '

' + + pick(lang, "脈絡 Context · pi 送給 API 的請求", "Context the request pi sends the API") + + f'

{meta}

{label}

{snippet}
' + ) + + +def text_html(runs: list[dict]) -> str: + """The text a harness streamed (delta), by channel: thinking in italics, tool calls in monospace; no token + boundaries, since it saw none.""" + return '
' + "".join(f'{esc(r["text"])}' for r in runs) + "
" + + +def streamed_html(step: dict, lang: str) -> str: + """The model section of a step whose harness asks a model's API (pi): the text as it streamed, and what the + provider reported once the reply was done. No number here was guessed: an API sends no probabilities.""" + sep, done, meta = '|', step["done"], [] + if done: + meta.append(pick(lang, f'生成 {done["n_tokens"]:,} tokens', f'wrote {done["n_tokens"]:,} tokens') + + (f' · {done["tok_per_s"]} tok/s' if done.get("tok_per_s") else "")) + if done.get("first_ms") is not None: + meta.append(pick(lang, f"第一段輸出在 {fmt_ms(done['first_ms'])} 後回來", f"first output after {fmt_ms(done['first_ms'])}")) + usage = done.get("usage") or {} + if usage.get("prompt"): + cached = pick(lang, f"，其中 {usage['cached']:,} 來自 cache", f", {usage['cached']:,} from the cache") if usage.get("cached") else "" + meta.append(pick(lang, f"讀了 {usage['prompt']:,} tokens{cached}", f"read {usage['prompt']:,} tokens{cached}")) + if usage.get("cost"): + meta.append(pick(lang, f"照 pi 的價目約 ${usage['cost']:.4f}", f"about ${usage['cost']:.4f} at pi's prices")) + if done["finish"] == "max_new_tokens": + meta.append(pick(lang, "到達 token 上限", "hit the token limit")) + elif done["finish"] == "user_stopped": + meta.append(pick(lang, "使用者中斷", "stopped by the user")) + elif done["finish"] == "error": + meta.append(pick(lang, "請求失敗", "the request failed")) + else: + chars = sum(len(r["text"]) for r in step["text"]) + meta.append(pick(lang, f"已收到 {chars:,} 字元…", f"{chars:,} characters so far…")) + legend = ( + '

' + pick(lang, "API 傳回來的是文字，不是 token，所以沒有機率可畫；token 數是 provider 報的。", + "The API sends back text, not tokens, so there are no probabilities to draw; the token counts are the provider's.") + + "" + pick(lang, '斜體 = 思考等寬 = 工具呼叫', + 'italic = thinkingmonospace = tool call') + "

" + ) + return ( + f'

{pick(lang, "模型", "Model")} ' + f'{pick(lang, "一次 API 請求，回覆一段一段傳回來", "one request to the API; the reply streams back")}

' + f'

{sep.join(meta)}

{legend}{text_html(step["text"])}
' + ) + + def call_html(name: str, arguments: dict | None) -> str: """A tool call as name(arguments JSON). An argument that runs over several lines, such as code, is shown as … in the JSON and written out below it.""" @@ -924,13 +1112,14 @@ def error_text(data: dict, lang: str) -> str | None: def stop_html(data: dict, lang: str) -> str: end = data["end"] - title, desc = STOP_REASONS[lang].get(end["stop_reason"], (end["stop_reason"], "")) + reasons = {**STOP_REASONS[lang], **(PI_STOP_REASONS[lang] if data["harness"] == "pi" else {})} + title, desc = reasons.get(end["stop_reason"], (end["stop_reason"], "")) last = data["steps"][-1] if data["steps"] else None piece = last["done"]["stop_piece"] if last and last["done"] else None if end["stop_reason"] == "no_tool_call" and piece: stop_chip, meaning = chip(piece, data["family"], lang), stop_meaning(piece, data["family"], lang) desc += pick(lang, f" 最後停在 {stop_chip}（{meaning}）。", f" It stopped at {stop_chip} ({meaning}).") - gen = sum(len(s["tokens"]) for s in data["steps"]) + gen = sum(generated(s) for s in data["steps"]) error = f'

{esc(error_text(data, lang))}

' if data["error"] else "" cls = "stop-card is-error" if end["stop_reason"] == "error" else "stop-card" totals = pick(lang, f'共 {len(data["steps"])} 步 · 生成 {gen:,} tokens', f'{n_steps(len(data["steps"]), lang)} · {gen:,} tokens generated') @@ -952,7 +1141,7 @@ def context_summary(step: dict, lang: str) -> str: ) + f" · {roles}" + (pick(lang, f" · 其中 {media_line(media, lang)} tokens", f" · of which {media_line(media, lang)} tokens") if media else "") -def context_html(step: dict, family: families.Family, lang: str) -> str: +def context_html(step: dict, family: model_families.Family, lang: str) -> str: prompt = step["context"]["prompt"] start = max(prompt.rfind(family.user_turn), 0) if step["n"] == 1 else step["context"]["added_from"] gen = step["done"]["raw"] if step["done"] else "".join(t["text"] for t in step["tokens"]) @@ -962,12 +1151,35 @@ def context_html(step: dict, family: families.Family, lang: str) -> str: ) +REQUEST_LEGEND = { + "zh": '

pi 送給模型 API 的請求，原樣寫成 JSON（圖片只留類型）這一步新加入

', + "en": '

the request pi sends the model\'s API, as JSON (an image as its type alone)new in this step

', +} + + +def request_summary(step: dict, lang: str) -> str: + req = step["request"] + roles = " · ".join(f"{r} {n}" for r, n in req["roles"].items()) + read = ((step["done"] or {}).get("usage") or {}).get("prompt") + tokens = pick(lang, f"provider 說讀了 {read:,} tokens · ", f"{read:,} tokens, as the provider counts them · ") if read else "" + return tokens + pick(lang, f"{len(req['text']):,} 字元", f"{len(req['text']):,} characters") + f" · {roles}" + + +def request_html(step: dict) -> str: + """The whole request a step sent, its new part marked.""" + req, text = step["request"], step["request"]["text"] + if req["added_from"] is None: + return f'
{esc(text)}
' + a, b = req["added_from"], req["added_to"] + return f'
{esc(text[:a])}{esc(text[a:b])}{esc(text[b:])}
' + + # ---------- model tab ---------- def model_html(info: dict, lang: str) -> str: """The model's card. A model a server runs (info["server"], vllm_backend.py) has no device, precision, parameter count or load time here, since its weights are not loaded on this machine; the card leaves those out.""" - family = families.get(info["family"]) + family = model_families.get(info["family"]) server = info.get("server") colors = {"language_model": "var(--c-model)", "audio_tower": "var(--c-env)", "vision_tower": "var(--c-context)", "visual": "var(--c-context)"} other = pick(lang, "其他", "other") @@ -1106,15 +1318,15 @@ def knobs_html(s: dict, defaults: dict, lang: str) -> str: def step_rows(data: dict, lang: str) -> list[dict]: rows = [] for s in data["steps"]: - if not s["context"]: + if not s["context"] and not s["done"]: continue probs = [t["p"] for t in s["tokens"] if not t["special"] and known(t, "p")] done = s["done"] or {} rows.append({ "step": s["n"], - pick(lang, "脈絡 tokens", "context tokens"): s["context"]["n_tokens"], + pick(lang, "脈絡 tokens", "context tokens"): context_tokens(s), "prefill ms": s["prefill"]["ms"] if s["prefill"] else None, - pick(lang, "生成 tokens", "generated tokens"): len(s["tokens"]), + pick(lang, "生成 tokens", "generated tokens"): generated(s), "tok/s": done.get("tok_per_s"), pick(lang, "平均機率", "mean probability"): round(sum(probs) / len(probs), 3) if probs else None, pick(lang, "停止", "stop"): done.get("stop_piece") or done.get("finish"), @@ -1126,7 +1338,7 @@ def step_rows(data: dict, lang: str) -> list[dict]: return rows -def tools_html(tools: dict, enabled: bool, family: families.Family, lang: str, offered: list[str] | None = None) -> str: +def tools_html(tools: dict, enabled: bool, family: model_families.Family, lang: str, offered: list[str] | None = None) -> str: """Every tool with its schema and environment; one not in `offered` (when given) is marked as kept from the model.""" items = [] for t in tools.values(): @@ -1154,7 +1366,7 @@ def tools_html(tools: dict, enabled: bool, family: families.Family, lang: str, o def special_tokens_html(info: dict, lang: str) -> str: """Every special token of the model's tokenizer, grouped, with its id and what it is for.""" - family = families.get(info["family"]) + family = model_families.get(info["family"]) ids = info["special_tokens"] groups = [(title, sub, [p for p, _, _ in rows if p in ids]) for title, sub, rows in family.special_groups[lang]] extra = [p for p in ids if p not in family.special_tokens] @@ -1200,14 +1412,14 @@ def field_value(value) -> str: return text if len(text) <= 80 else text[:79] + "…" -def files_html(folder: dict, notes: dict, family: families.Family, lang: str, served: bool = False) -> str: +def files_html(folder: dict, notes: dict, family: model_families.Family, lang: str, served: bool = False) -> str: """One folder's files, in MODEL_FILES order: size, what the kind of file is for, what the family's note says it holds and the values of the fields the note names. A file the family has a note on but the folder lacks is listed too; when a server runs the model (`served`), the weights it lacks are on the server.""" present = {file_kind(f["name"]) or f["name"] for f in folder["files"]} rows = [(f["name"], f) for f in folder["files"]] + [(name, None) for name in notes if name not in present] order = list(MODEL_FILES) - rows.sort(key=lambda row: (order.index(file_kind(row[0])) if file_kind(row[0]) else len(order), row[0])) + rows.sort(key=lambda row: (order.index(kind) if (kind := file_kind(row[0])) else len(order), row[0])) out, said = [], set() for name, file in rows: kind = file_kind(name) @@ -1234,12 +1446,12 @@ def files_html(folder: dict, notes: dict, family: families.Family, lang: str, se def folder_html(info: dict, lang: str) -> str: """The files in the model's folder, and in its MTP drafter's once that is loaded.""" - family = families.get(info["family"]) + family = model_families.get(info["family"]) folder = info["folder"] where = f'{esc(folder["path"])}' if folder["revision"]: rev = folder["revision"][:7] - if families.REVISIONS.get(info["model_id"]) == folder["revision"]: + if model_families.REVISIONS.get(info["model_id"]) == folder["revision"]: which = pick(lang, f"從 Hub 下載的 checkpoint，revision {rev}：app 固定下載這個 commit（`REVISIONS`），所以每次下載都是同樣的檔案。", f"A checkpoint downloaded from the Hub at revision {rev}, the commit the app pins it to (`REVISIONS`), so every download gets these same files. ") else: @@ -1280,3 +1492,85 @@ def folder_html(info: dict, lang: str) -> str: " Below is each file's size and purpose, and what it holds in this family's checkpoints; the values of the fields are read from the files.") + f'

{files_html(folder, family.file_notes, family, lang, served="server" in info)}{extra}' ) + + +# ---------- pi ---------- + +def pi_model_html(model: dict, version: str, lang: str) -> str: + """The model pi asks, from pi's own model object: it runs at its provider, and the page sees only what the API + sends back.""" + name, where = model.get("name") or model["id"], f'{model["provider"]}/{model["id"]}' + cost = model.get("cost") or {} + specs = [ + ("provider", model["provider"]), + ("API", model.get("api")), + (pick(lang, "脈絡上限", "context limit"), f'{model["contextWindow"]:,} tokens' if model.get("contextWindow") else None), + (pick(lang, "輸出上限", "output limit"), f'{model["maxTokens"]:,} tokens' if model.get("maxTokens") else None), + (pick(lang, "讀得懂", "reads"), " · ".join(MODALITIES[lang].get(k, k) for k in model.get("input") or ("text",))), + (pick(lang, "會思考", "thinks"), pick(lang, "會", "yes") if model.get("reasoning") else pick(lang, "不會", "no")), + (pick(lang, "每百萬 token", "per million tokens"), + pick(lang, f'讀 ${cost["input"]} · 寫 ${cost["output"]}', f'${cost["input"]} in · ${cost["output"]} out') if cost.get("input") is not None and cost.get("output") is not None else None), + ] + spec_html = "".join(f"
{esc(k)}
{esc(v)}
" for k, v in specs if v) + about = pick( + lang, + f"pi {version} 跑迴圈，透過 {model['provider']} 的 API 問這個模型；模型在 provider 的伺服器上跑，不在這台電腦上。" + "模型清單和登入資訊是 pi 自己的（`pi` 再輸入 `/login`，或環境裡的 API key）。" + "API 傳回來的是文字，不是 token，所以這裡沒有 token 的機率、特殊 token 和模型資料夾；pi 送出的請求（system prompt、工具和對話）在「脈絡」分頁。", + f"pi {version} runs the loop and asks this model through {model['provider']}'s API; the model runs on the provider's servers, " + "not on this computer. The models and the credentials are pi's own (`pi`, then `/login`, or an API key in the environment). " + "An API sends back text, not tokens, so there are no token probabilities, special tokens or model folder here; the request pi sends (its system prompt, its tools and the conversation) is in the Context tab.", + ) + return ( + f'

{esc(name)} {esc(where)}

{prose_html(about)}

' + f'
{spec_html}
' + ) + + +def pi_settings_html(s: dict, lang: str) -> str: + """What the sidebar gave pi for the turn.""" + page = s.get("page_model") # pi asking the page's own model, whose decoding the sidebar sets + if page: + thinking = (pick(lang, "思考模式", "Thinking mode"), pick(lang, "開", "on") if s.get("thinking", "off") != "off" else pick(lang, "關", "off"), + pick(lang, "pi 把它當成思考等級送出，模型端點再換成 chat template 的思考開關。", + "pi sends it as its thinking level, which the model endpoint turns into the chat template's thinking switch.")) + else: + thinking = (pick(lang, "思考", "Thinking"), s.get("thinking", "off"), + pick(lang, "pi 的思考等級：off 到 high，越高模型在回覆前想得越多（也越貴）。模型不會思考時只有 off。", + "pi's thinking level, from off to high: the higher, the more the model thinks before it replies (and the more it costs). A model that does not think has only off.")) + rows = [ + thinking, + (pick(lang, "執行工具前先問我", "Ask me before running a tool"), pick(lang, "開", "on") if s.get("approve_tools") else pick(lang, "關", "off"), + pick(lang, "pi 自己不問；網頁讓 pi 載入的 extension（pi_extension.ts）在每次執行工具前問你。", + "pi does not ask by itself; the extension the page loads into pi (pi_extension.ts) asks you before every tool call.")), + (pick(lang, "步數上限", "Max steps"), s.get("max_steps"), + pick(lang, "pi 沒有步數上限；跑滿設定的步數，網頁就請 pi 中止。", "pi has no step limit; once the loop has gone around this many times, the page asks pi to abort.")), + ] + body = "".join(f"
{k} {esc(v)}
{d}
" for k, v, d in rows) + return ( + f'

{pick(lang, "pi 的設定", "pi settings")}

' + + (pick(lang, "pi 問的是這個網頁的模型：解碼設定是側欄的，寫在上面；模型端點照它們生成，不管請求裡要什麼。", + "pi asks this page's model: the decoding settings are the sidebar's, above, and the model endpoint generates with them, whatever the request asks for.") + if page else pick(lang, "解碼的設定（temperature、top-k、top-p）在 provider 那邊，pi 不讓人改；側欄只給得了這些。", + "The decoding settings (temperature, top-k, top-p) are the provider's, and pi does not let them be changed; the sidebar gives only these.")) + + f'

{body}
' + ) + + +def pi_tools_html(tools: dict, offered: list[str], lang: str) -> str: + """pi's built-in tools, each with what it does and where it runs; one not in `offered` is marked as kept from the model.""" + items = [] + for name, (what, environment) in tools.items(): + off = "" if name in offered else '

' + pick(lang, "這一輪沒有提供給模型，見側欄的設定", "not offered to the model in this turn; see the sidebar") + "

" + items.append( + f'

  • {esc(name)}

    {esc(what[lang])}

    ' + f'

    {pick(lang, "環境：", "environment: ")}{esc(environment[lang])}

    {off}
    • ' + ) + return ( + f'

    {pick(lang, "動作空間：pi 的工具", "Action space: the tools pi has")}

    ' + + pick(lang, "pi 的工具在這台電腦上、這段對話的資料夾裡執行，沒有沙箱：bash 碰得到你碰得到的一切。想先看過每個呼叫，就打開「執行工具前先問我」。" + "工具的 schema 和說明是 pi 寫給模型的，在「脈絡」分頁 pi 送出的請求裡。", + "pi's tools run on this computer, in this conversation's folder, without a sandbox: bash reaches whatever you can. To see each call first, " + "turn on Ask me before running a tool. The tools' schemas and descriptions are pi's, for the model; they are in the request pi sends, in the Context tab.") + + f'

      {"".join(items)}
    ' + ) diff --git a/docker-compose/compose.gpu.yaml b/docker-compose/compose.gpu.yaml index 0c228fb..2a09c7e 100644 --- a/docker-compose/compose.gpu.yaml +++ b/docker-compose/compose.gpu.yaml @@ -21,7 +21,7 @@ services: # shares the machine's memory (a DGX Spark, say), never is. - --kv-cache-memory-bytes=${VLLM_KV_CACHE_MEMORY_BYTES:-2G} - --gpu-memory-utilization=${VLLM_GPU_MEMORY_UTILIZATION:-0.3} - # Speculative decoding (MTP): the page sets the model's --speculative-config from families.VLLM_SPECULATIVE, as + # Speculative decoding (MTP): the page sets the model's --speculative-config from model_families.VLLM_SPECULATIVE, as # JSON, unless the sidebar's MTP switch is off; by hand it is off ("None"). vLLM's CPU image runs neither model's, # so compose.yaml has neither flag. With a config, each step's counts in detail are what the page marks the guesses # from; without one vLLM refuses any but "none". diff --git a/docker-compose/compose.yaml b/docker-compose/compose.yaml index e418ef8..0959114 100644 --- a/docker-compose/compose.yaml +++ b/docker-compose/compose.yaml @@ -1,7 +1,7 @@ # A vLLM server in Docker: what runs the model when the sidebar's "What runs the model" is vLLM (Docker). The page # (vllm_docker.py) starts it with the model picked in the sidebar, # -# VLLM_MODEL= VLLM_REVISION= docker compose -f docker-compose/compose.yaml up -d +# VLLM_MODEL= VLLM_REVISION= docker compose -f docker-compose/compose.yaml up -d # # waits until it answers, and runs `docker compose down` when transformers is picked again. Started by hand, it serves # Qwen3.5 0.8B unless VLLM_MODEL and VLLM_REVISION say otherwise, and VLLM_URL=http://localhost:8000 points the page diff --git a/docs/screenshot-en.png b/docs/screenshot-en.png index 1e2489d..6b23874 100644 Binary files a/docs/screenshot-en.png and b/docs/screenshot-en.png differ diff --git a/docs/screenshot-zh.png b/docs/screenshot-zh.png index 0aacbf5..9d251cc 100644 Binary files a/docs/screenshot-zh.png and b/docs/screenshot-zh.png differ diff --git a/harness/__init__.py b/harness/__init__.py deleted file mode 100644 index 8232a2f..0000000 --- a/harness/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""The harness: everything that runs a turn, and nothing a person sees. - -agent.py is the loop itself (Run): context, model, parse, tool, observation, stop, with one event per stage for the -app to draw. tools.py has the tools it runs, with run_python's Docker sandbox built from sandbox/. media.py turns the -files attached to a question into what the model reads. families/ is everything model-specific, and backends/ is what -runs the model, behind one interface. i18n.py has the page's two languages, which the errors the model reads use too. - -Nothing here imports Streamlit, and importing the package loads nothing heavy: the app's guide imports families and -i18n with pygments alone. -""" diff --git a/pyproject.toml b/pyproject.toml index cc4a67b..e92e57e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,13 @@ dependencies = [ [dependency-groups] dev = [ + "mypy>=2.4.0", "openpyxl>=3.1", "pytest>=9.1.1", + "ruff>=0.17.2", + "types-openpyxl>=3.1.5.20261017", + "types-pygments>=2.21.0.20260819", + "types-pyyaml>=6.0.12.20260906", ] [tool.pytest.ini_options] @@ -26,6 +31,37 @@ testpaths = ["tests"] pythonpath = [".", "tests"] markers = ["model: loads the real model weights; set RUN_MODEL_TESTS=1 to run"] +# Lint only: there is no formatter, so `ruff check` never rewrites a line it has nothing to say about, and no rule +# checks line length. line-length only tells the import sorter when to wrap, and the code keeps an import on one line. +# The tests' own modules (fakes, samples, fake_pi, html_check) are imported from tests/ like the packages are from +# the project's folder, so both are src and sort with agent and app. +[tool.ruff] +src = [".", "tests"] +line-length = 140 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I", "B", "UP"] +# zip() pairs sequences built to the same length (a step's calls and their ids, topk's values and indices), so strict= +# would say nothing those lines do not already; and a lambda given a name is how the code picks one of several +# callables (app/page.py's model) or keeps a one-line check beside the asserts that use it. +ignore = ["B905", "E731"] + +[tool.mypy] +files = ["agent", "app", "tests", "streamlit_app.py"] +mypy_path = "tests" +python_version = "3.11" +warn_unused_ignores = true + +# The app's and the agent's code is checked inside functions without annotations too; the tests' only where annotated. +[[tool.mypy.overrides]] +module = ["agent.*", "app.*"] +check_untyped_defs = true + +# agent/sandbox/agent_loop_figures.py runs inside the sandbox image, where matplotlib is; the project never installs it. +[[tool.mypy.overrides]] +module = ["matplotlib.*"] +ignore_missing_imports = true + # The fixed versions after eMudhra's emSign CA certificates were compromised and removed from the CA bundles # (https://discuss.python.org/t/109576): older ones still trust them. uv refuses to run below required-version, # and the constraints keep a lock from ever taking an older certifi, pip or sigstore. diff --git a/streamlit_app.py b/streamlit_app.py index f992d7d..573b495 100644 --- a/streamlit_app.py +++ b/streamlit_app.py @@ -2,9 +2,9 @@ Run with: uv run streamlit run streamlit_app.py -The project has two parts. harness/ runs a turn: the agent loop, the tools it runs, the model's family and what runs -the model; it never imports Streamlit. app/ is what a person sees: the page, the HTML it draws from the harness's -events, and the How it works guide. Streamlit runs this file again on every interaction, and main() draws the page anew. +The project has two parts. agent/ runs a turn: the harness that runs the agent loop, the tools it runs, the model's +family and what runs the model; it never imports Streamlit. app/ is what a person sees: the page, the HTML it draws +from the agent's events, and the How it works guide. Streamlit runs this file again on every interaction, and main() draws the page anew. """ from app.page import main diff --git a/tests/fake_pi.py b/tests/fake_pi.py new file mode 100644 index 0000000..feb9842 --- /dev/null +++ b/tests/fake_pi.py @@ -0,0 +1,363 @@ +"""A scripted stand-in for pi in RPC mode (agent/harnesses/pi_harness.py), run as a child process the way pi is. + +It speaks pi's protocol as pi 1.4 does (docs/rpc.md, docs/json.md): commands on stdin, responses and events on +stdout, one JSON object per line. What its model says comes from the JSON file in FAKE_PI_SCRIPT: + + {"models": [pi model objects], "turns": [turn, ...], "delay": seconds between streamed pieces} + +and each turn is {"stop": pi's stopReason, "error": its errorMessage, "usage": {...}, "say": [item, ...]}, where an item +is {"thinking": text}, {"text": text} or {"tool": name, "args": {...}, "result": text, "fails": bool}. A turn with +"forever": true streams until it is aborted, and with "deaf": true even then. + +Loaded with --extension (pi_extension.ts), it does what the extension does: before each request to its model it +reports the request as a notification, with AGENT_LOOP_APPROVE it asks before each tool, and with AGENT_LOOP_ENDPOINT +it has the provider agent-loop, whose model is the page's own (model_endpoint.py): then each turn really asks that +endpoint over HTTP, as pi's openai-completions provider does, streams back what the model says and runs the tool calls +it gets, each answering what "results" says for its tool ("done" otherwise), until a reply calls no tool. + +What it was started with and every command it read are appended to FAKE_PI_LOG, one JSON per line. FakePi, at the +end, puts it in pi's place for a test. +""" + +import json +import os +import queue +import sys +import threading +import time +import urllib.error +import urllib.request + +ARGS = sys.argv[1:] +LOG = os.environ.get("FAKE_PI_LOG") +SCRIPT = json.loads(open(os.environ["FAKE_PI_SCRIPT"]).read()) if os.environ.get("FAKE_PI_SCRIPT") else {"turns": []} +DELAY = SCRIPT.get("delay", 0.005) +MODELS = SCRIPT.get("models", [{"id": "fake-model", "name": "Fake Model", "provider": "fake", "api": "openai-completions", + "reasoning": False, "input": ["text", "image"], "contextWindow": 32000, "maxTokens": 2048, + "cost": {"input": 1, "output": 2, "cacheRead": 0, "cacheWrite": 0}}]) +ENDPOINT = os.environ.get("AGENT_LOOP_ENDPOINT") +if ENDPOINT: # what pi_extension.ts registers + MODELS = [*MODELS, {"id": os.environ["AGENT_LOOP_MODEL"], "name": os.environ["AGENT_LOOP_MODEL"], "provider": "agent-loop", + "api": "openai-completions", "reasoning": True, "input": os.environ.get("AGENT_LOOP_INPUT", "text").split(","), + "contextWindow": int(os.environ["AGENT_LOOP_CONTEXT"]), "maxTokens": int(os.environ["AGENT_LOOP_MAX_TOKENS"]), + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}}] +SYSTEM = "You are an expert coding assistant operating inside pi, a coding agent harness." +inbox: queue.Queue = queue.Queue() +model: dict | None = MODELS[0] if MODELS else None +thinking = "off" + + +def log(entry: dict): + if LOG: + with open(LOG, "a") as f: + f.write(json.dumps(entry) + "\n") + + +def out(record: dict): + sys.stdout.write(json.dumps(record) + "\n") + sys.stdout.flush() + + +def read_stdin(): + for line in sys.stdin: + command = json.loads(line) + log({"command": command}) + inbox.put(command) + inbox.put(None) + + +class Aborted(Exception): + pass + + +def check(abortable=True): + """Handle the commands that came meanwhile; raise Aborted on an abort.""" + while True: + try: + command = inbox.get_nowait() + except queue.Empty: + return + if command is None: + sys.exit(0) + if command.get("type") == "abort": + out({"id": command.get("id"), "type": "response", "command": "abort", "success": True}) + if abortable and not SCRIPT.get("deaf"): + raise Aborted + else: + answer(command) + + +def answer(command: dict): + global model, thinking + kind, rid = command.get("type"), command.get("id") + ok = {"id": rid, "type": "response", "command": kind, "success": True} + if kind == "get_available_models": + out({**ok, "data": {"models": MODELS}}) + elif kind == "get_state": + out({**ok, "data": {"model": model, "thinkingLevel": "off", "isStreaming": False}}) + elif kind == "set_model": + found = next((m for m in MODELS if m["provider"] == command["provider"] and m["id"] == command["modelId"]), None) + if found: + model = found + out({**ok, "data": found}) + else: + out({**ok, "success": False, "error": f"Model not found: {command['provider']}/{command['modelId']}"}) + elif kind == "prompt": + if SCRIPT.get("refuse"): + out({**ok, "success": False, "error": SCRIPT["refuse"]}) + return + out({**ok, "data": {"disposition": "started"}}) + run(command) + elif kind == "set_thinking_level": + thinking = command["level"] + out(ok) + elif kind == "extension_ui_response": + pass # an answer to a dialog nobody is waiting for any more + else: + out(ok) + + +def wait_for_answer(dialog: str) -> bool: + """Block until the client answers the dialog, as pi's ctx.ui.confirm() does; an abort cancels it.""" + while True: + command = inbox.get() + if command is None: + sys.exit(0) + if command.get("type") == "extension_ui_response" and command.get("id") == dialog: + return bool(command.get("confirmed")) + if command.get("type") == "abort": + out({"id": command.get("id"), "type": "response", "command": "abort", "success": True}) + raise Aborted + answer(command) + + +def assistant(content, stop="pending", usage=None, error=None): + message = {"role": "assistant", "content": content, "api": "openai-completions", "provider": model["provider"], "model": model["id"], + "usage": usage or {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 0, "cost": {"total": 0}}, + "stopReason": stop} + return {**message, "errorMessage": error} if error else message + + +def stream(kind: str, index: int, text: str): + for i in range(0, len(text), 4): + check() + out({"type": "message_update", "assistantMessageEvent": {"type": kind, "contentIndex": index, "delta": text[i:i + 4]}}) + time.sleep(DELAY) + + +def tools() -> list[dict]: + """The tools pi declares to its model, as its openai-completions provider sends them.""" + names = ARGS[ARGS.index("--tools") + 1].split(",") if "--tools" in ARGS else [] + params = {"bash": {"command": {"type": "string"}}, "read": {"path": {"type": "string"}}} + return [{"type": "function", "function": {"name": n, "description": f"pi's {n} tool", + "parameters": {"type": "object", "properties": params.get(n, {"path": {"type": "string"}}), + "required": list(params.get(n, {"path": 0}))}}} for n in names] + + +def report(messages: list[dict]): + """What pi_extension.ts does before each request: the request, to the page.""" + assert model # set_model came first + if "--extension" in ARGS: + payload = {"model": model["id"], "messages": messages, "tools": tools(), "stream": True} + out({"type": "extension_ui_request", "id": f"notify-{time.monotonic_ns()}", "method": "notify", + "message": "agent-loop:request " + json.dumps(payload), "notifyType": "info"}) + + +def run_tool(call: dict, item: dict) -> str: + """Run one tool call: ask first if told to, then answer from the script; return the result the model reads.""" + out({"type": "tool_execution_start", "toolCallId": call["id"], "toolName": call["name"], "args": call["arguments"]}) + allowed = True + if os.environ.get("AGENT_LOOP_APPROVE"): + dialog = f"dialog-{call['id']}" + out({"type": "extension_ui_request", "id": dialog, "method": "confirm", "title": "agent-loop:approve", + "message": json.dumps({"toolCallId": call["id"], "toolName": call["name"], "input": call["arguments"]})}) + allowed = wait_for_answer(dialog) + if allowed: + time.sleep(DELAY) + text = item.get("result", "") + out({"type": "tool_execution_end", "toolCallId": call["id"], "toolName": call["name"], + "result": {"content": [{"type": "text", "text": text}], "details": {}}, "isError": bool(item.get("fails")), "durationMs": 3}) + return text + text = os.environ.get("AGENT_LOOP_REFUSAL", "") + out({"type": "tool_execution_end", "toolCallId": call["id"], "toolName": call["name"], + "result": {"content": [{"type": "text", "text": text}], "details": {}}, "isError": True}) + return text + + +def ask_endpoint(messages: list[dict]) -> dict: + """One turn's request to the page's model, as pi's openai-completions provider makes it; the assistant message the + stream adds up to, streamed on as pi streams it.""" + assert model and ENDPOINT # set_model came first, with the page's model + body = {"model": model["id"], "messages": messages, "tools": tools() or None, "stream": True, "stream_options": {"include_usage": True}, + "max_tokens": model["maxTokens"], "chat_template_kwargs": {"enable_thinking": thinking != "off"}} + request = urllib.request.Request(ENDPOINT + "/chat/completions", json.dumps({k: v for k, v in body.items() if v is not None}).encode(), + {"Content-Type": "application/json", "Authorization": f"Bearer {os.environ['AGENT_LOOP_TOKEN']}"}) + text, reasoning, finish = "", "", None + calls: list[dict] = [] + usage: dict = {} + try: + response = urllib.request.urlopen(request, timeout=30) + except urllib.error.HTTPError as e: # pi's provider ends the message in the error, unless pi was aborted meanwhile + check() + return assistant([], "error", error=f"{e.code} {e.read().decode()}") + with response: + for line in response: + check() + line = line.decode().strip() + if not line.startswith("data: ") or line == "data: [DONE]": + continue + chunk = json.loads(line[6:]) + usage = chunk.get("usage") or usage + for choice in chunk.get("choices", []): + delta, finish = choice.get("delta") or {}, choice.get("finish_reason") or finish + if piece := delta.get("reasoning_content"): + reasoning += piece + out({"type": "message_update", "assistantMessageEvent": {"type": "thinking_delta", "contentIndex": 0, "delta": piece}}) + if piece := delta.get("content"): + text += piece + out({"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "contentIndex": 1, "delta": piece}}) + calls += delta.get("tool_calls") or [] + content = ([{"type": "thinking", "thinking": reasoning}] if reasoning else []) + ([{"type": "text", "text": text}] if text else []) + content += [{"type": "toolCall", "id": c["id"], "name": c["function"]["name"], "arguments": json.loads(c["function"]["arguments"])} for c in calls] + stop = {"tool_calls": "toolUse", "length": "length", "stop": "stop"}.get(finish or "", "error") + usage = {"input": usage.get("prompt_tokens", 0), "output": usage.get("completion_tokens", 0), "cacheRead": 0, "cacheWrite": 0, "cost": {"total": 0}} + return assistant(content, stop, usage, None if stop != "error" else f"Provider finish_reason: {finish}") + + +def run_on_endpoint(prompt: dict): + """A run on the page's model: turns until a reply calls no tool, each a real request to the endpoint.""" + user = [{"type": "text", "text": prompt["message"]}] + [{"type": "image_url", "image_url": {"url": f"data:{i['mimeType']};base64,{i['data']}"}} + for i in prompt.get("images", [])] + messages: list[dict] = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}] + results = SCRIPT.get("results", {}) + for n in range(20): + if n: + check() + out({"type": "turn_start"}) + report(messages) + out({"type": "message_start", "message": assistant([])}) + message = ask_endpoint(messages) + out({"type": "message_end", "message": message}) + calls = [b for b in message["content"] if b["type"] == "toolCall"] + thinking_text = next((b["thinking"] for b in message["content"] if b["type"] == "thinking"), "") + text = next((b["text"] for b in message["content"] if b["type"] == "text"), "") + messages.append({"role": "assistant", "content": text or None, **({"reasoning_content": thinking_text} if thinking_text else {}), + **({"tool_calls": [{"id": c["id"], "type": "function", "function": {"name": c["name"], "arguments": json.dumps(c["arguments"])}} + for c in calls]} if calls else {})}) + for call in calls: + messages.append({"role": "tool", "tool_call_id": call["id"], "content": run_tool(call, {"result": results.get(call["name"], "done")})}) + out({"type": "turn_end", "message": message, "toolResults": [c["id"] for c in calls]}) + if not calls: + return + + +def run(prompt: dict): + out({"type": "agent_start"}) + out({"type": "turn_start"}) + out({"type": "message_start", "message": {"role": "user", "content": [{"type": "text", "text": prompt["message"]}]}}) + out({"type": "message_end", "message": {"role": "user", "content": [{"type": "text", "text": prompt["message"]}]}}) + assert model # set_model came first + aborted, calls = False, 0 + messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt["message"]}] + try: + if model["provider"] == "agent-loop": + run_on_endpoint(prompt) + for n, turn in enumerate(SCRIPT["turns"] if model["provider"] != "agent-loop" else []): + if n: + check() + out({"type": "turn_start"}) + report(messages) + time.sleep(DELAY) + check() + content: list = [] + out({"type": "message_start", "message": assistant([])}) + if turn.get("forever"): + while True: + stream("text_delta", 0, "more ") + for i, item in enumerate(turn.get("say", [])): + if "thinking" in item: + stream("thinking_delta", i, item["thinking"]) + content.append({"type": "thinking", "thinking": item["thinking"]}) + elif "text" in item: + stream("text_delta", i, item["text"]) + content.append({"type": "text", "text": item["text"]}) + else: + calls += 1 + call = {"type": "toolCall", "id": f"call_{calls}", "name": item["tool"], "arguments": item.get("args", {})} + out({"type": "message_update", "assistantMessageEvent": {"type": "toolcall_start", "contentIndex": i, "id": call["id"], "toolName": item["tool"]}}) + stream("toolcall_delta", i, json.dumps(call["arguments"])) + content.append(call) + usage = {"input": 1000, "output": 20, "cacheRead": 200, "cacheWrite": 0, "totalTokens": 1220, "cost": {"total": 0.0021}, **turn.get("usage", {})} + calls_here = [b for b in content if b["type"] == "toolCall"] + message = assistant(content, turn.get("stop", "toolUse" if calls_here else "stop"), usage, turn.get("error")) + out({"type": "message_end", "message": message}) + results = [] + text = "".join(b.get("text", "") for b in content if b["type"] == "text") + messages.append({"role": "assistant", "content": text or None, + **({"tool_calls": [{"id": c["id"], "type": "function", "function": {"name": c["name"], "arguments": json.dumps(c["arguments"])}} + for c in calls_here]} if calls_here else {})}) + for call in calls_here: + item = next(i for i in turn["say"] if i.get("tool") == call["name"]) + messages.append({"role": "tool", "tool_call_id": call["id"], "content": run_tool(call, item)}) + results.append(call["id"]) + out({"type": "turn_end", "message": message, "toolResults": results}) + if message["stopReason"] in ("stop", "length", "error"): + break + except Aborted: + aborted = True + message = assistant([], "aborted") + out({"type": "message_end", "message": message}) + out({"type": "turn_end", "message": message, "toolResults": []}) + out({"type": "agent_end", "messages": [], "willRetry": False}) + out({"type": "agent_settled", "aborted": aborted}) + + +def main(): + log({"args": ARGS, "cwd": os.getcwd(), "refusal": os.environ.get("AGENT_LOOP_REFUSAL"), "approve": os.environ.get("AGENT_LOOP_APPROVE"), + "endpoint": ENDPOINT}) + if ARGS == ["--version"]: + print(SCRIPT.get("version", "1.4.0")) + return + if SCRIPT.get("crash"): + sys.stderr.write(SCRIPT["crash"] + "\n") + sys.exit(1) + threading.Thread(target=read_stdin, daemon=True).start() + while True: + command = inbox.get() + if command is None: + return + try: + answer(command) + except Aborted: + pass + + +class FakePi: + """For a test: this file in pi's place (pi_harness.PI_COMMAND). say() writes what its model says next; log() reads back + what each pi started was given and was sent.""" + + def __init__(self, monkeypatch, folder): + from agent.harnesses import pi_harness # only in the test's process + + self.script, self.log_file = folder / "fake_pi_script.json", folder / "fake_pi.log" + monkeypatch.setattr(pi_harness, "PI_COMMAND", [sys.executable, __file__]) + monkeypatch.setenv("FAKE_PI_SCRIPT", str(self.script)) + monkeypatch.setenv("FAKE_PI_LOG", str(self.log_file)) + self.say() + + def say(self, *turns, **more): + self.script.write_text(json.dumps({"turns": list(turns), **more})) + + def log(self) -> list[dict]: + return [json.loads(line) for line in self.log_file.read_text().splitlines()] if self.log_file.exists() else [] + + def commands(self) -> list[dict]: + return [e["command"] for e in self.log() if "command" in e] + + def started(self) -> dict: + """What the last pi that ran a turn was started with (not a `pi --version`).""" + return [e for e in self.log() if "args" in e and e["args"] != ["--version"]][-1] + + +if __name__ == "__main__": + main() diff --git a/tests/fakes.py b/tests/fakes.py index d9c1511..f72ef74 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -13,9 +13,9 @@ import time from types import SimpleNamespace -from harness import families -from harness.backends.backend import Encoded -from harness.families import GEMMA4, QWEN3_5 +from agent import model_families +from agent.model_backends.backend import Encoded +from agent.model_families import GEMMA4, QWEN3_5 STR = '<|"|>' @@ -27,7 +27,7 @@ def fake_folder(repo: str, sizes: dict[str, int], fields: dict[str, dict]) -> di "files": [{"name": name, "bytes": n, "fields": fields.get(name, {})} for name, n in sizes.items()], } -INFO = { +INFO: dict = { "model_id": "fake/gemma-test", "architecture": "FakeForTests", "device": "cpu", @@ -63,12 +63,12 @@ def fake_folder(repo: str, sizes: dict[str, int], fields: dict[str, dict]) -> di "generation_config.json": {"eos_token_id": [1, 106, 50], "top_k": 64}, }), } -DRAFTER = { +DRAFTER: dict = { "model_id": "fake/gemma-test-assistant", "parameters": 77_700_000, "layers": 4, "hidden_size": 256, "draft_tokens": 6, "load_seconds": 0.1, "folder": fake_folder("fake--gemma-test-assistant", {"config.json": 2315, "model.safetensors": 157_565_344}, {"config.json": {"num_centroids": 2048}}), } -QWEN_INFO = { +QWEN_INFO: dict = { **INFO, "model_id": "fake/qwen-test", "parameters": 853_000_000, @@ -100,11 +100,11 @@ def fake_folder(repo: str, sizes: dict[str, int], fields: dict[str, dict]) -> di } -def family_of(model_id: str) -> families.Family: +def family_of(model_id: str) -> model_families.Family: return QWEN3_5 if "qwen" in model_id.lower() else GEMMA4 -def pieces(text: str, family: families.Family = GEMMA4) -> list[str]: +def pieces(text: str, family: model_families.Family = GEMMA4) -> list[str]: """Control tokens stay whole; everything else becomes one token per character.""" out = [] for part in re.split(f"({family.special_re.pattern})", text): @@ -137,7 +137,7 @@ def value(v): class FakeTokenizer: - def __init__(self, family: families.Family = GEMMA4): + def __init__(self, family: model_families.Family = GEMMA4): self.family = family def __call__(self, text: str, add_special_tokens: bool = False): @@ -145,7 +145,7 @@ def __call__(self, text: str, add_special_tokens: bool = False): class FakeLLM: - def __init__(self, outputs: list[str] | None = None, model_id: str | None = None, family: families.Family | None = None): + def __init__(self, outputs: list[str] | None = None, model_id: str | None = None, family: model_families.Family | None = None): self.family = family or family_of(model_id or "") info = QWEN_INFO if self.family is QWEN3_5 else INFO self.model_id = model_id or info["model_id"] @@ -153,8 +153,8 @@ def __init__(self, outputs: list[str] | None = None, model_id: str | None = None self.outputs = outputs if outputs is not None else [] # shared on purpose: tests append to it self.tokenizer = FakeTokenizer(self.family) self.context_length = info["context_length"] - self.drafter_id = families.DRAFTERS.get(self.model_id) - self.drafter = None + self.drafter_id = model_families.DRAFTERS.get(self.model_id) + self.drafter: object | None = None self.info = {**info, "model_id": self.model_id, "drafter_id": self.drafter_id} self.renders: list[dict] = [] self.encodes: list[list[dict]] = [] # the media handed to each encode() @@ -331,14 +331,14 @@ def qwen_tool_call(name: str, **args) -> str: def record_run(outputs: list, user: str = "台北現在幾點？", history: list | None = None, sampling=None, family=None, attachments=(), files=None, **settings): """Run the harness to the end on the calling thread; return the run, its events and the fake model.""" - from harness.agent import Run, RunSettings - from harness.backends.backend import Sampling + from agent.harnesses.simple_harness import RunSettings, SimpleHarness + from agent.model_backends.backend import Sampling fake = FakeLLM(list(outputs), family=family) if sampling and sampling.mtp: # a model with a drafter, already loaded fake.drafter_id = fake.info["drafter_id"] = DRAFTER["model_id"] fake.load_drafter() events: list[dict] = [] - run = Run(fake, history or [], user, RunSettings(sampling=sampling or Sampling(), **settings), events.append, attachments, files) + run = SimpleHarness(fake, history or [], user, RunSettings(sampling=sampling or Sampling(), **settings), events.append, attachments, files) run.run() return run, events, fake diff --git a/tests/guide_excerpts.txt b/tests/guide_excerpts.txt index fb79d22..3b9a33e 100644 --- a/tests/guide_excerpts.txt +++ b/tests/guide_excerpts.txt @@ -10,42 +10,68 @@ [[app/page.py|self.thread = threading.Thread]] self.thread = threading.Thread(target=self._work, daemon=True) -[[harness/agent.py|def emit(self, event_type]] +[[agent/harnesses/simple_harness.py|def emit(self, event_type]] def emit(self, event_type: str, **data): -[[harness/agent.py|def stop(self):]] +[[agent/harnesses/simple_harness.py|def stop(self):]] def stop(self): -[[harness/agent.py|def decide(self, call_id]] +[[agent/harnesses/simple_harness.py|def decide(self, call_id]] def decide(self, call_id: str, approved: bool): -[[harness/agent.py|def run(self):]] +[[agent/harnesses/harness.py|class Harness(Protocol):]] + class Harness(Protocol): + +[[agent/harnesses/simple_harness.py|def run(self):]] def run(self): -[[harness/backends/backend.py|class Backend(Protocol):]] +[[agent/harnesses/pi_harness.py|class Process:]] + class Process: + +agent/harnesses/pi_harness.py: def _on(self, rec: dict) -> bool: + def _on(self, rec: dict) -> bool: + """Turn one of pi's records into the page's events; True once pi has settled.""" + kind = rec.get("type") + if kind == "turn_start": + self._turn_start() + elif kind == "message_start" and self._open and rec["message"].get("role") == "assistant": + self._stream_t, self._wrote = time.perf_counter(), False + elif kind == "message_update" and self._open: + self._update(rec["assistantMessageEvent"]) + elif kind == "message_end" and self._open and rec["message"].get("role") == "assistant": + self._message_end(rec["message"]) + elif kind == "tool_execution_start" and self._open and not rec.get("parentToolCallId"): + self._tool_start(rec["toolCallId"], rec["toolName"], rec.get("args") or {}) + elif kind == "tool_execution_end" and rec.get("toolCallId") in self._calls and not rec.get("parentToolCallId"): + self._tool_end(rec["toolCallId"], rec.get("result") or {}, rec.get("isError", False), rec.get("durationMs")) + elif kind == "extension_ui_request": + self._dialog(rec) + return kind == "agent_settled" + +[[agent/model_backends/backend.py|class Backend(Protocol):]] class Backend(Protocol): -[[harness/backends/simple_backend.py|def generate(self, prompt]] +[[agent/model_backends/simple_backend.py|def generate(self, prompt]] def generate(self, prompt: Encoded, sampling: Sampling, should_stop: Callable[[], bool]) -> Iterator[dict]: -[[harness/backends/vllm_backend.py|class VLLMBackend:]] +[[agent/model_backends/vllm_backend.py|class VLLMBackend:]] class VLLMBackend: -[[harness/backends/vllm_docker.py|def start(model_id]] +[[agent/model_backends/vllm_docker.py|def start(model_id]] def start(model_id: str, say: Callable[[str, str], None], mtp: bool = False) -> dict: -[[harness/tools.py|TOOLS = {]] +[[agent/tools.py|TOOLS = {]] TOOLS = { -[[harness/i18n.py|def pick(]] - def pick(lang: str, zh: str, en: str) -> str: +[[agent/i18n.py|def pick(]] + def pick(lang: str, zh: T, en: T) -> T: [[app/render.py|def build(events: list[dict])]] def build(events: list[dict]) -> dict: == families == -harness/families/__init__.py: class Family: … def special_re(self) +agent/model_families/__init__.py: class Family: … def special_re(self) class Family: key: str name: str @@ -100,8 +126,8 @@ harness/families/__init__.py: class Family: … def special_re(self) pieces = sorted(self.special_tokens, key=len, reverse=True) return re.compile("|".join(re.escape(p) for p in pieces)) -harness/backends/simple_backend.py: self.family = families.for_model_type( … self.special_ids = set( - self.family = families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) +agent/model_backends/simple_backend.py: self.family = model_families.for_model_type( … self.special_ids = set( + self.family = model_families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) files = self.folder = _download(model_id) # The processor holds the tokenizer and, for a model that reads more than text, what turns files into its inputs. self.processor = AutoProcessor.from_pretrained(files) @@ -119,13 +145,13 @@ harness/backends/simple_backend.py: self.family = families.for_model_type( … s # Control tokens count as special even when the tokenizer does not mark them (Qwen's ). self.special_ids = set(self.tokenizer.all_special_ids) | self.eos_ids | self.token_ids(self.family.special_tokens) -[[harness/families/__init__.py|MODELS = {]] +[[agent/model_families/__init__.py|MODELS = {]] MODELS = { -[[harness/families/__init__.py|FAMILIES = {]] +[[agent/model_families/__init__.py|FAMILIES = {]] FAMILIES = {f.key: f for f in (GEMMA4, QWEN3_5)} -harness/families/qwen3_5.py: QWEN3_5 = Family( +agent/model_families/qwen3_5.py: QWEN3_5 = Family( QWEN3_5 = Family( key="qwen3_5", name="Qwen3.5", @@ -168,16 +194,16 @@ harness/families/qwen3_5.py: QWEN3_5 = Family( == files == -[[harness/backends/simple_backend.py|MODELS_DIR = Path(]] +[[agent/model_backends/simple_backend.py|MODELS_DIR = Path(]] MODELS_DIR = Path(os.environ.get("AGENT_LOOP_MODELS") or ROOT / "models").expanduser().absolute() -[[harness/families/__init__.py|REVISIONS = {]] +[[agent/model_families/__init__.py|REVISIONS = {]] REVISIONS = { -harness/backends/simple_backend.py: def _download( +agent/model_backends/simple_backend.py: def _download( def _download(model_id: str, only: str | None = None) -> str: """The model's folder in MODELS_DIR, after fetching the files it lacks (only `only`, if given), at the revision - families.REVISIONS pins it to. + model_families.REVISIONS pins it to. from_pretrained(local_files_only=True) is no test of the cache: once config.json is cached, a tokenizer whose own files are missing raises a ValueError (Gemma 4) or loads empty (Qwen3.5) rather than an OSError. @@ -185,15 +211,15 @@ harness/backends/simple_backend.py: def _download( can't be reached it returns the cached folder, so a downloaded model still starts offline.""" if os.path.isdir(model_id): # a checkpoint on disk rather than a Hugging Face id return model_id - return snapshot_download(model_id, revision=families.REVISIONS.get(model_id), allow_patterns=only, cache_dir=MODELS_DIR) + return snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), allow_patterns=only, cache_dir=MODELS_DIR) -harness/backends/vllm_docker.py: say("download", "") … snapshot_download(drafter +agent/model_backends/vllm_docker.py: say("download", "") … snapshot_download(drafter say("download", "") - snapshot_download(model_id, revision=families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) + snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), cache_dir=MODELS_DIR) if config and (drafter := config.get("model")): # the server reads models/ offline, its drafter's weights too snapshot_download(drafter, revision=config["revision"], cache_dir=MODELS_DIR) -[[harness/backends/vllm_docker.py|"AGENT_LOOP_MODELS": str(MODELS_DIR)]] +[[agent/model_backends/vllm_docker.py|"AGENT_LOOP_MODELS": str(MODELS_DIR)]] "AGENT_LOOP_MODELS": str(MODELS_DIR), "VLLM_SPECULATIVE_CONFIG": json.dumps(config) if config else "None", docker-compose/compose.yaml: - ${AGENT_LOOP_MODELS:-../models}:/models:ro … HF_HUB_OFFLINE: "1" @@ -210,73 +236,73 @@ docker-compose/compose.yaml: - ${VLLM_MODEL:-Qwen/Qwen3.5-0.8B} … --served-mod # Offline, vLLM names the model by its folder in models/; the page asks for it by its id. - --served-model-name=${VLLM_MODEL:-Qwen/Qwen3.5-0.8B} -[[harness/backends/vllm_backend.py|WEIGHTS = []] +[[agent/model_backends/vllm_backend.py|WEIGHTS = []] WEIGHTS = ["*.safetensors", "*.bin", "*.pt", "*.pth", "*.gguf"] -harness/backends/vllm_backend.py: def _checkpoint_files( +agent/model_backends/vllm_backend.py: def _checkpoint_files( def _checkpoint_files(model_id: str) -> str: - """The model's folder in MODELS_DIR with everything but its weights, at the revision families.REVISIONS pins. + """The model's folder in MODELS_DIR with everything but its weights, at the revision model_families.REVISIONS pins. The server should load the same revision (vllm serve --revision ) so that both read the same tokenizer and chat template.""" if os.path.isdir(model_id): # a checkpoint on disk rather than a Hugging Face id return model_id - return snapshot_download(model_id, revision=families.REVISIONS.get(model_id), cache_dir=MODELS_DIR, ignore_patterns=WEIGHTS) + return snapshot_download(model_id, revision=model_families.REVISIONS.get(model_id), cache_dir=MODELS_DIR, ignore_patterns=WEIGHTS) -[[harness/backends/vllm_backend.py|def family_of(]] - def family_of(model_id: str) -> families.Family | None: +[[agent/model_backends/vllm_backend.py|def family_of(]] + def family_of(model_id: str) -> model_families.Family | None: -[[harness/backends/simple_backend.py|def pick_device()]] +[[agent/model_backends/simple_backend.py|def pick_device()]] def pick_device() -> str: -[[harness/backends/simple_backend.py|self.family = families.for_model_type(]] - self.family = families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) +[[agent/model_backends/simple_backend.py|self.family = model_families.for_model_type(]] + self.family = model_families.for_model_type(AutoConfig.from_pretrained(_download(model_id, "config.json")).model_type) -[[harness/backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]] +[[agent/model_backends/simple_backend.py|self.processor = AutoProcessor.from_pretrained(files)]] self.processor = AutoProcessor.from_pretrained(files) -[[harness/backends/simple_backend.py|def render(self, messages]] +[[agent/model_backends/simple_backend.py|def render(self, messages]] def render(self, messages: list[dict], tools: list[dict] | None, thinking: bool, generation_prompt: bool = True) -> str: -[[harness/backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]] +[[agent/model_backends/simple_backend.py|self.model = AutoModelForMultimodalLM.from_pretrained(]] self.model = AutoModelForMultimodalLM.from_pretrained(files, dtype=self.dtype, device_map=self.device).eval() -[[harness/backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]] +[[agent/model_backends/simple_backend.py|eos = self.model.generation_config.eos_token_id]] eos = self.model.generation_config.eos_token_id -[[harness/backends/simple_backend.py|def _sampling_defaults(self)]] +[[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]] def _sampling_defaults(self) -> dict: -[[harness/backends/vllm_backend.py|self.family = families.for_model_type(config.model_type)]] - self.family = families.for_model_type(config.model_type) +[[agent/model_backends/vllm_backend.py|self.family = model_families.for_model_type(config.model_type)]] + self.family = model_families.for_model_type(config.model_type) -[[harness/backends/vllm_backend.py|self.context_length = card.get(]] +[[agent/model_backends/vllm_backend.py|self.context_length = card.get(]] self.context_length = card.get("max_model_len") or text_cfg.max_position_embeddings # the server's limit (--max-model-len) -[[harness/backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]] +[[agent/model_backends/vllm_backend.py|self.processor = AutoProcessor.from_pretrained(files)]] self.processor = AutoProcessor.from_pretrained(files) -[[harness/backends/vllm_backend.py|def _generation_config(]] +[[agent/model_backends/vllm_backend.py|def _generation_config(]] def _generation_config(folder: str, config) -> GenerationConfig: -[[harness/families/qwen3_5.py|sampling={]] +[[agent/model_families/qwen3_5.py|sampling={]] sampling={"temperature": 1.0, "top_p": 1.0, "top_k": 20}, -[[harness/backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]] +[[agent/model_backends/simple_backend.py|self.draft_tokens = self.drafter.generation_config.num_assistant_tokens]] self.draft_tokens = self.drafter.generation_config.num_assistant_tokens # guesses per forward pass -[[harness/backends/simple_backend.py|def describe_folder(]] +[[agent/model_backends/simple_backend.py|def describe_folder(]] def describe_folder(folder: str, notes: dict) -> dict: -[[harness/families/gemma4.py|file_notes=FILES,]] +[[agent/model_families/gemma4.py|file_notes=FILES,]] file_notes=FILES, -[[harness/families/qwen3_5.py|file_notes=FILES,]] +[[agent/model_families/qwen3_5.py|file_notes=FILES,]] file_notes=FILES, [[app/render.py|elif served and kind == "model.safetensors":]] elif served and kind == "model.safetensors": -harness/families/__init__.py: class FileNote: +agent/model_families/__init__.py: class FileNote: class FileNote: """What one file in a checkpoint's folder holds in this family. The Model tab shows the note next to the file, with the values in this checkpoint of `fields`, dotted paths into a JSON file (text_config.sliding_window).""" @@ -286,11 +312,11 @@ harness/families/__init__.py: class FileNote: == submit == -[[harness/backends/simple_backend.py|def _sampling_defaults(self)]] +[[agent/model_backends/simple_backend.py|def _sampling_defaults(self)]] def _sampling_defaults(self) -> dict: [[app/page.py|key="model_id", on_change=switch_model]] - st.selectbox(tr("模型", "Model"), list(menu), format_func=menu.get, key="model_id", on_change=switch_model, disabled=busy, help=about) + st.selectbox(tr("模型", "Model"), list(menu), format_func=menu.__getitem__, key="model_id", on_change=switch_model, disabled=busy, help=about) app/page.py: @st.cache_resource(max_entries=1 … def release_model(): @st.cache_resource(max_entries=1, show_spinner=False) # one model in memory: switching models frees the last one @@ -306,10 +332,18 @@ app/page.py: @st.cache_resource(max_entries=1 … def release_model(): gc.collect() empty_cache() -[[harness/backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]] +[[agent/harnesses/pi_harness.py|def probe()]] + def probe() -> dict: + +app/page.py: if st.session_state.pi_source == "page": … llm=current_llm()) + if st.session_state.pi_source == "page": + model_picker(model_menu(), busy) + return PiAgent(st.session_state.model_id, found["version"], llm=current_llm()) + +[[agent/model_backends/simple_backend.py|torch._C._cuda_clearCublasWorkspaces()]] torch._C._cuda_clearCublasWorkspaces() -[[harness/backends/simple_backend.py|def load_drafter(self):]] +[[agent/model_backends/simple_backend.py|def load_drafter(self):]] def load_drafter(self): [[app/page.py|if not st.session_state.get("engine_picked"):]] @@ -350,15 +384,15 @@ app/page.py: def current_llm() -> Backend: if not slot.lock.acquire(blocking=False): app/page.py: def start_run( - def start_run(llm: Backend, text: str, uploads: list[tuple[str, bytes]] = ()) -> bool: + def start_run(who: "Backend | PiAgent", text: str, uploads: Sequence[tuple[str, bytes]] = ()) -> bool: """Ask the question in a new run, with `uploads` (file name, contents) saved in the conversation's folder first.""" slot = generation_slot() if not slot.lock.acquire(blocking=False): st.toast(tr("另一個 agent 正在執行（可能在別的分頁），請稍等。", "Another agent is running (maybe in another tab). Please wait.")) return False try: - attachments = [media.save(chat_folder(), name, data, llm.modalities) for name, data in uploads] - run = LiveRun(llm, st.session_state.history, text, attachments, chat_folder(), current_settings(llm), slot) + attachments = [media.save(chat_folder(), name, data, who.modalities) for name, data in uploads] + run = LiveRun(harness_for(who, text, attachments), text, attachments, slot) except Exception: slot.lock.release() raise @@ -379,17 +413,28 @@ app/page.py: def current_settings( lang=s.lang, ) +app/page.py: def harness_for( + def harness_for(who: "Backend | PiAgent", text: str, attachments: list[media.Attachment]) -> Callable[[Callable[[dict], None]], Harness]: + """What runs the turn: pi, or this project's harness on the model in memory, with the sidebar's settings now.""" + history, folder = st.session_state.history, chat_folder() + if isinstance(who, PiAgent): + settings, session = pi_settings(who), st.session_state.chat_id + page = pi_harness.PageModel(who.llm, current_settings(who.llm).sampling) if who.llm else None + return lambda emit: pi_harness.PiHarness(settings, history, text, emit, attachments, folder, session, page) + run_settings = current_settings(who) + return lambda emit: SimpleHarness(who, history, text, run_settings, emit, attachments, folder) + app/page.py: class LiveRun: class LiveRun: - """One user turn: the harness thread and the events it has emitted so far.""" + """One user turn: the harness thread and the events it has emitted so far. `make` makes the harness (simple_harness.SimpleHarness + or pi_harness.PiHarness) given the function it sends its events to.""" - def __init__(self, llm: Backend, history: list[dict], text: str, attachments: list[media.Attachment], files: Path, - settings: RunSettings, slot: Slot): + def __init__(self, make: Callable[[Callable[[dict], None]], Harness], text: str, attachments: list[media.Attachment], slot: Slot): self.text = text self.attachments = attachments self.events: list[dict] = [] self.absorbed = False - self.run = Run(llm, history, text, settings, self.events.append, attachments, files) + self.run = make(self.events.append) self._slot = slot slot.run = self self.thread = threading.Thread(target=self._work, daemon=True) @@ -412,12 +457,12 @@ app/page.py: class LiveRun: == prepare == -[[harness/agent.py|self.messages = [*history,]] +[[agent/harnesses/simple_harness.py|self.messages = [*history,]] self.messages = [*history, {"role": "user", "content": media.user_content(user_text, list(attachments))}] -harness/agent.py: def __init__(self, llm: Backend, history +agent/harnesses/simple_harness.py: def __init__(self, llm: Backend, history def __init__(self, llm: Backend, history: list[dict], user_text: str, settings: RunSettings, emit: Callable[[dict], None], - attachments: list[media.Attachment] = (), files: Path | None = None): + attachments: Sequence[media.Attachment] = (), files: Path | None = None): self.id = uuid.uuid4().hex[:12] self.llm = llm self.settings = settings @@ -431,10 +476,10 @@ harness/agent.py: def __init__(self, llm: Backend, history self._decisions: dict[str, bool] = {} self._decided = threading.Condition() -[[harness/agent.py|SYSTEM_WITH_TOOLS = ]] +[[agent/harnesses/simple_harness.py|SYSTEM_WITH_TOOLS = ]] SYSTEM_WITH_TOOLS = """You are a helpful assistant running inside an agent loop. -harness/agent.py: def run(self): … stop_reason, final = "max_steps", "" +agent/harnesses/simple_harness.py: def run(self): … stop_reason, final = "max_steps", "" def run(self): s = self.settings family = self.llm.family # how this model writes its turns, thinking and tool calls @@ -442,7 +487,7 @@ harness/agent.py: def run(self): … stop_reason, final = "max_steps", "" schemas = {t.name: t.parameters for t in TOOLS.values()} system = {"role": "system", "content": SYSTEM_WITH_TOOLS if tools else SYSTEM_NO_TOOLS} self.emit( - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, settings=_settings_dict(s), tools=list(self.tools), ) @@ -451,16 +496,34 @@ harness/agent.py: def run(self): … stop_reason, final = "max_steps", "" prev_sequence = "" stop_reason, final = "max_steps", "" -harness/tools.py: def schema(self) -> dict: +agent/tools.py: def schema(self) -> dict: def schema(self) -> dict: return { "type": "function", "function": {"name": self.name, "description": self.description, "parameters": self.parameters}, } +agent/harnesses/pi_harness.py: self._pi = Process(self._args() … self._loop() + self._pi = Process(self._args(), self.folder, self._inbox, self._env()) + # Each waits for its answer: pi handles commands concurrently, and a prompt must not go to the model it starts on. + self._pi.request({"type": "set_model", "provider": s.provider, "modelId": s.model}) + self._pi.request({"type": "set_thinking_level", "level": s.thinking}) + # A failed request is retried by pi as a new turn, which would be a step of its own; the page shows the error instead. + self._pi.request({"type": "set_auto_retry", "enabled": False}) + message, images = self._message() + if not self._stopped_meanwhile(): + self._pi.request({"type": "prompt", "message": message, **({"images": images} if images else {})}) + self._loop() + +[[agent/harnesses/pi_harness.py|FLAGS = ]] + FLAGS = ["--mode", "rpc", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files"] + +[[agent/harnesses/pi_harness.py|def _message(self)]] + def _message(self) -> tuple[str, list[dict]]: + == context == -harness/agent.py: for step in range(1, s.max_steps + 1): … roles=_role_counts([system, *self.messages]) +agent/harnesses/simple_harness.py: for step in range(1, s.max_steps + 1): … roles=_role_counts([system, *self.messages]) for step in range(1, s.max_steps + 1): self.emit("step_start", step=step) prompt = self.llm.render([system, *self.messages], tools, s.thinking) @@ -479,7 +542,38 @@ harness/agent.py: for step in range(1, s.max_steps + 1): … roles=_role_counts( roles=_role_counts([system, *self.messages]), ) -harness/backends/simple_backend.py: def render(self, messages +[[agent/harnesses/model_endpoint.py|def conversation(]] + def conversation(body: dict, llm: Backend, folder: Path) -> list[dict]: + +[[agent/harnesses/model_endpoint.py|def seen_before(]] + def seen_before(messages: list[dict]) -> list[dict]: + +agent/harnesses/model_endpoint.py: messages = conversation(body, llm, serving.folder) … emit("context", + messages = conversation(body, llm, serving.folder) + prompt = llm.render(messages, tools, thinking) + encoded = llm.encode(prompt, media.parts(messages)) + except Exception as e: # pi shows it as the request's error + self._json(400, {"error": {"message": f"{type(e).__name__}: {e}"}}) + return + before = seen_before(messages) + # Media sit in user messages, all before what is new; so the new part is the same text, further along (SimpleHarness). + if before: + previous = llm.render(before, tools, thinking, generation_prompt=False) + added_from = len(encoded.text) - len(prompt) + common_prefix(prompt, previous) + else: + added_from = 0 + emit("context", prompt=encoded.text, n_tokens=encoded.n_tokens, added_from=added_from, media_tokens=encoded.media_tokens, + context_length=llm.context_length, roles={role: sum(m["role"] == role for m in messages) for role in dict.fromkeys(m["role"] for m in messages)}) + +[[agent/harnesses/pi_harness.py|def _request(self, payload)]] + def _request(self, payload): + +agent/harnesses/pi_extension.ts: pi.on("before_provider_request" … }); + pi.on("before_provider_request", (event, ctx) => { + ctx.ui.notify(REQUEST_MARK + JSON.stringify(event.payload), "info"); + }); + +agent/model_backends/simple_backend.py: def render(self, messages def render(self, messages: list[dict], tools: list[dict] | None, thinking: bool, generation_prompt: bool = True) -> str: """The chat template turns messages and tool schemas into the one string the model reads. With `generation_prompt` it also opens the model's turn, so the model writes next.""" @@ -487,10 +581,10 @@ harness/backends/simple_backend.py: def render(self, messages messages, tools=tools or None, tokenize=False, add_generation_prompt=generation_prompt, enable_thinking=thinking ) -[[harness/agent.py|encoded = self.llm.encode(]] +[[agent/harnesses/simple_harness.py|encoded = self.llm.encode(]] encoded = self.llm.encode(prompt, media.parts(self.messages)) -harness/agent.py: def common_prefix( +agent/harnesses/simple_harness.py: def common_prefix( def common_prefix(a: str, b: str) -> int: n = min(len(a), len(b)) i = 0 @@ -499,11 +593,14 @@ harness/agent.py: def common_prefix( return i [[app/render.py|def context_html(]] - def context_html(step: dict, family: families.Family, lang: str) -> str: + def context_html(step: dict, family: model_families.Family, lang: str) -> str: + +[[app/render.py|def request_html(]] + def request_html(step: dict) -> str: == media == -harness/backends/simple_backend.py: def _modalities(self) +agent/model_backends/simple_backend.py: def _modalities(self) def _modalities(self) -> tuple[str, ...]: """What the checkpoint reads besides text: each kind needs an encoder in the config and a part of the processor to feed it. It is asked of the checkpoint, not the family, since checkpoints of one family can differ.""" @@ -515,13 +612,13 @@ harness/backends/simple_backend.py: def _modalities(self) } return tuple(kind for kind, ok in reads.items() if ok) -[[harness/backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]] +[[agent/model_backends/vllm_backend.py|self.modalities = tuple(kind for kind in readable if kind not in refused)]] self.modalities = tuple(kind for kind in readable if kind not in refused) [[app/render.py|def refused_html(]] def refused_html(server: dict | None, lang: str) -> str: -harness/backends/vllm_backend.py: def _probe( +agent/model_backends/vllm_backend.py: def _probe( def _probe(self, hi: list[int], kinds: tuple[str, ...]) -> tuple[bool, bool, dict]: """Ask the server, with one-token requests that start from `hi` (a short prompt's ids), whether it has /inference/v1/generate (--enable-scale-out), whether it sends sampling masks (--return-sampling-mask), and which @@ -552,10 +649,10 @@ harness/backends/vllm_backend.py: def _probe( refused[kind] = {"zh": f"伺服器不收：{e.text['en']}", "en": f"the server refused it: {e.text['en']}"} return True, masks, refused -[[harness/media.py|def save(]] +[[agent/media.py|def save(]] def save(folder: Path, name: str, data: bytes, modalities: tuple[str, ...]) -> Attachment: -harness/media.py: def user_content( +agent/media.py: def user_content( def user_content(text: str, attachments: list[Attachment]) -> str | list[dict]: """The user's message: the text alone, or the media first (as their models are trained to read them), then the text, and a note on the files only run_python can open: where they are and how a text file begins. The note is @@ -568,7 +665,7 @@ harness/media.py: def user_content( words = "\n\n".join(t for t in (text, note) if t) return [a.part() for a in attachments if a.kind != "file"] + ([{"type": "text", "text": words}] if words else []) -harness/backends/simple_backend.py: def encode(self, prompt +agent/model_backends/simple_backend.py: def encode(self, prompt def encode(self, prompt: str, parts: list[dict]) -> Encoded: """The prompt as the model reads it. Without media that is the prompt's tokens. With media (`parts`, the conversation's media in the order of their placeholders), the processor replaces each placeholder with the @@ -577,7 +674,7 @@ harness/backends/simple_backend.py: def encode(self, prompt ids = self.tokenizer(prompt, add_special_tokens=False, return_tensors="pt").input_ids return Encoded(prompt, ids, ids.shape[-1]) files = {kind: [p["path"] for p in parts if p["type"] == kind] for kind in media.MEDIA} - inputs = {} + inputs: dict = {} if files["image"]: inputs["images"] = [media.load_image(path) for path in files["image"]] if files["audio"]: @@ -597,10 +694,10 @@ harness/backends/simple_backend.py: def encode(self, prompt text = self.tokenizer.decode(ids[0], skip_special_tokens=False) return Encoded(text, ids, ids.shape[-1], dict(out.to(self.device, dtype=self.dtype)), {k: n for k, n in counts.items() if n}) -[[harness/media.py|def load_video(]] +[[agent/media.py|def load_video(]] def load_video(path: str) -> tuple[np.ndarray, dict]: -harness/backends/simple_backend.py: def _encoded_media( +agent/model_backends/simple_backend.py: def _encoded_media( def _encoded_media(self, inputs: dict) -> dict: """Run the vision encoder over the prompt's images and videos before the prefill, as generate() does, and pass its output in their place. Leaving it to the forward pass breaks videos on Gemma 4: transformers 5.20's @@ -611,17 +708,17 @@ harness/backends/simple_backend.py: def _encoded_media( for kind in ("image", "video"): encoder = getattr(base, f"get_{kind}_features", None) args = {k: inputs[k] for k in inspect.signature(encoder).parameters if k in inputs} if encoder else {} - if any(k.startswith("pixel_values") for k in args): + if encoder and any(k.startswith("pixel_values") for k in args): encoded[kind] = encoder(**args, return_dict=True) if not encoded: return inputs # The pixels go; the rest stays, such as the grid sizes Qwen3.5's positions are computed from. return {**{k: v for k, v in inputs.items() if not k.startswith("pixel_values")}, "mm_encoder_outputs": encoded} -[[harness/backends/vllm_backend.py|def content_part(]] +[[agent/model_backends/vllm_backend.py|def content_part(]] def content_part(part: dict, audio_rate: int) -> dict: -harness/backends/vllm_backend.py: def encode(self, prompt +agent/model_backends/vllm_backend.py: def encode(self, prompt def encode(self, prompt: str, parts: list[dict]) -> Encoded: """The prompt's token ids, which are what the server reads. With media (`parts`, in the order of their placeholders), the server expands each placeholder into the tokens its file takes, so it is asked how first, with @@ -640,7 +737,7 @@ harness/backends/vllm_backend.py: def encode(self, prompt == model == -harness/agent.py: done = None … self.emit("model_end" +agent/harnesses/simple_harness.py: done = None … self.emit("model_end" done = None for ev in self.llm.generate(encoded, s.sampling, self._stop.is_set): kind = ev.pop("type") @@ -648,13 +745,48 @@ harness/agent.py: done = None … self.emit("model_end" done = ev else: self.emit(kind, step=step, **ev) + assert done is not None # generate() always ends with done raw = done.pop("raw") self.emit("model_end", step=step, raw=raw, **done) -[[harness/backends/simple_backend.py|def _forward(]] +[[agent/harnesses/pi_harness.py|def _serving_step(self)]] + def _serving_step(self) -> int | None: + +agent/harnesses/model_endpoint.py: for ev in llm.generate(encoded, serving.sampling + for ev in llm.generate(encoded, serving.sampling, lambda: gone or serving.stopped()): + kind = ev.pop("type") + if kind == "done": + done = ev + continue + emit(kind, **ev) + if kind == "token" and ev["channel"] in ("thought", "content"): + send({"reasoning_content" if ev["channel"] == "thought" else "content": ev["text"]}) + +[[agent/harnesses/pi_harness.py|def _update(self, e: dict)]] + def _update(self, e: dict): + +[[app/render.py|def streamed_html(]] + def streamed_html(step: dict, lang: str) -> str: + +[[agent/harnesses/pi_harness.py|def _message_end(self, message: dict)]] + def _message_end(self, message: dict): + +agent/harnesses/pi_harness.py: def _update(self, e: dict): + def _update(self, e: dict): + kind = e.get("type") + if kind == "text_delta": + self._delta("content", e.get("delta", "")) + elif kind == "thinking_delta": + self._delta("thought", e.get("delta", "")) + elif kind == "toolcall_start": + self._delta("tool_call", ("\n" if self._wrote else "") + e.get("toolName", "") + " ") + elif kind == "toolcall_delta": + self._delta("tool_call", e.get("delta", "")) + +[[agent/model_backends/simple_backend.py|def _forward(]] def _forward(self, ids: torch.Tensor, past, keep: int, drafting: bool = False, **media): -harness/backends/simple_backend.py: def generate(self, prompt … yield {"type": "prefill" +agent/model_backends/simple_backend.py: def generate(self, prompt … yield {"type": "prefill" def generate(self, prompt: Encoded, sampling: Sampling, should_stop: Callable[[], bool]) -> Iterator[dict]: """Yield a prefill event, one event per token, then a done event.""" ids = prompt.ids.to(self.device) @@ -675,15 +807,15 @@ harness/backends/simple_backend.py: def generate(self, prompt … yield {"type": prefill_ms = (time.perf_counter() - start) * 1000 yield {"type": "prefill", "n_tokens": ids.shape[-1], "ms": round(prefill_ms, 1)} -[[harness/families/gemma4.py|channel_switch={]] +[[agent/model_families/gemma4.py|channel_switch={]] channel_switch={"<|channel>": "thought", "": "content", "<|tool_call>": "tool_call", "": "content"}, -[[harness/families/qwen3_5.py|channel_switch={]] +[[agent/model_families/qwen3_5.py|channel_switch={]] channel_switch={"": "thought", "": "content", "": "tool_call", "": "content"}, -harness/backends/simple_backend.py: decode_start = time.perf_counter() … **details, +agent/model_backends/simple_backend.py: decode_start = time.perf_counter() … **details, decode_start = time.perf_counter() - tokens = self._drafted(out, sampling, mtp) if drafting else self._one_at_a_time(out, sampling) + tokens = self._drafted(out, sampling, mtp) if mtp is not None else self._one_at_a_time(out, sampling) for token_id, details in tokens: if should_stop(): finish = "user_stopped" @@ -714,14 +846,14 @@ harness/backends/simple_backend.py: decode_start = time.perf_counter() … **det **details, } -harness/backends/simple_backend.py: if token_id in self.eos_ids: … if len(generated) == sampling.max_new_tokens: +agent/model_backends/simple_backend.py: if token_id in self.eos_ids: … if len(generated) == sampling.max_new_tokens: if token_id in self.eos_ids: finish = "stop_token" break if len(generated) == sampling.max_new_tokens: break -harness/backends/simple_backend.py: def _one_at_a_time( +agent/model_backends/simple_backend.py: def _one_at_a_time( def _one_at_a_time(self, out, s: Sampling) -> Iterator[tuple[int, dict]]: """Plain decoding: sample a token, feed it back with the cache for the next position's logits, repeat.""" while True: @@ -729,7 +861,7 @@ harness/backends/simple_backend.py: def _one_at_a_time( yield token_id, details out = self._forward(torch.tensor([[token_id]], device=self.device), out.past_key_values, 1) -harness/backends/simple_backend.py: decode_s = time.perf_counter() - decode_start … "raw": self.tokenizer.decode(generated +agent/model_backends/simple_backend.py: decode_s = time.perf_counter() - decode_start … "raw": self.tokenizer.decode(generated decode_s = time.perf_counter() - decode_start yield { "type": "done", @@ -742,7 +874,7 @@ harness/backends/simple_backend.py: decode_s = time.perf_counter() - decode_star "raw": self.tokenizer.decode(generated, skip_special_tokens=False), } -harness/backends/vllm_backend.py: def generate(self, prompt … yield {"type": "prefill" +agent/model_backends/vllm_backend.py: def generate(self, prompt … yield {"type": "prefill" def generate(self, prompt: Encoded, sampling: Sampling, should_stop: Callable[[], bool]) -> Iterator[dict]: """Yield the events SimpleBackend.generate yields, from the server's stream: prefill once the first token is back (its time is the time to the first token), one event per token, then done.""" @@ -765,13 +897,13 @@ harness/backends/vllm_backend.py: def generate(self, prompt … yield {"type": " decode_start = time.perf_counter() yield {"type": "prefill", "n_tokens": prompt.n_tokens, "ms": round((decode_start - start) * 1000, 1)} -[[harness/backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]] +[[agent/model_backends/vllm_backend.py|# From here on, as in SimpleBackend.generate.]] # From here on, as in SimpleBackend.generate. -[[harness/backends/vllm_backend.py|tokens.close()]] +[[agent/model_backends/vllm_backend.py|tokens.close()]] tokens.close() # closing the stream early tells the server to stop generating -harness/backends/vllm_backend.py: if finish is None: … "raw": self.tokenizer.decode(generated +agent/model_backends/vllm_backend.py: if finish is None: … "raw": self.tokenizer.decode(generated if finish is None: reason = end.get("finish_reason") if reason not in ("stop", "length"): @@ -792,7 +924,7 @@ harness/backends/vllm_backend.py: if finish is None: … "raw": self.tokenizer.d == sampling == -harness/backends/simple_backend.py: def _sample(self, logits +agent/model_backends/simple_backend.py: def _sample(self, logits def _sample(self, logits: torch.Tensor, s: Sampling, draft: int | None = None) -> tuple[int, dict]: """Pick the next token and report the model's distribution and what the settings did to it. A `draft` (the MTP drafter's guess) is kept with the probability the settings give it, which in greedy @@ -818,7 +950,7 @@ harness/backends/simple_backend.py: def _sample(self, logits kept = int(((cand_probs.cumsum(-1) - cand_probs) < s.top_p).sum()) cand_ids, cand_probs = cand_ids[:kept], cand_probs[:kept] / cand_probs[:kept].sum() weights = cand_probs - at = (cand_ids == draft).nonzero().flatten() if draft is not None else [] + at: torch.Tensor | list = (cand_ids == draft).nonzero().flatten() if draft is not None else [] if len(at) and float(torch.rand(())) < float(cand_probs[at[0]]): pick = int(at[0]) else: @@ -843,7 +975,7 @@ harness/backends/simple_backend.py: def _sample(self, logits ], } -harness/backends/vllm_backend.py: def _sampling_params( +agent/model_backends/vllm_backend.py: def _sampling_params( def _sampling_params(self, prompt: Encoded, s: Sampling) -> dict: """Each sidebar setting, and every other sampling setting set to do nothing, so that a default the server took from the checkpoint (a repetition penalty, say) cannot change the distribution the page shows.""" @@ -859,7 +991,7 @@ harness/backends/vllm_backend.py: def _sampling_params( "stop_token_ids": sorted(self.eos_ids), } -harness/backends/vllm_backend.py: def _details( +agent/model_backends/vllm_backend.py: def _details( def _details(self, token_id: int, logprobs: dict, n: int, s: Sampling) -> dict: """What /v1/completions said about the n-th token of a chunk: its log probability, and those of the 5 most likely tokens, keyed "token_id:". The token's rank is known only when it is one of those 5.""" @@ -874,10 +1006,10 @@ harness/backends/vllm_backend.py: def _details( rank = 1 + sum(v > logprob for _, v in top) if any(i == token_id for i, _ in top) else None return self._token_details(token_id, logprob, listed, rank, None, s) -[[harness/backends/vllm_backend.py|def logprobs_mode(]] +[[agent/model_backends/vllm_backend.py|def logprobs_mode(]] def logprobs_mode(listed: list[float]) -> str: -harness/backends/vllm_backend.py: def _token_details( +agent/model_backends/vllm_backend.py: def _token_details( def _token_details(self, token_id: int, logprob: float, listed: dict[int, float], rank: int | None, mask: list[int] | None, s: Sampling) -> dict: """The details of a token event, from what the server said about the token: its log probability, those of the @@ -885,7 +1017,7 @@ harness/backends/vllm_backend.py: def _token_details( depends on the server's --logprobs-mode. Only the model's own log probabilities are p, logprob and the top 5. Processed ones are the distribution the token was drawn from, so the token's is its sampling probability, and the tokens top-k and top-p cut come back as UNREPORTED. Logits are neither.""" - details = {"rank": rank} if rank else {} + details: dict = {"rank": rank} if rank else {} if self.logprobs_mode == "raw_logprobs": # Equally likely tokens go lowest id first, as in SimpleBackend. ranked = sorted(listed.items(), key=lambda c: (-c[1], c[0])) @@ -902,26 +1034,26 @@ harness/backends/vllm_backend.py: def _token_details( details["kept"] = sum(v > UNREPORTED for v in listed.values()) return self._greedy(details, token_id, s) -[[harness/backends/vllm_backend.py|def _refuses_greedy(]] +[[agent/model_backends/vllm_backend.py|def _refuses_greedy(]] def _refuses_greedy(self, ids: list[int]) -> bool: -[[harness/backends/vllm_backend.py|def _greedy(]] +[[agent/model_backends/vllm_backend.py|def _greedy(]] def _greedy(self, details: dict, token_id: int, s: Sampling) -> dict: [[app/render.py|def heat(p: float)]] def heat(p: float) -> str: -[[harness/backends/backend.py|TOKEN_DETAILS = (]] +[[agent/model_backends/backend.py|TOKEN_DETAILS = (]] TOKEN_DETAILS = ("p", "logprob", "rank", "p_sampled", "kept", "top", "draft") == mtp == -[[harness/families/__init__.py|DRAFTERS = {]] +[[agent/model_families/__init__.py|DRAFTERS = {]] DRAFTERS = { -harness/backends/simple_backend.py: def load_drafter(self): +agent/model_backends/simple_backend.py: def load_drafter(self): def load_drafter(self): - """Load the MTP drafter that goes with this model (families.DRAFTERS), if any, and warm it up.""" + """Load the MTP drafter that goes with this model (model_families.DRAFTERS), if any, and warm it up.""" with self._drafter_lock: if self.drafter is not None or self.drafter_id is None: return @@ -941,14 +1073,14 @@ harness/backends/simple_backend.py: def load_drafter(self): "folder": describe_folder(folder, self.family.drafter_file_notes), } -harness/backends/simple_backend.py: def _draft( +agent/model_backends/simple_backend.py: def _draft( def _draft(self, token_id: int, hidden: torch.Tensor, shared: dict, position: int) -> list[int]: """The drafter's greedy guesses for the tokens after `token_id`. It has no cache of its own: it attends to the model's keys and values, so every guess sits at the position of `token_id`, and what it has guessed so far comes in through its input, the embedding of the last guess next to the drafter's last hidden state.""" embed = self.model.get_input_embeddings() positions = torch.tensor([[position]], device=self.device) - guesses = [] + guesses: list[int] = [] while len(guesses) < self.draft_tokens and token_id not in self.eos_ids: x = torch.cat([embed(torch.tensor([[token_id]], device=self.device)), hidden], -1) out = self.drafter(inputs_embeds=x, position_ids=positions, shared_kv_states=shared) @@ -956,7 +1088,7 @@ harness/backends/simple_backend.py: def _draft( guesses.append(token_id) return guesses -harness/backends/simple_backend.py: def _drafted( +agent/model_backends/simple_backend.py: def _drafted( def _drafted(self, out, s: Sampling, stats: dict) -> Iterator[tuple[int, dict]]: """MTP decoding. After each token the drafter guesses the next few, and one forward pass over the token and the guesses gives the model's distribution at every guessed position. _sample checks the guesses in order: each @@ -990,8 +1122,8 @@ harness/backends/simple_backend.py: def _drafted( shared = {k: (key[:, :, : key.shape[2] - dropped], value[:, :, : value.shape[2] - dropped]) for k, (key, value) in out.shared_kv_states.items()} -harness/backends/simple_backend.py: at = (cand_ids == draft) … pick = int(torch.multinomial(weights, 1)) - at = (cand_ids == draft).nonzero().flatten() if draft is not None else [] +agent/model_backends/simple_backend.py: at: torch.Tensor | list = (cand_ids == draft) … pick = int(torch.multinomial(weights, 1)) + at: torch.Tensor | list = (cand_ids == draft).nonzero().flatten() if draft is not None else [] if len(at) and float(torch.rand(())) < float(cand_probs[at[0]]): pick = int(at[0]) else: @@ -1000,23 +1132,24 @@ harness/backends/simple_backend.py: at = (cand_ids == draft) … pick = int(torc weights[at[0]] = 0 pick = int(torch.multinomial(weights, 1)) -[[harness/families/__init__.py|VLLM_SPECULATIVE = {]] +[[agent/model_families/__init__.py|VLLM_SPECULATIVE = {]] VLLM_SPECULATIVE = { -[[harness/backends/vllm_docker.py|def speculative(model_id]] +[[agent/model_backends/vllm_docker.py|def speculative(model_id]] def speculative(model_id: str, where: str) -> dict | None: [[app/render.py|def with_drafts(]] def with_drafts(tokens: list[dict], mtp: dict) -> list[dict]: -harness/backends/vllm_backend.py: def drafted_tokens( +agent/model_backends/vllm_backend.py: def drafted_tokens( def drafted_tokens(accepted: list[int], drafted: list[int], n_tokens: int) -> dict | None: """Which of a step's tokens were guesses the model kept (kept_at) and which replace a guess it turned down (turned_down_at), from vLLM's per-step counts (--per-request-spec-decode-metrics detailed). The first token comes from the prefill; after it, each verify step gives the guesses it kept and then one token of the model's own, which replaces the next guess when it kept fewer than were drafted. vLLM does not count a step without guesses, so when the steps do not account for every token, nobody can tell which tokens were guessed: None.""" - kept, turned_down, at = [], [], 1 + kept: list[int] = [] + turned_down, at = [], 1 for n_kept, n_drafted in zip(accepted, drafted): kept += range(at, at + n_kept) if n_kept < n_drafted: @@ -1029,19 +1162,19 @@ harness/backends/vllm_backend.py: def drafted_tokens( == backends == -[[harness/backends/backend.py|class Backend(Protocol):]] +[[agent/model_backends/backend.py|class Backend(Protocol):]] class Backend(Protocol): -harness/backends/backend.py: class Backend(Protocol): +agent/model_backends/backend.py: class Backend(Protocol): class Backend(Protocol): """What runs the model. The page holds one at a time (app.page.current_llm) and runs one turn on it at a time.""" model_id: str - family: families.Family # how the model writes its turns, thinking and tool calls (families/) + family: model_families.Family # how the model writes its turns, thinking and tool calls (model_families/) modalities: tuple[str, ...] # what it reads besides text: "image", "audio", "video" context_length: int # the most tokens it reads at once info: dict # what the Model tab shows, and in "defaults" the recommended temperature, top_p and top_k - drafter_id: str | None # the MTP drafter that goes with the model (families.DRAFTERS), if any + drafter_id: str | None # the MTP drafter that goes with the model (model_families.DRAFTERS), if any drafter: object | None # that drafter once load_drafter() has loaded it; Sampling.mtp turns it on only then def warmup(self) -> None: @@ -1073,7 +1206,7 @@ harness/backends/backend.py: class Backend(Protocol): [[app/page.py|llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id)]] llm = vllm_backend.VLLMBackend(model_id, url) if url else SimpleBackend(model_id) -harness/backends/backend.py: TOKEN_FIELDS = ( … TOKEN_DETAILS = ( +agent/model_backends/backend.py: TOKEN_FIELDS = ( … TOKEN_DETAILS = ( TOKEN_FIELDS = ("index", "id", "piece", "text", "special", "channel") # What a token event carries only when the backend knows it: # p, logprob the token's probability in the model's own distribution (temperature 1, nothing cut off), and its log @@ -1091,24 +1224,24 @@ harness/backends/backend.py: TOKEN_FIELDS = ( … TOKEN_DETAILS = ( def known(item: dict, key: str) -> bool: [[app/render.py|def candidates_html(]] - def candidates_html(tok: dict, family: families.Family, lang: str) -> str: + def candidates_html(tok: dict, family: model_families.Family, lang: str) -> str: [[tests/fakes.py|class FakeLLM:]] class FakeLLM: -[[harness/backends/vllm_docker.py|def start(model_id]] +[[agent/model_backends/vllm_docker.py|def start(model_id]] def start(model_id: str, say: Callable[[str, str], None], mtp: bool = False) -> dict: -[[harness/backends/vllm_docker.py|def device()]] +[[agent/model_backends/vllm_docker.py|def device()]] def device() -> str: [[app/page.py|def release_model():]] def release_model(): -[[harness/backends/vllm_docker.py|def stop():]] +[[agent/model_backends/vllm_docker.py|def stop():]] def stop(): -[[harness/backends/vllm_docker.py|def _stop_at_exit():]] +[[agent/model_backends/vllm_docker.py|def _stop_at_exit():]] def _stop_at_exit(): [[app/page.py|def model_menu()]] @@ -1116,16 +1249,33 @@ harness/backends/backend.py: TOKEN_FIELDS = ( … TOKEN_DETAILS = ( == parse == -harness/agent.py: prefix = family.thought_prefix if … self.emit("parsed" +agent/harnesses/simple_harness.py: prefix = family.thought_prefix if … self.emit("parsed" prefix = family.thought_prefix if prompt.endswith(family.thought_prefix) else "" parsed = family.parse(prefix + raw, schemas, s.lang) calls = parsed["tool_calls"] if tools else [] self.emit("parsed", step=step, thinking=parsed["thinking"], content=parsed["content"], tool_calls=calls) -[[harness/families/gemma4.py|CALL_RE = re.compile]] +agent/harnesses/model_endpoint.py: prefix = family.thought_prefix if … send(finish=finish) + prefix = family.thought_prefix if prompt.endswith(family.thought_prefix) else "" + calls = [c for c in family.parse(prefix + raw, schemas, serving.lang)["tool_calls"] if not c["error"]] if tools else [] + if calls: + send({"tool_calls": [{"index": i, "id": f"call_{uuid.uuid4().hex[:8]}", "type": "function", + "function": {"name": c["name"], "arguments": json.dumps(c["arguments"] or {}, ensure_ascii=False)}} + for i, c in enumerate(calls)]}) + finish = "length" if done["finish"] == "max_new_tokens" else "tool_calls" if calls else "stop" + send(finish=finish) + +agent/harnesses/pi_harness.py: blocks = message.get("content") or [] … self.emit("parsed" + blocks = message.get("content") or [] + calls = [{"name": b.get("name", ""), "arguments": b.get("arguments") or {}, "raw_args": json.dumps(b.get("arguments") or {}, ensure_ascii=False), + "error": None} for b in blocks if b.get("type") == "toolCall"] + thinking = "\n\n".join(b["thinking"] for b in blocks if b.get("type") == "thinking" and b.get("thinking")) + self.emit("parsed", step=self._step, thinking=thinking, content=self._text(message), tool_calls=calls) + +[[agent/model_families/gemma4.py|CALL_RE = re.compile]] CALL_RE = re.compile(r"<\|tool_call>call:(\w+)(.*?)", re.S) -harness/families/gemma4.py: def parse_output( +agent/model_families/gemma4.py: def parse_output( def parse_output(raw: str, schemas: dict | None = None, lang: str = "zh") -> dict: """Split one model turn into thinking, tool calls and visible content. Gemma marks each value's type itself, so the tool schemas are not needed. Errors are worded in `lang`.""" @@ -1162,7 +1312,7 @@ harness/families/gemma4.py: def parse_output( rest = rest.replace(marker, "") return {"thinking": thinking, "tool_calls": calls, "content": rest.strip()} -harness/families/gemma4.py: def value(self): +agent/model_families/gemma4.py: def value(self): def value(self): self.skip_ws() if self.text.startswith(STR, self.pos): @@ -1176,7 +1326,7 @@ harness/families/gemma4.py: def value(self): return self.json_string() return self.scalar() -harness/families/qwen3_5.py: def parse_output( +agent/model_families/qwen3_5.py: def parse_output( def parse_output(raw: str, schemas: dict | None = None, lang: str = "zh") -> dict: """Split one model turn into thinking, tool calls and visible content. `schemas` maps a tool name to its parameters schema, which says what type each plain-text value is. Errors are worded in `lang`.""" @@ -1201,7 +1351,7 @@ harness/families/qwen3_5.py: def parse_output( rest = rest.replace(marker, "") return {"thinking": thinking, "tool_calls": calls, "content": rest.strip()} -harness/families/qwen3_5.py: def convert(value: str, kind +agent/model_families/qwen3_5.py: def convert(value: str, kind def convert(value: str, kind: str | None): """Turn a plain-text value into the schema's type. If it does not fit, keep the text and let the schema check report the wrong type to the model.""" @@ -1217,7 +1367,7 @@ harness/families/qwen3_5.py: def convert(value: str, kind == stop == -harness/agent.py: if done["finish"] == "user_stopped": … if not calls: +agent/harnesses/simple_harness.py: if done["finish"] == "user_stopped": … if not calls: if done["finish"] == "user_stopped": stop_reason, final = "user_stopped", parsed["content"] self.messages.append({"role": "assistant", "content": final}) @@ -1228,10 +1378,10 @@ harness/agent.py: if done["finish"] == "user_stopped": … if not calls: self.messages.append({"role": "assistant", "content": final}) break -[[harness/agent.py|stop_reason = "approval_timeout" if self._timed_out]] +[[agent/harnesses/simple_harness.py|stop_reason = "approval_timeout" if self._timed_out]] stop_reason = "approval_timeout" if self._timed_out else "user_stopped" -harness/agent.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" +agent/harnesses/simple_harness.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" if format_errors >= MAX_FORMAT_ERRORS: stop_reason = "format_errors" break @@ -1245,9 +1395,36 @@ harness/agent.py: if format_errors >= MAX_FORMAT_ERRORS: … self.emit("run_end" self.emit("run_end", stop_reason=stop_reason, final=final, messages=self.messages) +agent/harnesses/pi_harness.py: def _turn_start(self): + def _turn_start(self): + if self._aborting: # pi goes on to the turn its tool results start before the abort reaches it + self._open = False + return + if self._step >= self.settings.max_steps: # pi has no step limit of its own; the page's is kept by stopping it + self._abort("max_steps") + self._open = False + return + self._step += 1 + self._open = True + self._turn_t = self._stream_t = time.perf_counter() + self.emit("step_start", step=self._step) + with self._unserved: + self._waiting = self._step + self._unserved.notify_all() + +agent/harnesses/pi_harness.py: def _stop_reason(self) -> str: + def _stop_reason(self) -> str: + if self._aborting: + return self._aborting + if self._error: + return "error" + if self._last and self._last.get("stopReason") == "length": + return "max_new_tokens" + return "no_tool_call" + == record == -harness/agent.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( +agent/harnesses/simple_harness.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( ids = [f"call_{step}_{i}" for i in range(len(calls))] assistant = { "role": "assistant", @@ -1267,7 +1444,7 @@ harness/agent.py: ids = [f"call_{step}_{i}" … prev_sequence = self.llm.render( == check == -harness/agent.py: def _handle_call( … args, errors = tool.validate(args, lang) +agent/harnesses/simple_harness.py: def _handle_call( … args, errors = tool.validate(args, lang) def _handle_call(self, step: int, cid: str, call: dict, seen: list) -> str: """Validate, approve and execute one tool call. Returns the observation text for the model.""" lang = self.settings.lang @@ -1281,7 +1458,7 @@ harness/agent.py: def _handle_call( … args, errors = tool.validate(args, lang) else: args, errors = tool.validate(args, lang) -harness/tools.py: def validate(self, args: dict +agent/tools.py: def validate(self, args: dict def validate(self, args: dict, lang: str = "zh") -> tuple[dict, list[str]]: """Check arguments against the schema; return them with defaults filled in, plus any errors in `lang`.""" props = self.parameters.get("properties", {}) @@ -1299,7 +1476,7 @@ harness/tools.py: def validate(self, args: dict filled.update(args) return filled, errors -harness/agent.py: key = (call["name"], json.dumps … {"error": "; ".join(errors)} +agent/harnesses/simple_harness.py: key = (call["name"], json.dumps … {"error": "; ".join(errors)} key = (call["name"], json.dumps(args, sort_keys=True, ensure_ascii=False)) repeated = key in seen seen.append(key) @@ -1317,7 +1494,7 @@ harness/agent.py: key = (call["name"], json.dumps … {"error": "; ".join(errors if errors: return self._observe(step, cid, call["name"], {"error": "; ".join(errors)}, ok=False, ms=0) -harness/agent.py: if self.settings.approve_tools: … return self._observe(step, cid, call["name"], {"error": error} +agent/harnesses/simple_harness.py: if self.settings.approve_tools: … return self._observe(step, cid, call["name"], {"error": error} if self.settings.approve_tools: self.emit("approval_request", step=step, call_id=cid, name=call["name"], arguments=args) approved = self._wait_for_decision(cid) @@ -1333,10 +1510,10 @@ harness/agent.py: if self.settings.approve_tools: … return self._observe(step, error = pick(lang, "使用者拒絕執行這個工具呼叫", "The user refused to run this tool call") return self._observe(step, cid, call["name"], {"error": error}, ok=False, ms=0) -[[harness/agent.py|APPROVAL_TIMEOUT = ]] +[[agent/harnesses/simple_harness.py|APPROVAL_TIMEOUT = ]] APPROVAL_TIMEOUT = 300 # seconds; an unanswered request (say, from a closed tab) must not hold the model forever -harness/agent.py: def _wait_for_decision( +agent/harnesses/simple_harness.py: def _wait_for_decision( def _wait_for_decision(self, cid: str) -> bool: deadline = time.monotonic() + APPROVAL_TIMEOUT with self._decided: @@ -1349,6 +1526,23 @@ harness/agent.py: def _wait_for_decision( self._decided.wait(timeout=min(left, 0.5)) return self._decisions.get(cid, False) +[[agent/harnesses/pi_harness.py|def _tool_start(self]] + def _tool_start(self, call_id: str, name: str, args: dict): + +[[agent/harnesses/pi_harness.py|def _dialog(self, rec: dict)]] + def _dialog(self, rec: dict): + +[[agent/harnesses/pi_harness.py|def _decided(self]] + def _decided(self, call_id: str, approved: bool): + +agent/harnesses/pi_extension.ts: pi.on("tool_call" … return undefined; + pi.on("tool_call", async (event, ctx) => { + const call = JSON.stringify({ toolCallId: event.toolCallId, toolName: event.toolName, input: event.input }); + if (!(await ctx.ui.confirm(APPROVE_TITLE, call))) { + return { block: true, reason: env.AGENT_LOOP_REFUSAL || "The user refused to run this tool call" }; + } + return undefined; + app/page.py: for call in render.pending_approvals(data): for call in render.pending_approvals(data): with st.container(border=True): @@ -1366,7 +1560,7 @@ app/page.py: for call in render.pending_approvals(data): == execute == -harness/agent.py: self.emit("tool_start" … return self._observe(step, cid, call["name"], result, ok=ok +agent/harnesses/simple_harness.py: self.emit("tool_start" … return self._observe(step, cid, call["name"], result, ok=ok self.emit("tool_start", step=step, call_id=cid, name=call["name"], environment=tool.environment) start = time.perf_counter() try: @@ -1377,16 +1571,16 @@ harness/agent.py: self.emit("tool_start" … return self._observe(step, cid, cal result, ok = {"error": f"{type(e).__name__}: {e}"}, False return self._observe(step, cid, call["name"], result, ok=ok, ms=(time.perf_counter() - start) * 1000) -[[harness/tools.py|def get_current_time(]] +[[agent/tools.py|def get_current_time(]] def get_current_time(timezone: str = "Asia/Taipei") -> dict: -[[harness/tools.py|def search_wikipedia(]] +[[agent/tools.py|def search_wikipedia(]] def search_wikipedia(query: str, lang: str = "en") -> dict: -[[harness/tools.py|def run_python(]] +[[agent/tools.py|def run_python(]] def run_python(code: str, files: Path | None = None) -> dict | ToolResult: -harness/tools.py: def _eval(node): +agent/tools.py: def _eval(node): def _eval(node): if isinstance(node, ast.Expression): return _eval(node.body) @@ -1412,10 +1606,10 @@ harness/tools.py: def _eval(node): return FUNCS[node.func.id](*args) raise ToolError(f"不支援的運算：{ast.unparse(node)}", f"Unsupported operation: {ast.unparse(node)}") -[[harness/tools.py|def sandbox_user(]] +[[agent/tools.py|def sandbox_user(]] def sandbox_user() -> str: -harness/tools.py: def sandbox_command( +agent/tools.py: def sandbox_command( def sandbox_command(name: str, files: Path | None) -> list[str]: """docker run for one piece of code, read from stdin. The container has no network, a read-only file system except a small /tmp, a cap on memory, CPU and processes, no Linux capabilities, the host's own user (never root), @@ -1431,15 +1625,27 @@ harness/tools.py: def sandbox_command( cmd += ["-v", f"{Path(files).resolve()}:/data:ro"] return cmd + [SANDBOX_IMAGE, "python", "-I", "-"] -[[harness/tools.py|def save_figures(]] +[[agent/tools.py|def save_figures(]] def save_figures(charts: str, folder: Path | None) -> list[Path]: -[[harness/tools.py|def sandbox_problem(]] +[[agent/tools.py|def sandbox_problem(]] def sandbox_problem() -> dict[str, str] | None: +agent/harnesses/pi_harness.py: def _tool_start(self + def _tool_start(self, call_id: str, name: str, args: dict): + key = (name, json.dumps(args, sort_keys=True, ensure_ascii=False)) + repeated = key in self._seen + self._seen.append(key) + self._calls[call_id] = {"name": name, "arguments": args, "step": self._step} + environment = environment_of(name) + self.emit("tool_call", step=self._step, call_id=call_id, name=name, arguments=args, raw_args=json.dumps(args, ensure_ascii=False), + errors=[], repeated=repeated, environment=environment) + if not self.settings.approve_tools: # otherwise it starts once the user allows it (_decided) + self.emit("tool_start", step=self._step, call_id=call_id, name=name, environment=environment) + == observe == -harness/agent.py: def _observe( +agent/harnesses/simple_harness.py: def _observe( def _observe(self, step: int, cid: str, name: str, result: dict | ToolResult, ok: bool, ms: float) -> str: # The model reads a ToolResult's data; its images (run_python's charts) are only for the page to show. result, images = (result.data, result.images) if isinstance(result, ToolResult) else (result, []) @@ -1450,7 +1656,7 @@ harness/agent.py: def _observe( self.emit("tool_result", step=step, call_id=cid, name=name, ok=ok, result=text, truncated=truncated, ms=round(ms, 1), images=images) return text -harness/agent.py: for cid, call in zip(ids, calls): … if format_errors >= MAX_FORMAT_ERRORS: +agent/harnesses/simple_harness.py: for cid, call in zip(ids, calls): … if format_errors >= MAX_FORMAT_ERRORS: for cid, call in zip(ids, calls): result = self._handle_call(step, cid, call, seen) if call["error"]: @@ -1468,6 +1674,9 @@ harness/agent.py: for cid, call in zip(ids, calls): … if format_errors >= MAX_ stop_reason = "format_errors" break +[[agent/harnesses/pi_harness.py|def _tool_end(self]] + def _tool_end(self, call_id: str, result: dict, is_error: bool, ms: float | None): + == ui == app/page.py: @st.fragment(run_every= … viz_panel(llm) @@ -1489,17 +1698,18 @@ app/page.py: @st.fragment(run_every= … viz_panel(llm) app/render.py: def build(events: list[dict]) def build(events: list[dict]) -> dict: - data = {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + data: dict = {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} for ev in events: kind = ev["type"] data["last"], data["t"] = kind, ev["t"] if kind == "run_start": - data["model"], data["family"] = ev["model"], families.get(ev["family"]) - data["settings"] = ev["settings"] + # A harness that does not know how the model writes its turns (pi) sends no family. + data["model"], data["family"] = ev["model"], model_families.get(ev["family"]) if ev.get("family") else None + data["harness"], data["settings"] = ev.get("harness", "simple"), ev["settings"] elif kind == "step_start": data["steps"].append({ "n": ev["step"], "t0": ev["t"], "t1": ev["t"], "context": None, "prefill": None, - "tokens": [], "done": None, "parsed": None, "calls": {}, + "tokens": [], "text": [], "request": None, "done": None, "parsed": None, "calls": {}, }) elif kind == "run_end": data["end"] = ev @@ -1510,7 +1720,12 @@ app/render.py: def build(events: list[dict]) step["t1"] = ev["t"] if kind == "token": step["tokens"].append(ev) - elif kind in ("context", "prefill", "parsed"): + elif kind == "delta": # kept as runs of one channel: [{"channel", "text"}] + if step["text"] and step["text"][-1]["channel"] == ev["channel"]: + step["text"][-1] = {**step["text"][-1], "text": step["text"][-1]["text"] + ev["text"]} + else: + step["text"].append({"channel": ev["channel"], "text": ev["text"]}) + elif kind in ("request", "context", "prefill", "parsed"): step[kind] = ev elif kind == "model_end": step["done"] = ev @@ -1537,10 +1752,12 @@ app/render.py: def build(events: list[dict]) app/render.py: state["active"], state["edge"] = { state["active"], state["edge"] = { - "step_start": ("context", "env-context" if step and step["n"] > 1 else None), + # pi sends the request as its turn starts, and says nothing more until the model's output comes back. + "step_start": ("context", "context-model" if data["harness"] == "pi" else "env-context" if step and step["n"] > 1 else None), "context": ("context", "context-model"), "prefill": ("model", "context-model"), "token": ("model", None), + "delta": ("model", None), "model_end": ("harness", "model-harness"), "parsed": ("harness", "model-harness"), "tool_call": ("harness", None), @@ -1562,7 +1779,7 @@ app/page.py: def absorb_finished( st.session_state.history = end["messages"] run.absorbed = True -harness/agent.py: def stop(self): … def decide(self, call_id +agent/harnesses/simple_harness.py: def stop(self): … def decide(self, call_id def stop(self): self._stop.set() with self._decided: @@ -1573,46 +1790,70 @@ harness/agent.py: def stop(self): … def decide(self, call_id self._decisions[call_id] = approved self._decided.notify_all() +[[agent/harnesses/pi_harness.py|def _abort(self, reason: str)]] + def _abort(self, reason: str): + +agent/harnesses/pi_harness.py: def _abort(self, reason: str): + def _abort(self, reason: str): + if self._aborting: + return + self._aborting = reason + for call_id, (dialog, _) in list(self._pending.items()): + timed_out = reason == "approval_timeout" + self.emit("approval", step=self._calls[call_id]["step"], call_id=call_id, approved=False, timed_out=timed_out, stopped=not timed_out) + self.pi.send({"type": "extension_ui_response", "id": dialog, "cancelled": True}) + self._pending.clear() + self.pi.send({"type": "abort"}) + self._kill_at = time.monotonic() + STOP_GRACE + with self._unserved: # a request waiting for its step learns now, not after WAIT_FOR_STEP, that none will open + self._unserved.notify_all() + == events == -[[harness/agent.py|"run_start", run_id=]] - "run_start", run_id=self.id, model=self.llm.model_id, family=family.key, +[[agent/harnesses/simple_harness.py|"run_start", run_id=]] + "run_start", run_id=self.id, harness="simple", model=self.llm.model_id, family=family.key, -[[harness/agent.py|self.emit("step_start"]] +[[agent/harnesses/simple_harness.py|self.emit("step_start"]] self.emit("step_start", step=step) -[[harness/agent.py|"context",]] +[[agent/harnesses/pi_harness.py|self.emit("request"]] + self.emit("request", step=self._step, text=text, added_from=added_from, added_to=added_to, + +[[agent/harnesses/simple_harness.py|"context",]] "context", -[[harness/backends/simple_backend.py|yield {"type": "prefill"]] +[[agent/model_backends/simple_backend.py|yield {"type": "prefill"]] yield {"type": "prefill", "n_tokens": ids.shape[-1], "ms": round(prefill_ms, 1)} -[[harness/backends/simple_backend.py|"type": "token",]] +[[agent/model_backends/simple_backend.py|"type": "token",]] "type": "token", -[[harness/agent.py|self.emit("model_end"]] +[[agent/harnesses/pi_harness.py|self.emit("delta"]] + self.emit("delta", step=self._step, channel=channel, text=text) + +[[agent/harnesses/simple_harness.py|self.emit("model_end"]] self.emit("model_end", step=step, raw=raw, **done) -[[harness/agent.py|self.emit("parsed"]] +[[agent/harnesses/simple_harness.py|self.emit("parsed"]] self.emit("parsed", step=step, thinking=parsed["thinking"], content=parsed["content"], tool_calls=calls) -[[harness/agent.py|"tool_call",]] +[[agent/harnesses/simple_harness.py|"tool_call",]] "tool_call", -[[harness/agent.py|self.emit("approval_request"]] +[[agent/harnesses/simple_harness.py|self.emit("approval_request"]] self.emit("approval_request", step=step, call_id=cid, name=call["name"], arguments=args) -[[harness/agent.py|self.emit("approval",]] +[[agent/harnesses/simple_harness.py|self.emit("approval",]] self.emit("approval", step=step, call_id=cid, approved=approved, timed_out=self._timed_out, stopped=stopped) -[[harness/agent.py|self.emit("tool_start"]] +[[agent/harnesses/simple_harness.py|self.emit("tool_start"]] self.emit("tool_start", step=step, call_id=cid, name=call["name"], environment=tool.environment) -[[harness/agent.py|self.emit("tool_result"]] +[[agent/harnesses/simple_harness.py|self.emit("tool_result"]] self.emit("tool_result", step=step, call_id=cid, name=name, ok=ok, result=text, truncated=truncated, ms=round(ms, 1), images=images) -[[harness/agent.py|self.emit("error"]] +[[agent/harnesses/simple_harness.py|self.emit("error"]] self.emit("error", message=f"{type(e).__name__}: {e}", **texts) -[[harness/agent.py|self.emit("run_end"]] +[[agent/harnesses/simple_harness.py|self.emit("run_end"]] self.emit("run_end", stop_reason=stop_reason, final=final, messages=self.messages) diff --git a/tests/samples.py b/tests/samples.py index e7ade61..dc80bbe 100644 --- a/tests/samples.py +++ b/tests/samples.py @@ -51,6 +51,7 @@ def _audio(path: Path, container_format: str, codec: str): frames = [r for frame in source.decode(audio=0) for r in resampler.resample(frame)] + resampler.resample(None) with av.open(str(path), "w", format=container_format) as out: stream = out.add_stream(codec, rate=rate, layout="mono") + assert isinstance(stream, av.AudioStream) # codec names an audio codec for frame in frames: frame.pts = None for packet in stream.encode(frame): @@ -75,6 +76,7 @@ def ball_frames(n: int = 32, size=(320, 240)): def _video(path: Path, container_format: str, codec: str): with av.open(str(path), "w", format=container_format) as out: stream = out.add_stream(codec, rate=Fraction(4)) + assert isinstance(stream, av.VideoStream) # codec names a video codec stream.width, stream.height, stream.pix_fmt = 320, 240, "yuv420p" for pixels in ball_frames(): for packet in stream.encode(av.VideoFrame.from_ndarray(pixels, format="rgb24")): @@ -104,6 +106,7 @@ def _data(path: Path, ext: str): book = Workbook() sheet = book.active + assert sheet is not None # a new workbook has one sheet sheet.append(columns) for row in rows: sheet.append([row["month"], row["region"], int(row["units"]), int(row["revenue"])]) diff --git a/tests/test_app.py b/tests/test_app.py index 14ff64f..61ad536 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -20,12 +20,14 @@ import streamlit as st from streamlit.testing.v1 import AppTest +from agent import tools +from agent.harnesses import pi_harness, simple_harness +from agent.model_backends import vllm_backend, vllm_docker +from agent.model_backends.backend import Sampling +from agent.model_families import GEMMA4, QWEN3_5 from app import page, render +from fake_pi import FakePi from fakes import MEDIA_TOKENS, FakeLLM, tool_call -from harness import agent, tools -from harness.backends import vllm_backend, vllm_docker -from harness.backends.backend import Sampling -from harness.families import GEMMA4, QWEN3_5 APP = Path(__file__).resolve().parent.parent / "streamlit_app.py" ANSWER = "答案是 42。" @@ -925,7 +927,7 @@ def test_another_tab_can_stop_a_leftover_run(at, new_tab, outputs): def test_unanswered_approval_frees_the_model(at, new_tab, outputs, monkeypatch): - monkeypatch.setattr(agent, "APPROVAL_TIMEOUT", 0.2) + monkeypatch.setattr(simple_harness, "APPROVAL_TIMEOUT", 0.2) start_waiting_for_approval(at, outputs) settle(at) assert any(c.endswith("停止：等待確認逾時") for c in captions(at)) @@ -1201,3 +1203,202 @@ def test_a_new_chat_removes_the_conversations_files(at, outputs): assert folder.exists() click(at, "新對話") assert not folder.exists() and at.session_state["runs"] == [] + + +# ---------- pi ---------- + +TIME = {"tool": "bash", "args": {"command": "date"}, "result": "Mon Oct 26 14:00:00 CST 2026\n"} + + +@pytest.fixture +def fake_pi(monkeypatch, tmp_path): + """tests/fake_pi.py in pi's place, with one model; `say` scripts what its model says.""" + return FakePi(monkeypatch, tmp_path) + + +def harness(at): + return next(b for b in at.button_group if b.key == "harness") + + +def pick_pi(at, source="pi"): + """Pick pi under What runs the loop, asking pi's own models ("pi") or the page's ("page", what it starts on).""" + harness(at).set_value("pi").run() + assert not at.exception, at.exception + if source != "page": + next(b for b in at.button_group if b.key == "pi_source").set_value(source).run() + assert not at.exception, at.exception + return at + + +def test_the_built_in_harness_is_picked_at_first_and_pi_can_be(at): + assert harness(at).value == "simple" and harness(at).options == ["內建 harness", "pi"] + + +def test_with_pi_the_sidebar_has_its_models_and_settings_and_no_decoding(at, fake_pi, models): + pick_pi(at) + assert [s.value for s in at.selectbox if s.key == "pi_model"] == ["fake/fake-model"] + assert {t.label for t in at.toggle} == {"執行工具前先問我"} + assert {s.label for s in at.slider} == {"步數上限"} + assert next(b for b in at.button_group if b.key == "pi_tools").value == list(pi_harness.DEFAULT_TOOLS) + assert any("pi 1.4.0 · `fake/fake-model`" in c for c in captions(at)) + assert 'class="intro"' in page_html(at) and "pi 的 agent 迴圈" in page_html(at) + assert len(models) == 1 # the model loaded before is not loaded again, nor another + + +@pytest.mark.parametrize("source, shows, does_not", [("page", "滑過 token 可以看候選", "沒有機率"), ("pi", "沒有機率", "滑過 token")]) +def test_pis_intro_says_what_shows_of_the_model_pi_asks(at, fake_pi, source, shows, does_not): + pick_pi(at, source) + html = page_html(at) + assert "pi 的 agent 迴圈" in html and "請求" in html and shows in html and does_not not in html + + +def test_a_question_to_pi_draws_its_loop(at, fake_pi): + fake_pi.say({"say": [{"thinking": "Use date."}, TIME]}, {"say": [{"text": "下午兩點。"}]}) + pick_pi(at) + ask(at, "現在幾點？") + assert "下午兩點。" in [m.value for m in at.chat_message[1].markdown] + html = page_html(at) + assert "Step 1 · bash" in html and "STEP 2" in html + assert "API 傳回的已經是 JSON" in html and "這台電腦的 shell" in html and "Mon Oct 26" in html + assert "讀了 1,200 tokens，其中 200 來自 cache" in html + assert "停止：模型沒有再呼叫工具" in html and len(buttons(at, "看這一步的完整脈絡 →")) == 2 # each step's request + assert Path(fake_pi.started()["cwd"]).name == at.session_state["chat_id"] + + +def test_pi_gets_the_sidebars_settings(at, fake_pi): + fake_pi.say({"say": [{"text": "好。"}]}) + pick_pi(at) + next(b for b in at.button_group if b.key == "pi_tools").set_value(["read", "ls"]).run() + at.slider[0].set_value(3).run() + ask(at, "嗨") + args = fake_pi.started()["args"] + assert args[args.index("--tools") + 1] == "read,ls" + assert at.session_state["runs"][0].events[0]["settings"]["max_steps"] == 3 + + +def test_pi_asks_before_running_a_tool_through_the_page(at, fake_pi): + fake_pi.say({"say": [TIME]}, {"say": [{"text": "下午兩點。"}]}) + pick_pi(at) + toggle(at, "執行工具前先問我").set_value(True) + at.run() + submit(at, "現在幾點？") + wait_for_event(at, "approval_request") + assert "**Agent 想執行這個工具，要允許嗎？** \n`bash({\"command\": \"date\"})`" in [m.value for m in at.markdown] + click(at, "允許") + settle(at) + assert "你允許執行" in page_html(at) and "下午兩點。" in [m.value for m in at.markdown] + + +def test_pi_can_be_stopped(at, fake_pi): + fake_pi.say({"forever": True}) + pick_pi(at) + submit(at, "說個沒完") + wait_for_event(at, "delta") + click(at, "停止這次執行") + settle(at) + assert "停止：使用者中斷" in page_html(at) and "網頁請 pi 中止" in page_html(at) + + +def test_with_pi_on_a_provider_the_context_tab_says_what_it_cannot_show_and_the_model_tab_shows_pis_model(at, fake_pi): + fake_pi.say({"say": [{"text": "好。"}]}) + pick_pi(at) + tab(at).set_value("context").run() + assert any("pi 每一步送給模型的請求" in i.value for i in at.info) + ask(at, "嗨") + assert any("pi 和 provider 都不說" in c for c in captions(at)) + tab(at).set_value("model").run() + html = page_html(at) + assert "Fake Model" in html and "fake/fake-model" in html and "32,000 tokens" in html + assert "動作空間：pi 的工具" in html and "沒有沙箱" in html and "特殊 token（" not in html + assert len(at.dataframe) == 1 + + +def test_pi_without_a_model_says_how_to_set_one_up_and_tries_again_when_asked(at, fake_pi): + fake_pi.say(models=[]) + pick_pi(at, "page") + assert not at.error # the page's own model needs no credentials + next(b for b in at.button_group if b.key == "pi_source").set_value("pi").run() + assert any("pi 還沒有可以用的模型" in e.value for e in at.error) + assert harness(at).value == "pi" # and the switch back is still there + fake_pi.say() + click(at, "再試一次") + assert not at.error and [s.value for s in at.selectbox if s.key == "pi_model"] == ["fake/fake-model"] + + +def test_pi_that_is_not_installed_says_how_to_install_it(at, monkeypatch): + monkeypatch.setattr(pi_harness, "PI_COMMAND", ["no-such-pi-anywhere"]) + pick_pi(at, "page") + assert any(pi_harness.INSTALL in e.value for e in at.error) + + +def test_with_pi_docker_is_never_looked_for(at, fake_pi, sandbox): + pick_pi(at) + fake_pi.say({"say": [{"text": "好。"}]}) + ask(at, "嗨") + assert sandbox["looked"] == 0 + assert ".csv" in at.chat_input[0].proto.file_type # pi has bash, so a data file is of use to it + + +def test_back_from_pi_the_built_in_harness_keeps_its_model_and_settings(at, fake_pi, models): + next(s for s in at.slider if s.label == "temperature").set_value(0.3).run() + pick_pi(at, "pi") + harness(at).set_value("simple").run() + assert next(s for s in at.slider if s.label == "temperature").value == 0.3 + assert at.session_state["model_id"] == "google/gemma-4-E2B-it" and len(models) == 1 + + +def test_switching_what_runs_the_loop_starts_a_new_conversation(at, fake_pi, outputs): + outputs.append(ANSWER) + ask(at) + pick_pi(at) + assert at.session_state["runs"] == [] and at.session_state["history"] == [] + + +@pytest.mark.parametrize("source", ["pi", "page"]) +def test_an_english_page_with_pi_has_no_chinese_left(browser, new_tab, fake_pi, outputs, source): + browser["locale"] = "en-US" + at = new_tab() + fake_pi.say({"say": [{"thinking": "Use date."}, TIME]}, {"say": [{"text": "It is 2 pm."}]}, results={"bash": TIME["result"]}) + outputs += [tool_call("bash", command="date"), "It is 2 pm."] + pick_pi(at, source) + ask(at, "What time is it?") + for view in ("loop", "context", "model"): + tab(at).set_value(view).run() + text = "\n".join([drawn_text(at), *(e.label for e in at.expander)]) + assert not CJK.search(text), CJK.search(text).string[max(CJK.search(text).start() - 60, 0):CJK.search(text).end() + 20] + + +# ---------- pi asking the page's own model ---------- + +def test_pi_starts_on_the_pages_model_with_its_menu_and_decoding(at, fake_pi, models): + pick_pi(at, "page") + assert next(b for b in at.button_group if b.key == "pi_source").value == "page" + assert at.session_state["model_id"] == "google/gemma-4-E2B-it" and len(models) == 1 # the model already in memory + assert {s.label for s in at.slider} >= {"temperature", "top-p", "top-k", "步數上限", "每步 token 上限"} + assert {t.label for t in at.toggle} == {"思考模式", "執行工具前先問我", "MTP 推測解碼"} + assert any(c.startswith("pi 1.4.0 · `google/gemma-4-E2B-it` · Gemma 4") for c in captions(at)) + + +def test_a_question_to_pi_on_the_pages_model_draws_every_token(at, fake_pi, models, outputs): + fake_pi.say(results={"bash": TIME["result"]}) + outputs += [tool_call("bash", command="date"), "下午兩點。"] + pick_pi(at, "page") + next(s for s in at.slider if s.label == "temperature").set_value(0.0).run() + ask(at, "現在幾點？") + assert "下午兩點。" in [m.value for m in at.chat_message[1].markdown] + html = page_html(at) + assert "Step 1 · bash" in html and 'class="tok ' in html and "模型端點從 Gemma 4 格式解析成 JSON 交給 pi" in html + assert "operating inside pi" in models[0].prompts[0] and models[0].samplings[0].temperature == 0.0 + assert len(buttons(at, "看這一步的完整脈絡 →")) == 2 + tab(at).set_value("context").run() + assert "operating inside pi" in page_html(at) and any(e.label.startswith("pi 送出的請求（JSON）") for e in at.expander) + tab(at).set_value("model").run() + assert "特殊 token（24 個）" in page_html(at) and "pi 問的是這個網頁的模型" in page_html(at) + + +def test_pi_on_a_provider_shows_its_request_in_the_context_tab(at, fake_pi): + fake_pi.say({"say": [TIME]}, {"say": [{"text": "下午兩點。"}]}) + pick_pi(at) + ask(at, "現在幾點？") + tab(at).set_value("context").run() + assert ""role": "tool"" in page_html(at) and '' in page_html(at) diff --git a/tests/test_chat_template.py b/tests/test_chat_template.py index 69b0724..6e5a117 100644 --- a/tests/test_chat_template.py +++ b/tests/test_chat_template.py @@ -10,13 +10,13 @@ import pytest from huggingface_hub import try_to_load_from_cache +from agent import media, model_families +from agent.harnesses.simple_harness import SYSTEM_NO_TOOLS, SYSTEM_WITH_TOOLS, RunSettings, SimpleHarness +from agent.model_backends import simple_backend +from agent.model_backends.simple_backend import MODELS_DIR, SimpleBackend +from agent.tools import TOOLS from app import render from fakes import FakeLLM, gemma_call, qwen_call -from harness import families, media -from harness.agent import SYSTEM_NO_TOOLS, SYSTEM_WITH_TOOLS, Run, RunSettings -from harness.backends import simple_backend -from harness.backends.simple_backend import MODELS_DIR, SimpleBackend -from harness.tools import TOOLS from html_check import assert_balanced SAMPLES = Path(__file__).resolve().parent.parent / "app" / "assets" / "examples" @@ -28,14 +28,14 @@ OBSERVATION = {"gemma4": "response:{name}{{value:", "qwen3_5": "<|im_start|>user\n\n"} -@pytest.fixture(scope="module", params=list(families.MODELS)) +@pytest.fixture(scope="module", params=list(model_families.MODELS)) def model(request): """A SimpleBackend with the real tokenizer and family but no weights.""" from transformers import AutoConfig, AutoTokenizer # Look for the files themselves: with only config.json cached, local_files_only=True fails on Gemma's # tokenizer with a ValueError and loads Qwen's empty, rather than raising an OSError. - revision = families.REVISIONS.get(request.param) + revision = model_families.REVISIONS.get(request.param) files = [try_to_load_from_cache(request.param, f, cache_dir=MODELS_DIR, revision=revision) for f in ("config.json", "tokenizer.json", "tokenizer_config.json", "chat_template.jinja")] if not all(isinstance(f, str) for f in files): pytest.skip(f"{request.param} at {revision} is not downloaded into {MODELS_DIR}") @@ -43,7 +43,7 @@ def model(request): m = SimpleBackend.__new__(SimpleBackend) m.tokenizer = AutoTokenizer.from_pretrained(folder) m.config = AutoConfig.from_pretrained(folder) - m.family = families.for_model_type(m.config.model_type) + m.family = model_families.for_model_type(m.config.model_type) m.folder, m.model_id = folder, request.param return m @@ -78,7 +78,7 @@ def call_text(family, name, args): def after_call(name, args, result='{"ok": true}'): - """The messages the harness has after one tool call, built the way agent.Run builds them.""" + """The messages the harness has after one tool call, built the way simple_harness.SimpleHarness builds them.""" return [ system(), {"role": "user", "content": "問題"}, @@ -91,9 +91,9 @@ def after_call(name, args, result='{"ok": true}'): def test_every_file_in_the_model_folder_is_explained_and_has_the_fields_its_note_names(model): folders = [(model.folder, model.family.file_notes)] - if model.model_id in families.DRAFTERS: - drafter = families.DRAFTERS[model.model_id] - config = try_to_load_from_cache(drafter, "config.json", cache_dir=MODELS_DIR, revision=families.REVISIONS.get(drafter)) + if model.model_id in model_families.DRAFTERS: + drafter = model_families.DRAFTERS[model.model_id] + config = try_to_load_from_cache(drafter, "config.json", cache_dir=MODELS_DIR, revision=model_families.REVISIONS.get(drafter)) if isinstance(config, str): folders.append((os.path.dirname(config), model.family.drafter_file_notes)) for folder, notes in folders: @@ -239,7 +239,7 @@ def test_step_card_marks_only_the_observation_as_new(model, monkeypatch, before, end = "好。" + next(iter(family.stop_tokens)) events = [] llm = TemplateLLM(model, [scripted(family, before, args, thinking), end]) - Run(llm, [], "ReAct 是誰提出的？", RunSettings(thinking=thinking), events.append).run() + SimpleHarness(llm, [], "ReAct 是誰提出的？", RunSettings(thinking=thinking), events.append).run() [parsed, _] = [e for e in events if e["type"] == "parsed"] assert [c["arguments"] for c in parsed["tool_calls"]] == [args] first, second = (e for e in events if e["type"] == "context") @@ -266,7 +266,7 @@ def test_with_media_the_step_card_still_marks_only_the_observation_as_new(with_p (("receipt.png", "image"), ("ball.mp4", "video"))] events = [] llm = MediaTemplateLLM(with_processor, [scripted(family, "", {"query": "ReAct"}, False), "好。" + next(iter(family.stop_tokens))]) - Run(llm, [], "這是什麼？", RunSettings(), events.append, files).run() + SimpleHarness(llm, [], "這是什麼？", RunSettings(), events.append, files).run() first, second = (e for e in events if e["type"] == "context") assert first["media_tokens"]["image"] > 100 and first["media_tokens"]["video"] > 100 assert first["n_tokens"] == len(with_processor.tokenizer(first["prompt"], add_special_tokens=False).input_ids) diff --git a/tests/test_gemma4.py b/tests/test_gemma4.py index 04749cf..9d62649 100644 --- a/tests/test_gemma4.py +++ b/tests/test_gemma4.py @@ -1,6 +1,6 @@ import pytest -from harness.families.gemma4 import ParseError, parse_args, parse_output +from agent.model_families.gemma4 import ParseError, parse_args, parse_output S = '<|"|>' diff --git a/tests/test_guide.py b/tests/test_guide.py index 429c668..2bf99b0 100644 --- a/tests/test_guide.py +++ b/tests/test_guide.py @@ -16,9 +16,9 @@ import pytest +from agent.i18n import LANGS from app import guide from app.guide import Code, Tabs, span, statement_end -from harness.i18n import LANGS from html_check import assert_balanced CODES = [item for sec in guide.SECTIONS for item in guide.parts(sec.body) if isinstance(item, Code)] @@ -142,10 +142,10 @@ def test_the_snapshot_follows_the_quoted_code_not_its_line_numbers(tmp_path, mon shutil.copy(guide.ROOT / file, tmp_path / file) monkeypatch.setattr(guide, "ROOT", tmp_path) before = quoted_sections() - agent_py = tmp_path / "harness/agent.py" + agent_py = tmp_path / "agent/harnesses/simple_harness.py" agent_py.write_text("# a line that moves every other one down\n" + agent_py.read_text(encoding="utf-8"), encoding="utf-8") assert quoted_sections() == before - code = next(code for code in CODES if code.file == "harness/agent.py") + code = next(code for code in CODES if code.file == "agent/harnesses/simple_harness.py") lines = agent_py.read_text(encoding="utf-8").splitlines() lines[span(code)[0] - 1] += " # edited" agent_py.write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -169,7 +169,7 @@ def test_event_table_points_at_the_line_that_sends_it(kind, file, text, _carries def test_event_table_covers_every_event_the_harness_sends(): events = set() - for file in ("harness/agent.py", "harness/backends/simple_backend.py"): + for file in ("agent/harnesses/simple_harness.py", "agent/model_backends/simple_backend.py", "agent/harnesses/pi_harness.py"): for node in ast.walk(guide.source(file)[1]): if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "emit" and isinstance(node.args[0], ast.Constant): events.add(node.args[0].value) @@ -234,10 +234,10 @@ def test_every_file_and_line_on_the_page_links_to_it_on_github(lang): assert len(chips) == html.count(' 50 # every chip, in the text and in the event table excerpts = re.findall(rf'
    ([\w./-]+)[^<]+{link}', html) assert len(excerpts) == html.count('
    ') == len(CODES) - for file, line, linked, lo, hi, title, label in chips: + for file, line, linked, lo, hi, _title, label in chips: assert (linked, lo, hi) == (file, line, "") assert file in label and line in label - for code, (file, linked, lo, hi, title, label) in zip(CODES, excerpts): + for code, (file, linked, lo, hi, _title, _label) in zip(CODES, excerpts): assert file == linked == code.file and (int(lo), int(hi or lo)) == span(code) for *_, title, label in chips + excerpts: assert "GitHub" in title and "GitHub" in label @@ -330,20 +330,20 @@ def test_the_root_page_sends_a_reader_to_their_language(tmp_path): def test_prose_markup_is_escaped(): - html = guide.prose_html("a `x` **粗** [[harness/agent.py|def run(self):]] {{stop}}", "zh", "0123456789abcdef") + html = guide.prose_html("a `x` **粗** [[agent/harnesses/simple_harness.py|def run(self):]] {{stop}}", "zh", "0123456789abcdef") assert html.startswith("a <b> x<y> 粗 ") - line = guide.find_line("harness/agent.py", "def run(self):") - assert f'>harness/agent.py:{line}agent/harnesses/simple_harness.py:{line} list[tuple[int, str]]: + """Each module the file imports, with its line: `import a.b` and `from a.b import c` both name a.b. A relative + import stays inside agent/, so it is left out.""" + found = [] + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"), str(path))): + if isinstance(node, ast.Import): + found += [(node.lineno, alias.name) for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.append((node.lineno, node.module)) + return found + + +def test_the_check_sees_every_way_of_importing(tmp_path): + module = tmp_path / "module.py" + module.write_text("import streamlit as st\nfrom app import render\nfrom app.page import main\n" + "def later():\n import app.guide\nfrom . import tools\nimport application\n") + assert sorted(name for _, name in imported(module)) == ["app", "app.guide", "app.page", "application", "streamlit"] + assert sorted(name for _, name in imported(module) if name.split(".")[0] in APP_ONLY) == ["app", "app.guide", "app.page", "streamlit"] + + +@pytest.mark.parametrize("path", AGENT_FILES, ids=lambda p: str(p.relative_to(ROOT))) +def test_the_agent_never_imports_the_app_or_streamlit(path): + found = [f"{path.relative_to(ROOT)}:{line} imports {name}" for line, name in imported(path) if name.split(".")[0] in APP_ONLY] + assert not found, found + + +def test_every_module_of_the_agent_is_read(): + assert {p.relative_to(ROOT).parts[1] for p in AGENT_FILES} >= {"harnesses", "model_backends", "model_families", "tools.py", "media.py"} diff --git a/tests/test_media.py b/tests/test_media.py index 96d589d..2062eb0 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -10,8 +10,8 @@ from PIL import Image import samples -from harness import media -from harness.media import Attachment +from agent import media +from agent.media import Attachment GEMMA = ("image", "audio", "video") QWEN = ("image", "video") @@ -106,7 +106,7 @@ def test_the_sample_recording_decodes(): assert 5.5 < len(samples) / 16000 < 6.2 -def write_video(path: Path, frames: int, size=(64, 48), fps=10): +def write_video(path: Path, frames: int, size: tuple[int, int] = (64, 48), fps=10): with av.open(str(path), "w") as container: stream = container.add_stream("libx264", rate=Fraction(fps)) stream.width, stream.height, stream.pix_fmt = *size, "yuv420p" diff --git a/tests/test_model_endpoint.py b/tests/test_model_endpoint.py new file mode 100644 index 0000000..ec10544 --- /dev/null +++ b/tests/test_model_endpoint.py @@ -0,0 +1,207 @@ +"""harnesses/model_endpoint.py: the page's model served to pi as an OpenAI-compatible chat completions endpoint. + +The requests here are what pi's openai-completions provider sends (pi 1.4, as pi_extension.ts registers the +endpoint), made over HTTP on 127.0.0.1 to the endpoint with a FakeLLM behind it. +""" + +import base64 +import json +import threading +import urllib.error +import urllib.request + +import pytest + +from agent.harnesses import model_endpoint +from agent.harnesses.model_endpoint import conversation, endpoint +from agent.model_backends.backend import Sampling +from agent.model_families import GEMMA4, QWEN3_5 +from fakes import FakeLLM, qwen_tool_call, tool_call + +SYSTEM = {"role": "system", "content": "You are an expert coding assistant operating inside pi."} +BASH = {"type": "function", "function": {"name": "bash", "description": "Run a command", + "parameters": {"type": "object", "properties": {"command": {"type": "string"}}, "required": ["command"]}}} + + +@pytest.fixture +def served(tmp_path): + """The endpoint serving a FakeLLM for a turn whose events are `events`, every request drawn in step 1.""" + state = {"events": [], "stopped": False, "step": 1} + + def serve(outputs, family=None, sampling=None): + llm = FakeLLM(list(outputs), family=family) + endpoint().serve(model_endpoint.Serving(llm, sampling or Sampling(), "en", tmp_path, lambda: state["step"], + lambda kind, **data: state["events"].append({"type": kind, **data}), lambda: state["stopped"])) + return llm + + state["serve"] = serve + yield state + endpoint().stop_serving() + + +def post(body: dict, token: str | None = None) -> tuple[int, list[dict] | dict]: + """POST a chat completion; the stream's chunks, or the error's JSON.""" + request = urllib.request.Request(endpoint().url + "/chat/completions", json.dumps(body).encode(), + {"Content-Type": "application/json", "Authorization": f"Bearer {token or endpoint().token}"}) + try: + with urllib.request.urlopen(request, timeout=10) as response: + lines = [line.decode().strip() for line in response if line.strip()] # each event ends with a blank line + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read()) + return 200, [json.loads(line[6:]) for line in lines if line.startswith("data: {")] + ([{"done": True}] if lines[-1] == "data: [DONE]" else []) + + +def ask(messages, tools=(BASH,), thinking=False) -> dict: + return {"model": "m", "messages": messages, "tools": list(tools) or None, "stream": True, "stream_options": {"include_usage": True}, + "max_tokens": 512, "chat_template_kwargs": {"enable_thinking": thinking}} + + +def deltas(chunks, key): + return "".join(c["choices"][0]["delta"].get(key) or "" for c in chunks if c.get("choices")) + + +def finish_reason(chunks): + return next(c["choices"][0]["finish_reason"] for c in chunks if c.get("choices") and c["choices"][0]["finish_reason"]) + + +# ---------- who it answers ---------- + +def test_it_listens_on_this_machine_alone(): + assert endpoint().url.startswith("http://127.0.0.1:") and endpoint().url.endswith("/v1") + + +def test_it_answers_only_with_its_token(served): + served["serve"](["Hi."]) + status, error = post(ask([SYSTEM, {"role": "user", "content": "hi"}]), token="wrong") + assert status == 401 and "Agent Loop Visualizer" in error["error"]["message"] + + +def test_it_answers_no_request_while_no_turn_is_served(): + endpoint().stop_serving() + status, error = post(ask([SYSTEM, {"role": "user", "content": "hi"}])) + assert status == 503 + + +# ---------- what it sends back ---------- + +def test_a_reply_streams_as_content_with_its_usage(served): + served["serve"](["It is 2 pm."]) + status, chunks = post(ask([SYSTEM, {"role": "user", "content": "What time is it?"}])) + assert status == 200 and chunks[0]["choices"][0]["delta"] == {"role": "assistant"} + assert deltas(chunks, "content") == "It is 2 pm." and finish_reason(chunks) == "stop" + assert chunks[-1] == {"done": True} # [DONE] + usage, context, done = chunks[-2]["usage"], served["events"][0], served["events"][-1] + assert usage["completion_tokens"] == done["n_tokens"] and usage["prompt_tokens"] == context["n_tokens"] + + +def test_a_tool_call_in_the_models_format_goes_back_as_tool_calls(served): + served["serve"]([tool_call("bash", command="date")]) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "Time?"}])) + [call] = next(c["choices"][0]["delta"]["tool_calls"] for c in chunks if c.get("choices") and c["choices"][0]["delta"].get("tool_calls")) + assert call["function"] == {"name": "bash", "arguments": '{"command": "date"}'} and call["id"].startswith("call_") + assert finish_reason(chunks) == "tool_calls" and deltas(chunks, "content") == "" # the call's markup is not the reply + + +def test_qwens_tool_call_too(served): + served["serve"]([qwen_tool_call("bash", command="date")], family=QWEN3_5) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "Time?"}])) + assert finish_reason(chunks) == "tool_calls" + + +def test_thinking_streams_as_reasoning_content(served): + llm = served["serve"](["Use the clock.Two."]) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "Time?"}], thinking=True)) + assert llm.renders[0]["thinking"] is True + assert deltas(chunks, "reasoning_content") == "Use the clock." and deltas(chunks, "content") == "Two." + + +def test_a_call_the_family_cannot_parse_is_not_sent(served): + served["serve"](["<|tool_call>call:bash{command:<|tool_call|>"]) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "Time?"}])) + assert not any(c.get("choices") and c["choices"][0]["delta"].get("tool_calls") for c in chunks) + + +def test_the_sidebars_token_limit_is_length(served): + served["serve"](["A long reply that goes on."], sampling=Sampling(max_new_tokens=3)) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "Talk."}])) + assert finish_reason(chunks) == "length" + + +def test_a_backend_that_fails_says_so_in_the_finish_reason(served): + served["serve"]([RuntimeError("the vLLM server went away")]) + _, chunks = post(ask([SYSTEM, {"role": "user", "content": "hi"}])) + assert finish_reason(chunks) == "error: RuntimeError: the vLLM server went away" + + +# ---------- what the page draws ---------- + +def test_the_request_is_drawn_as_the_steps_context_prefill_tokens_and_end(served): + served["serve"](["It is 2 pm."]) + post(ask([SYSTEM, {"role": "user", "content": "What time is it?"}])) + events = served["events"] + kinds = [e["type"] for e in events] + assert kinds[:2] == ["context", "prefill"] and kinds[-1] == "model_end" and set(kinds[2:-1]) == {"token"} + assert all(e["step"] == 1 for e in events) + context = events[0] + assert "operating inside pi" in context["prompt"] and context["roles"] == {"system": 1, "user": 1} and context["added_from"] == 0 + assert events[-1]["raw"] == "It is 2 pm." and events[-1]["stop_piece"] == "" + + +def test_what_is_new_in_a_step_is_the_tool_result(served): + served["serve"](["It is 2 pm."]) + call = {"id": "call_1", "type": "function", "function": {"name": "bash", "arguments": '{"command":"date"}'}} + post(ask([SYSTEM, {"role": "user", "content": "Time?"}, {"role": "assistant", "content": None, "tool_calls": [call]}, + {"role": "tool", "tool_call_id": "call_1", "content": "14:00"}])) + context = served["events"][0] + new = context["prompt"][context["added_from"]:] + assert "14:00" in new and "Time?" not in new + + +def test_a_request_outside_every_step_is_answered_but_not_drawn(served): + served["serve"](["A summary."]) + served["step"] = None + status, chunks = post(ask([SYSTEM, {"role": "user", "content": "Summarize."}])) + assert status == 200 and deltas(chunks, "content") == "A summary." and served["events"] == [] + + +def test_a_stopped_turn_stops_the_model_before_its_next_token(served): + served["serve"](["It is 2 pm."]) + served["stopped"] = True + post(ask([SYSTEM, {"role": "user", "content": "Time?"}])) + assert served["events"][-1]["finish"] == "user_stopped" and not [e for e in served["events"] if e["type"] == "token"] + + +# ---------- the conversation, as the chat templates take it ---------- + +def test_the_conversation_is_the_one_simple_harness_keeps(tmp_path): + call = {"id": "call_1", "type": "function", "function": {"name": "bash", "arguments": '{"command":"date"}'}} + body = ask([SYSTEM, {"role": "system", "content": "More rules."}, {"role": "user", "content": [{"type": "text", "text": "Time?"}]}, + {"role": "assistant", "content": "Let me see.", "tool_calls": [call], "reasoning_content": "Use date."}, + {"role": "tool", "tool_call_id": "call_1", "content": "14:00"}]) + gemma, qwen = (conversation(body, FakeLLM(family=f), tmp_path) for f in (GEMMA4, QWEN3_5)) + assert gemma[0] == {"role": "system", "content": SYSTEM["content"] + "\n\nMore rules."} + assert gemma[1] == {"role": "user", "content": "Time?"} + assert gemma[2]["tool_calls"] == [{"id": "call_1", "type": "function", "function": {"name": "bash", "arguments": {"command": "date"}}}] + assert gemma[2]["content"] == ("Let me see." if GEMMA4.text_with_calls else "") + assert gemma[2][GEMMA4.reasoning_key] == "Use date." and qwen[2][QWEN3_5.reasoning_key] == "Use date." + assert gemma[3] == {"role": "tool", "tool_call_id": "call_1", "name": "bash", "content": "14:00"} + + +def test_an_image_is_saved_once_under_its_hash_and_becomes_a_part(tmp_path): + url = "data:image/png;base64," + base64.b64encode(b"\x89PNG fake").decode() + body = ask([{"role": "user", "content": [{"type": "text", "text": "What is this?"}, {"type": "image_url", "image_url": {"url": url}}]}]) + first = conversation(body, FakeLLM(), tmp_path)[0]["content"] + second = conversation(body, FakeLLM(), tmp_path)[0]["content"] + assert first == second and first[0] == {"type": "text", "text": "What is this?"} + assert first[1]["type"] == "image" and first[1]["size"] == 9 and len(list((tmp_path / "pi-images").iterdir())) == 1 + + +def test_one_request_at_a_time(served): + served["serve"](["One.", "Two."]) + results = [] + threads = [threading.Thread(target=lambda: results.append(post(ask([SYSTEM, {"role": "user", "content": "hi"}])))) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(10) + assert sorted(deltas(chunks, "content") for _, chunks in results) == ["One.", "Two."] diff --git a/tests/test_families.py b/tests/test_model_families.py similarity index 73% rename from tests/test_families.py rename to tests/test_model_families.py index 1a37701..66093fe 100644 --- a/tests/test_families.py +++ b/tests/test_model_families.py @@ -4,21 +4,21 @@ import pytest -from harness import families -from harness.families import GEMMA4, QWEN3_5 -from harness.i18n import LANGS +from agent import model_families +from agent.i18n import LANGS +from agent.model_families import GEMMA4, QWEN3_5 CJK = re.compile(r"[\u3000-\u303f\u4e00-\u9fff\uff00-\uffef]") def test_model_type_picks_the_family(): - assert families.for_model_type("gemma4") is GEMMA4 - assert families.for_model_type("qwen3_5") is QWEN3_5 + assert model_families.for_model_type("gemma4") is GEMMA4 + assert model_families.for_model_type("qwen3_5") is QWEN3_5 with pytest.raises(ValueError, match="'llama' are not supported; supported: gemma4, qwen3_5"): - families.for_model_type("llama") + model_families.for_model_type("llama") -@pytest.mark.parametrize("family", families.FAMILIES.values(), ids=list(families.FAMILIES)) +@pytest.mark.parametrize("family", model_families.FAMILIES.values(), ids=list(model_families.FAMILIES)) def test_a_family_explains_every_token_it_relies_on(family): """Stop and channel tokens are drawn as chips with a note, so each needs one.""" for piece in (*family.stop_tokens, *family.channel_switch, family.user_turn.split("\n")[0].removesuffix("user")): @@ -27,7 +27,7 @@ def test_a_family_explains_every_token_it_relies_on(family): assert family.thought_prefix.startswith(next(p for p, c in family.channel_switch.items() if c == "thought")) -@pytest.mark.parametrize("family", families.FAMILIES.values(), ids=list(families.FAMILIES)) +@pytest.mark.parametrize("family", model_families.FAMILIES.values(), ids=list(model_families.FAMILIES)) def test_special_re_prefers_the_longest_token(family): for piece in family.special_tokens: assert family.special_re.match(piece).group(0) == piece @@ -49,22 +49,22 @@ def test_a_byte_that_is_only_part_of_a_character_shows_in_hex(): def test_every_model_and_drafter_downloads_at_a_pinned_commit(): - assert set(families.REVISIONS) == set(families.MODELS) | set(families.DRAFTERS.values()) - assert all(re.fullmatch(r"[0-9a-f]{40}", sha) for sha in families.REVISIONS.values()) + assert set(model_families.REVISIONS) == set(model_families.MODELS) | set(model_families.DRAFTERS.values()) + assert all(re.fullmatch(r"[0-9a-f]{40}", sha) for sha in model_families.REVISIONS.values()) def test_vllm_drafts_a_listed_model_with_the_mtp_layer_in_its_checkpoint_or_the_drafter_transformers_uses(): - assert set(families.VLLM_SPECULATIVE) <= set(families.MODELS) - for model_id, config in families.VLLM_SPECULATIVE.items(): + assert set(model_families.VLLM_SPECULATIVE) <= set(model_families.MODELS) + for model_id, config in model_families.VLLM_SPECULATIVE.items(): assert config["num_speculative_tokens"] > 0 - assert config.get("model") == families.DRAFTERS.get(model_id) or config == {"method": "mtp", "num_speculative_tokens": config["num_speculative_tokens"]} + assert config.get("model") == model_families.DRAFTERS.get(model_id) or config == {"method": "mtp", "num_speculative_tokens": config["num_speculative_tokens"]} def test_the_menu_lists_one_model_of_each_family(): - assert len(families.MODELS) == len(families.FAMILIES) + assert len(model_families.MODELS) == len(model_families.FAMILIES) -@pytest.mark.parametrize("family", families.FAMILIES.values(), ids=list(families.FAMILIES)) +@pytest.mark.parametrize("family", model_families.FAMILIES.values(), ids=list(model_families.FAMILIES)) def test_a_family_explains_itself_in_both_languages(family): """The model tab lists the same tokens in the same groups whichever language the page is in.""" groups = {lang: [[piece for piece, _, _ in rows] for _, _, rows in family.special_groups[lang]] for lang in LANGS} @@ -78,7 +78,7 @@ def test_a_family_explains_itself_in_both_languages(family): assert not [text for text in english if CJK.search(text)] -@pytest.mark.parametrize("family", list(families.FAMILIES.values()), ids=lambda f: f.key) +@pytest.mark.parametrize("family", list(model_families.FAMILIES.values()), ids=lambda f: f.key) def test_a_family_explains_its_files_in_both_languages(family): """A note on a file names the same fields and tokens in both languages, and the fields it shows are dotted names.""" for name, note in {**family.file_notes, **family.drafter_file_notes}.items(): diff --git a/tests/test_pi_harness.py b/tests/test_pi_harness.py new file mode 100644 index 0000000..c5b2fd8 --- /dev/null +++ b/tests/test_pi_harness.py @@ -0,0 +1,585 @@ +"""harnesses/pi_harness.py: the page's turn run by pi, through pi's RPC mode. + +Most tests start tests/fake_pi.py in pi's place, which speaks pi 1.4's protocol from a script, so they need neither +Node nor a model. With RUN_PI_TESTS=1 and pi installed (or PI_COMMAND set), the last ones start the real pi, pointed at +a scripted OpenAI-compatible server on 127.0.0.1, so they touch no network either. +""" + +import base64 +import json +import os +import shutil +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from agent.harnesses import model_endpoint, pi_harness +from agent.harnesses.harness import ALWAYS, SOMETIMES, Harness +from agent.harnesses.simple_harness import RunSettings, SimpleHarness +from agent.media import Attachment +from agent.model_backends.backend import Sampling +from agent.model_families import QWEN3_5 +from fake_pi import FakePi +from fakes import FakeLLM, qwen_tool_call, tool_call + +TIME = {"tool": "bash", "args": {"command": "date"}, "result": "Mon Oct 26 14:00:00 CST 2026\n"} +ANSWER = {"text": "It is 2 pm."} + + +@pytest.fixture +def fake(monkeypatch, tmp_path): + """pi is tests/fake_pi.py, saying what the test scripts; the conversation's folder is `folder`.""" + fake = FakePi(monkeypatch, tmp_path) + fake.folder = tmp_path / "chat" + return fake + + +def start(fake, settings=None, text="What time is it?", attachments=()): + """Run a turn in a thread, as the page does; return the run, its events and the thread.""" + events = [] + run = pi_harness.PiHarness(settings or pi_harness.PiSettings("fake", "fake-model", lang="en"), [], text, events.append, attachments, fake.folder, "chat1") + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + return run, events, thread + + +def finish(run, events, thread): + thread.join(20) + assert not thread.is_alive() + assert events[-1]["type"] == "run_end" + return events + + +def run_pi(fake, *turns, settings=None, **kw): + fake.say(*turns) + return finish(*start(fake, settings, **kw)) + + +def wait_for(events, kind, timeout=10): + deadline = time.monotonic() + timeout + while not any(e["type"] == kind for e in events): + assert time.monotonic() < deadline, f"no {kind}" + time.sleep(0.01) + return next(e for e in events if e["type"] == kind) + + +def of(events, kind): + return [e for e in events if e["type"] == kind] + + +def end(events): + return events[-1] + + +# ---------- the interface ---------- + +def test_both_harnesses_are_a_harness(fake): + assert isinstance(pi_harness.PiHarness(pi_harness.PiSettings("fake", "fake-model"), [], "hi", print), Harness) + assert isinstance(SimpleHarness(object(), [], "hi", RunSettings(), print), Harness) + + +def test_every_event_pi_sends_is_one_the_interface_names(fake): + events = run_pi(fake, {"say": [{"thinking": "Use date."}, TIME]}, {"say": [ANSWER]}) + assert {e["type"] for e in events} <= set(ALWAYS) | set(SOMETIMES) + assert [e["type"] for e in events][:2] == ["run_start", "step_start"] + + +# ---------- one turn ---------- + +def test_a_turn_becomes_the_pages_events(fake): + events = run_pi(fake, {"say": [{"thinking": "Use date."}, {"text": "Let me check."}, TIME]}, {"say": [ANSWER]}) + first = events[0] + assert first["type"] == "run_start" and first["harness"] == "pi" and first["family"] is None + assert first["model"] == "fake/fake-model" and first["tools"] == list(pi_harness.DEFAULT_TOOLS) + assert [e["step"] for e in of(events, "step_start")] == [1, 2] + streamed = {} + for e in of(events, "delta"): + if e["step"] == 1: + streamed[e["channel"]] = streamed.get(e["channel"], "") + e["text"] + assert streamed == {"thought": "Use date.", "content": "Let me check.", "tool_call": '\nbash {"command": "date"}'} + done = of(events, "model_end")[0] + assert done["finish"] == "end_turn" and done["n_tokens"] == 20 and done["stop_piece"] is None + assert done["usage"] == {"prompt": 1200, "cached": 200, "output": 20, "cost": 0.0021} # the prompt counts what came from the cache + parsed = of(events, "parsed")[0] + assert parsed["thinking"] == "Use date." and parsed["content"] == "Let me check." + assert parsed["tool_calls"] == [{"name": "bash", "arguments": {"command": "date"}, "raw_args": '{"command": "date"}', "error": None}] + call = of(events, "tool_call")[0] + assert call["call_id"] == "call_1" and call["environment"] == pi_harness.TOOLS["bash"][1] and not call["errors"] and not call["repeated"] + assert of(events, "tool_start")[0]["call_id"] == "call_1" + result = of(events, "tool_result")[0] + assert result["ok"] and result["result"] == TIME["result"] and result["ms"] == 3 and result["step"] == 1 + assert end(events)["stop_reason"] == "no_tool_call" and end(events)["final"] == "It is 2 pm." + + +def test_the_order_of_a_step_follows_the_loop(fake): + kinds = [e["type"] for e in run_pi(fake, {"say": [TIME]}, {"say": [ANSWER]}) if e["type"] != "delta"] + assert kinds == ["run_start", "step_start", "request", "model_end", "parsed", "tool_call", "tool_start", "tool_result", + "step_start", "request", "model_end", "parsed", "run_end"] + + +def test_pi_starts_in_the_conversations_folder_on_its_session_with_the_pages_tools_only(fake): + run_pi(fake, {"say": [ANSWER]}, settings=pi_harness.PiSettings("fake", "fake-model", tools=("read", "ls"), lang="en")) + started = fake.started() + assert Path(started["cwd"]).resolve() == fake.folder.resolve() + args = started["args"] + assert args[:len(pi_harness.FLAGS)] == pi_harness.FLAGS and "--no-extensions" in args and "--no-context-files" in args + assert args[args.index("--session-id") + 1] == "chat1" + assert Path(args[args.index("--session-dir") + 1]) == fake.folder / pi_harness.SESSION_DIR + assert args[args.index("--tools") + 1] == "read,ls" and args[args.index("--extension") + 1] == str(pi_harness.EXTENSION) + assert started["approve"] is None and started["endpoint"] is None # pi asks before nothing, and asks its own model + + +def test_the_model_is_set_and_retries_turned_off_before_the_question_goes(fake): + run_pi(fake, {"say": [ANSWER]}, settings=pi_harness.PiSettings("fake", "fake-model", thinking="high")) + kinds = [c["type"] for c in fake.commands()] + assert kinds[:4] == ["set_model", "set_thinking_level", "set_auto_retry", "prompt"] + commands = fake.commands() + assert commands[0]["provider"] == "fake" and commands[0]["modelId"] == "fake-model" + assert commands[1]["level"] == "high" and commands[2]["enabled"] is False + + +def test_no_tools_means_none_at_all(fake): + events = run_pi(fake, {"say": [ANSWER]}, settings=pi_harness.PiSettings("fake", "fake-model", tools=())) + assert "--no-tools" in fake.started()["args"] and events[0]["settings"]["use_tools"] is False + + +def test_attachments_are_named_for_the_model_and_images_sent_inline(fake, tmp_path): + photo, data = tmp_path / "photo.png", tmp_path / "sales.csv" + photo.write_bytes(b"\x89PNG fake") + data.write_text("month,revenue\n") + attachments = [Attachment("photo.png", "image", str(photo), 9), Attachment("sales.csv", "file", str(data), 14)] + run_pi(fake, {"say": [ANSWER]}, attachments=attachments, text="What is this?") + prompt = next(c for c in fake.commands() if c["type"] == "prompt") + assert prompt["message"] == "What is this?\n\nAttached, in the current folder: photo.png (9 B), sales.csv (14 B)" + assert prompt["images"] == [{"type": "image", "data": base64.b64encode(b"\x89PNG fake").decode(), "mimeType": "image/png"}] + + +def test_a_repeated_call_is_marked(fake): + events = run_pi(fake, {"say": [TIME]}, {"say": [TIME]}, {"say": [ANSWER]}) + assert [c["repeated"] for c in of(events, "tool_call")] == [False, True] + + +def test_a_failed_tool_is_a_failed_result(fake): + events = run_pi(fake, {"say": [{**TIME, "result": "command not found", "fails": True}]}, {"say": [ANSWER]}) + result = of(events, "tool_result")[0] + assert not result["ok"] and result["result"] == "command not found" + + +def test_a_reply_cut_at_the_token_limit_says_so(fake): + events = run_pi(fake, {"say": [{"text": "A long"}], "stop": "length"}) + assert of(events, "model_end")[0]["finish"] == "max_new_tokens" and end(events)["stop_reason"] == "max_new_tokens" + + +def test_the_next_question_continues_the_same_session(fake): + run_pi(fake, {"say": [ANSWER]}) + run_pi(fake, {"say": [ANSWER]}, text="And now?") + sessions = [e["args"][e["args"].index("--session-id") + 1] for e in fake.log() if "--session-id" in e.get("args", [])] + assert sessions == ["chat1", "chat1"] + + +# ---------- what pi sends its model ---------- + +def test_each_step_shows_the_request_pi_sends_and_marks_what_is_new(fake): + events = run_pi(fake, {"say": [TIME]}, {"say": [ANSWER]}) + first, second = of(events, "request") + assert [first["step"], second["step"]] == [1, 2] + assert json.loads(first["text"])["messages"][0]["content"].startswith("You are an expert coding assistant operating inside pi") + assert first["added_from"] is None and first["roles"] == {"system": 1, "user": 1} + new = second["text"][second["added_from"]:second["added_to"]] + assert '"role": "assistant"' in new and '"role": "tool"' in new and TIME["result"].strip() in new and '"role": "user"' not in new + assert second["roles"] == {"system": 1, "user": 1, "assistant": 1, "tool": 1} + + +def test_a_request_shows_no_image_bytes(): + assert pi_harness.without_media({"url": "data:image/png;base64," + "A" * 40, "data": "QUJD" * 300, "text": "hi"}) == { + "url": "data:image/png;base64,…(40 characters)", "data": "…(1,200 base64 characters)", "text": "hi"} + + +# ---------- pi asking the page's own model ---------- + +def on_page_model(fake, outputs, family=None, text="What time is it?", attachments=(), hold=False, **settings): + """Start a turn of pi asking a FakeLLM through model_endpoint, as the page does with pi picked on the page's model; + with `hold`, the model waits for llm.gate before it generates.""" + fake.say(results={"bash": TIME["result"]}) + llm = FakeLLM(list(outputs), family=family) + llm.gate = threading.Event() if hold else None + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings(model_endpoint.PROVIDER, llm.model_id, **{"lang": "en", **settings}), [], text, + events.append, attachments, fake.folder, "chat1", pi_harness.PageModel(llm, Sampling())) + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + return llm, run, events, thread + + +def test_pi_asks_the_pages_model_and_every_token_is_drawn(fake): + llm, *started = on_page_model(fake, [tool_call("bash", command="date"), "It is 2 pm."]) + events = finish(*started) + assert events[0]["family"] == "gemma4" and events[0]["model"] == llm.model_id and events[0]["settings"]["page_model"] + assert fake.started()["endpoint"] == model_endpoint.endpoint().url + kinds = [e["type"] for e in events if e["type"] != "token"] + assert kinds == ["run_start", "step_start", "request", "context", "prefill", "model_end", "parsed", "tool_call", "tool_start", "tool_result", + "step_start", "request", "context", "prefill", "model_end", "parsed", "run_end"] + assert not of(events, "delta") # the tokens say it all + assert {e["step"] for e in of(events, "token")} == {1, 2} + first, second = of(events, "context") + assert "operating inside pi" in first["prompt"] and first["step"] == 1 and second["step"] == 2 + assert TIME["result"] in second["prompt"][second["added_from"]:] and "What time is it?" not in second["prompt"][second["added_from"]:] + done = of(events, "model_end")[0] + assert done["stop_piece"] == "<|tool_response>" and done["raw"].startswith("<|tool_call>") + assert of(events, "tool_call")[0]["arguments"] == {"command": "date"} and of(events, "tool_result")[0]["result"] == TIME["result"] + assert llm.renders[1]["messages"][-1] == {"role": "tool", "tool_call_id": of(events, "tool_call")[0]["call_id"], "name": "bash", "content": TIME["result"]} + assert end(events)["stop_reason"] == "no_tool_call" and end(events)["final"] == "It is 2 pm." + + +def test_pi_asks_qwen_too(fake): + llm, *started = on_page_model(fake, [qwen_tool_call("bash", command="date"), "Two.<|im_end|>"], family=QWEN3_5) + events = finish(*started) + assert events[0]["family"] == "qwen3_5" and of(events, "tool_call")[0]["arguments"] == {"command": "date"} + assert end(events)["final"] == "Two." + + +@pytest.mark.parametrize("level, on", [("off", False), ("medium", True)]) +def test_pis_thinking_level_is_the_pages_thinking_mode(fake, level, on): + llm, *started = on_page_model(fake, ["好的It is 2 pm." if on else "It is 2 pm."], thinking=level) + events = finish(*started) + assert llm.renders[0]["thinking"] is on + assert end(events)["final"] == "It is 2 pm." + + +def test_an_image_reaches_the_pages_model(fake, tmp_path): + photo = tmp_path / "photo.png" + photo.write_bytes(b"\x89PNG fake") + llm, *started = on_page_model(fake, ["A receipt."], attachments=[Attachment("photo.png", "image", str(photo), 9)]) + events = finish(*started) + [part] = llm.encodes[0] + assert part["type"] == "image" and Path(part["path"]).read_bytes() == b"\x89PNG fake" + assert of(events, "context")[0]["media_tokens"] == {"image": 6} + + +def test_stop_stops_the_pages_model_before_its_next_token(fake): + llm, run, events, thread = on_page_model(fake, ["It is 2 pm."], hold=True) + wait_for(events, "context") + run.stop() + deadline = time.monotonic() + 5 + while run._aborting is None and time.monotonic() < deadline: + time.sleep(0.01) + llm.gate.set() + finish(run, events, thread) + assert end(events)["stop_reason"] == "user_stopped" and of(events, "model_end")[0]["finish"] == "user_stopped" + assert not of(events, "token") + + +def late_turn_starts(monkeypatch): + """Make the page read each turn's start after the first 0.3 s late, so the request pi sends for it gets to the + endpoint first and waits for its step.""" + turn_start = pi_harness.PiHarness._turn_start + + def late(self): + if self._step: + time.sleep(0.3) + turn_start(self) + + monkeypatch.setattr(pi_harness.PiHarness, "_turn_start", late) + + +def assert_the_endpoint_is_free(): + """The endpoint serves the next question's first request at once: no request of the turn that ended still holds the model.""" + lock = model_endpoint.endpoint()._lock + assert lock.acquire(timeout=2) + lock.release() + + +def test_the_step_limit_on_the_pages_model_does_not_keep_the_request_past_it_waiting(fake, monkeypatch): + # The abort has to tell the request of the turn past the limit that its step will not open, or it waits WAIT_FOR_STEP holding the model. + late_turn_starts(monkeypatch) + llm, *started = on_page_model(fake, [tool_call("bash", command="date"), "It is 2 pm."], max_steps=1) + began = time.monotonic() + events = finish(*started) + assert time.monotonic() - began < model_endpoint.WAIT_FOR_STEP / 2 + assert_the_endpoint_is_free() + assert [e["step"] for e in of(events, "step_start")] == [1] and end(events)["stop_reason"] == "max_steps" + assert len(llm.renders) == 1 and len(llm.prompts) == 1 # the request past the limit is turned away, not run undrawn + + +@pytest.mark.parametrize("approved", [True, False]) +def test_ask_first_holds_a_call_of_the_pages_model_and_the_model_reads_the_answer(fake, approved): + llm, run, events, thread = on_page_model(fake, [tool_call("bash", command="date"), "It is 2 pm."], approve_tools=True, lang="zh") + request = wait_for(events, "approval_request") + assert request["name"] == "bash" and request["arguments"] == {"command": "date"} and not of(events, "tool_start") + run.decide(request["call_id"], approved) + finish(run, events, thread) + assert bool(of(events, "tool_start")) is approved and of(events, "tool_result")[0]["ok"] is approved + assert llm.renders[1]["messages"][-1]["content"] == (TIME["result"] if approved else "使用者拒絕執行這個工具呼叫") + assert end(events)["stop_reason"] == "no_tool_call" + + +# ---------- asking first ---------- + +def asking(fake, lang="en"): + fake.say({"say": [TIME]}, {"say": [ANSWER]}) + return start(fake, pi_harness.PiSettings("fake", "fake-model", approve_tools=True, lang=lang)) + + +def test_ask_first_loads_the_extension_and_holds_the_call(fake): + run, events, thread = asking(fake) + request = wait_for(events, "approval_request") + assert request["call_id"] == "call_1" and request["name"] == "bash" and request["arguments"] == {"command": "date"} + assert fake.started()["approve"] == "1" and not of(events, "tool_start") + run.decide("call_1", True) + finish(run, events, thread) + kinds = [e["type"] for e in events] + assert kinds.index("approval") < kinds.index("tool_start") < kinds.index("tool_result") + assert of(events, "approval")[0]["approved"] and of(events, "tool_result")[0]["ok"] + + +@pytest.mark.parametrize("lang, refusal", [("zh", "使用者拒絕執行這個工具呼叫"), ("en", "The user refused to run this tool call")]) +def test_a_rejected_call_does_not_run_and_the_model_reads_why_in_the_turns_language(fake, lang, refusal): + run, events, thread = asking(fake, lang) + wait_for(events, "approval_request") + run.decide("call_1", False) + finish(run, events, thread) + assert not of(events, "tool_start") + result = of(events, "tool_result")[0] + assert not result["ok"] and result["result"] == refusal + assert fake.started()["refusal"] == refusal + assert end(events)["stop_reason"] == "no_tool_call" + + +def test_stopping_while_a_call_waits_answers_it_as_stopped(fake): + run, events, thread = asking(fake) + wait_for(events, "approval_request") + run.stop() + finish(run, events, thread) + approval = of(events, "approval")[0] + assert not approval["approved"] and approval["stopped"] and not approval["timed_out"] + assert end(events)["stop_reason"] == "user_stopped" + + +def test_an_unanswered_call_times_out_and_stops_pi(fake, monkeypatch): + monkeypatch.setattr(pi_harness, "APPROVAL_TIMEOUT", 0.2) + run, events, thread = asking(fake) + finish(run, events, thread) + approval = of(events, "approval")[0] + assert approval["timed_out"] and not approval["approved"] + assert end(events)["stop_reason"] == "approval_timeout" + + +# ---------- stopping ---------- + +def test_stop_aborts_pi_mid_reply(fake): + fake.say({"forever": True}) + run, events, thread = start(fake) + wait_for(events, "delta") + run.stop() + finish(run, events, thread) + assert end(events)["stop_reason"] == "user_stopped" + assert any(c["type"] == "abort" for c in fake.commands()) + assert of(events, "model_end")[-1]["finish"] == "user_stopped" + + +def test_pi_that_does_not_stop_when_asked_is_killed(fake, monkeypatch): + monkeypatch.setattr(pi_harness, "STOP_GRACE", 0.3) + fake.say({"forever": True}, deaf=True) + run, events, thread = start(fake) + wait_for(events, "delta") + run.stop() + finish(run, events, thread) + assert end(events)["stop_reason"] == "user_stopped" + + +def test_stop_before_pi_has_started_sends_no_question(fake): + fake.say({"say": [ANSWER]}) + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings("fake", "fake-model"), [], "hi", events.append, (), fake.folder, "chat1") + run.stop() + run.run() + assert end(events)["stop_reason"] == "user_stopped" and not of(events, "step_start") + assert "prompt" not in [c["type"] for c in fake.commands()] + + +def test_the_step_limit_stops_pi_and_no_step_past_it_is_drawn(fake): + fake.say(*[{"say": [TIME]}] * 4, {"say": [ANSWER]}) + events = finish(*start(fake, pi_harness.PiSettings("fake", "fake-model", max_steps=2))) + assert [e["step"] for e in of(events, "step_start")] == [1, 2] + assert end(events)["stop_reason"] == "max_steps" and events[0]["settings"]["max_steps"] == 2 + + +# ---------- what goes wrong ---------- + +def test_a_failed_request_is_an_error(fake): + events = run_pi(fake, {"say": [], "stop": "error", "error": "Connection error."}) + assert of(events, "error")[0]["message"] == "pi: Connection error." + assert end(events)["stop_reason"] == "error" and end(events)["final"] == "" + + +def test_a_refused_question_is_said_in_both_languages(fake): + fake.say(refuse="No API key found for the selected model.") + events = finish(*start(fake)) + error = of(events, "error")[0] + assert "No API key found" in error["text"]["en"] and "pi 拒絕了 `prompt`" in error["text"]["zh"] + assert end(events)["stop_reason"] == "error" + + +def test_a_model_pi_does_not_have_is_an_error(fake): + fake.say() + events = finish(*start(fake, pi_harness.PiSettings("fake", "gone"))) + assert "Model not found: fake/gone" in of(events, "error")[0]["text"]["en"] + + +def test_pi_that_dies_at_start_says_its_last_words(fake): + fake.say(crash="SyntaxError: Unexpected token (Node.js 20 is too old)") + events = finish(*start(fake)) + error = of(events, "error")[0]["text"] + assert "Node.js 20 is too old" in error["en"] and "pi 意外結束了" in error["zh"] + + +def test_pi_that_is_not_installed_says_how_to_install_it(fake, monkeypatch): + monkeypatch.setattr(pi_harness, "PI_COMMAND", ["no-such-pi-anywhere"]) + events = finish(*start(fake)) + error = of(events, "error")[0]["text"] + assert pi_harness.INSTALL in error["en"] and pi_harness.INSTALL in error["zh"] and "PI_COMMAND" in error["en"] + + +# ---------- what pi can run ---------- + +def test_probe_lists_the_models_pi_has_and_the_one_it_starts_on(fake): + fake.say(version="1.4.0") + found = pi_harness.probe() + assert found["version"] == "1.4.0" and found["default"] == "fake/fake-model" + assert [f'{m["provider"]}/{m["id"]}' for m in found["models"]] == ["fake/fake-model"] + + +def test_probe_without_models_has_no_default(fake): + fake.say(models=[]) + assert pi_harness.probe() == {"version": "1.4.0", "models": [], "default": None} + + +def test_probe_of_pi_that_is_not_installed(monkeypatch): + monkeypatch.setattr(pi_harness, "PI_COMMAND", ["no-such-pi-anywhere"]) + with pytest.raises(pi_harness.PiError) as e: + pi_harness.probe() + assert pi_harness.INSTALL in e.value.text["zh"] + + +# ---------- the real pi ---------- + +REAL = os.environ.get("RUN_PI_TESTS") == "1" + + +class ScriptedModel(BaseHTTPRequestHandler): + """An OpenAI-compatible chat completions endpoint: it calls bash once, then answers from the result.""" + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + self.server.requests.append(body) + has_result = any(m.get("role") == "tool" for m in body["messages"]) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + + def chunk(delta, finish=None): + return {"id": "c", "object": "chat.completion.chunk", "created": 0, "model": "scripted", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} + + if has_result: + chunks = [chunk({"role": "assistant", "content": "It says "}), chunk({"content": "hi."}), chunk({}, "stop")] + else: + chunks = [chunk({"role": "assistant", "content": "Let me run it."}), + chunk({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", "function": {"name": "bash", "arguments": '{"command": "echo hi"}'}}]}), + chunk({}, "tool_calls")] + chunks.append({"id": "c", "object": "chat.completion.chunk", "created": 0, "model": "scripted", "choices": [], + "usage": {"prompt_tokens": 1500, "completion_tokens": 9, "total_tokens": 1509}}) + for c in chunks: + self.wfile.write(f"data: {json.dumps(c)}\n\n".encode()) + self.wfile.flush() + time.sleep(0.02) + self.wfile.write(b"data: [DONE]\n\n") + + def log_message(self, *args): + pass + + +@pytest.fixture +def real_pi(monkeypatch, tmp_path): + if not REAL: + pytest.skip("RUN_PI_TESTS=1 starts the real pi") + if not (shutil.which(pi_harness.PI_COMMAND[0]) or Path(pi_harness.PI_COMMAND[0]).exists()): + pytest.skip(f"{pi_harness.PI_COMMAND[0]} is not installed") + server = ThreadingHTTPServer(("127.0.0.1", 0), ScriptedModel) + server.requests = [] + threading.Thread(target=server.serve_forever, daemon=True).start() + agent = tmp_path / "agent" # pi's own folder, with only the scripted model in it + agent.mkdir() + (agent / "models.json").write_text(json.dumps({"providers": {"scripted": { + "baseUrl": f"http://127.0.0.1:{server.server_port}/v1", "api": "openai-completions", "apiKey": "none", + "models": [{"id": "scripted-model", "name": "Scripted", "contextWindow": 32000, "maxTokens": 1024, + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}}]}}})) + monkeypatch.setenv("PI_CODING_AGENT_DIR", str(agent)) + monkeypatch.setenv("PI_OFFLINE", "1") + yield server + server.shutdown() + + +def test_the_real_pi_lists_the_model_it_has(real_pi): + found = pi_harness.probe() + assert [f'{m["provider"]}/{m["id"]}' for m in found["models"]] == ["scripted/scripted-model"] + + +@pytest.mark.parametrize("approve", [False, True]) +def test_the_real_pi_runs_a_turn_the_page_can_draw(real_pi, tmp_path, approve): + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings("scripted", "scripted-model", approve_tools=approve, lang="en"), [], "Run echo hi", events.append, + folder=tmp_path / "chat", session="chat1") + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + if approve: + run.decide(wait_for(events, "approval_request", timeout=60)["call_id"], True) + thread.join(60) + assert end(events)["stop_reason"] == "no_tool_call", [e for e in events if e["type"] == "error"] + assert end(events)["final"] == "It says hi." + assert of(events, "tool_result")[0]["result"].strip() == "hi" + assert of(events, "model_end")[0]["usage"]["prompt"] == 1500 + assert "".join(e["text"] for e in of(events, "delta") if e["step"] == 2) == "It says hi." + system = real_pi.requests[0]["messages"][0] + assert system["role"] in ("system", "developer") and "pi" in str(system["content"]) + + +@pytest.mark.parametrize("thinking", ["off", "medium"]) +def test_the_real_pi_asks_the_pages_model_through_the_endpoint(real_pi, tmp_path, thinking): + think = "" if thinking == "off" else "Run it." # a thought the prompt opened ends before the call + llm = FakeLLM([think + tool_call("bash", command="echo hi"), think.replace("Run it", "It printed hi") + "It says hi."]) + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings(model_endpoint.PROVIDER, llm.model_id, thinking=thinking, lang="en"), [], "Run echo hi", + events.append, folder=tmp_path / "chat", session="chat1", page=pi_harness.PageModel(llm, Sampling())) + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + thread.join(60) + assert end(events)["stop_reason"] == "no_tool_call", [e for e in events if e["type"] == "error"] + assert end(events)["final"] == "It says hi." and of(events, "tool_result")[0]["result"].strip() == "hi" + assert [e["step"] for e in of(events, "context")] == [1, 2] and of(events, "token") and not of(events, "delta") + assert "pi" in llm.renders[0]["messages"][0]["content"] and llm.renders[0]["thinking"] is (thinking != "off") + assert [r["step"] for r in of(events, "request")] == [1, 2] + assert not real_pi.requests # the scripted provider's model was never asked + + +def test_the_real_pi_on_the_pages_model_asks_first_and_keeps_the_step_limit(real_pi, tmp_path, monkeypatch): + late_turn_starts(monkeypatch) + llm = FakeLLM([tool_call("bash", command="echo hi"), "It says hi."]) + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings(model_endpoint.PROVIDER, llm.model_id, approve_tools=True, max_steps=1, lang="en"), [], "Run echo hi", + events.append, folder=tmp_path / "chat", session="chat1", page=pi_harness.PageModel(llm, Sampling())) + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + request = wait_for(events, "approval_request", timeout=60) + assert request["name"] == "bash" and not of(events, "tool_start") + decided = time.monotonic() + run.decide(request["call_id"], True) + thread.join(60) + assert time.monotonic() - decided < model_endpoint.WAIT_FOR_STEP / 2 + assert_the_endpoint_is_free() + assert end(events)["stop_reason"] == "max_steps", [e for e in events if e["type"] == "error"] + assert of(events, "tool_result")[0]["result"].strip() == "hi" and [e["step"] for e in of(events, "step_start")] == [1] + assert len(llm.renders) == 1 # the request past the limit is turned away diff --git a/tests/test_qwen3_5.py b/tests/test_qwen3_5.py index 90806d5..75fd6b4 100644 --- a/tests/test_qwen3_5.py +++ b/tests/test_qwen3_5.py @@ -2,10 +2,10 @@ import pytest +from agent.model_families import QWEN3_5 +from agent.model_families.qwen3_5 import convert, parse_output +from agent.tools import TOOLS from fakes import qwen_call, qwen_tool_call, record_run -from harness.families import QWEN3_5 -from harness.families.qwen3_5 import convert, parse_output -from harness.tools import TOOLS SCHEMAS = {t.name: t.parameters for t in TOOLS.values()} TYPED = {"f": {"properties": { diff --git a/tests/test_real_model.py b/tests/test_real_model.py index a437329..8a0008b 100644 --- a/tests/test_real_model.py +++ b/tests/test_real_model.py @@ -5,6 +5,7 @@ import os import re +import shutil import subprocess import sys from pathlib import Path @@ -13,12 +14,13 @@ import torch import samples +from agent import media, model_families, tools +from agent.harnesses import model_endpoint, pi_harness +from agent.harnesses.simple_harness import SYSTEM_WITH_TOOLS, RunSettings, SimpleHarness +from agent.model_backends.backend import Backend +from agent.model_backends.simple_backend import Sampling, SimpleBackend +from agent.tools import TOOLS from app import render -from harness import families, media, tools -from harness.agent import SYSTEM_WITH_TOOLS, Run, RunSettings -from harness.backends.backend import Backend -from harness.backends.simple_backend import SimpleBackend, Sampling -from harness.tools import TOOLS from html_check import assert_balanced from test_render import render_everything @@ -30,7 +32,7 @@ ] -@pytest.fixture(scope="module", params=list(families.MODELS)) +@pytest.fixture(scope="module", params=list(model_families.MODELS)) def gemma(request): return SimpleBackend(request.param) @@ -66,10 +68,28 @@ def test_greedy_decoding_matches_generate(gemma): assert ours == theirs[: len(ours)] +def test_the_real_pi_asks_the_model_through_the_endpoint_and_every_token_is_drawn(gemma, tmp_path, monkeypatch): + """pi (on the PATH, or PI_COMMAND) asking this model through model_endpoint, with no credentials of its own.""" + if not (shutil.which(pi_harness.PI_COMMAND[0]) or Path(pi_harness.PI_COMMAND[0]).exists()): + pytest.skip(f"{pi_harness.PI_COMMAND[0]} is not installed") + (tmp_path / "agent").mkdir() + monkeypatch.setenv("PI_CODING_AGENT_DIR", str(tmp_path / "agent")) + events = [] + run = pi_harness.PiHarness(pi_harness.PiSettings(model_endpoint.PROVIDER, gemma.model_id, tools=("bash",), max_steps=4, lang="en"), [], + "Use the bash tool to print today's date, then tell me the date.", events.append, folder=tmp_path / "chat", + session="c", page=pi_harness.PageModel(gemma, Sampling(temperature=0, max_new_tokens=256))) + run.run() + calls = [e for e in events if e["type"] == "tool_call"] + assert calls and calls[0]["name"] == "bash" and "date" in calls[0]["arguments"]["command"], [e for e in events if e["type"] in ("model_end", "error")] + assert events[-1]["stop_reason"] == "no_tool_call" and events[-1]["final"] + assert [e["step"] for e in events if e["type"] == "context"] == [1, 2] + assert all(e["rank"] == 1 for e in events if e["type"] == "token") # greedy, and every token drawn with its rank + + def test_agent_uses_a_tool_and_answers(gemma): events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=256), max_steps=4) - Run(gemma, [], "請用 calculator 工具計算 12345 * 6789。", settings, events.append).run() + SimpleHarness(gemma, [], "請用 calculator 工具計算 12345 * 6789。", settings, events.append).run() assert not [e for e in events if e["type"] == "error"] calls = [e for e in events if e["type"] == "tool_call"] @@ -159,7 +179,7 @@ def decode(mtp): def test_the_model_reads_the_total_on_the_receipt(gemma): events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=64), use_tools=False) - Run(gemma, [], "What is the TOTAL printed on this receipt? Answer with the number only.", settings, events.append, [attach("receipt.png", gemma)]).run() + SimpleHarness(gemma, [], "What is the TOTAL printed on this receipt? Answer with the number only.", settings, events.append, [attach("receipt.png", gemma)]).run() assert not [e for e in events if e["type"] == "error"] assert "370" in events[-1]["final"] [context] = [e for e in events if e["type"] == "context"] @@ -171,7 +191,7 @@ def test_the_model_writes_code_that_runs_in_the_sandbox(gemma, tmp_path): events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=512), max_steps=4, lang="en") question = "Use run_python to compute the sum of the squares of the integers from 1 to 100." - Run(gemma, [], question, settings, events.append, files=tmp_path).run() + SimpleHarness(gemma, [], question, settings, events.append, files=tmp_path).run() assert not [e for e in events if e["type"] == "error"] calls = [e for e in events if e["type"] == "tool_call" and e["name"] == "run_python"] results = [e for e in events if e["type"] == "tool_result" and e["name"] == "run_python"] @@ -207,7 +227,7 @@ def test_the_model_reads_every_media_format_it_has_an_encoder_for(gemma, tmp_pat question, right = MEDIA_QUESTIONS[kind] events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=128), use_tools=False, lang="en") - Run(gemma, [], question, settings, events.append, [attachment], tmp_path).run() + SimpleHarness(gemma, [], question, settings, events.append, [attachment], tmp_path).run() assert not [e for e in events if e["type"] == "error"] assert next(e for e in events if e["type"] == "context")["media_tokens"][kind] > 0 assert right(events[-1]["final"]), events[-1]["final"] @@ -220,7 +240,7 @@ def test_the_model_opens_every_data_format_with_run_python(gemma, tmp_path, ext) attachment = media.Attachment(path.name, "file", str(path), path.stat().st_size) events = [] settings = RunSettings(sampling=Sampling(temperature=0, max_new_tokens=512), max_steps=3, lang="en") - Run(gemma, [], f"Use run_python to open /data/{path.name} and print its first two rows.", settings, events.append, [attachment], tmp_path).run() + SimpleHarness(gemma, [], f"Use run_python to open /data/{path.name} and print its first two rows.", settings, events.append, [attachment], tmp_path).run() results = [e for e in events if e["type"] == "tool_result" and e["name"] == "run_python"] assert any(r["ok"] and "2025" in r["result"] for r in results), results @@ -233,7 +253,7 @@ def test_letting_a_warmed_up_model_go_hands_its_gpu_memory_back(): script = """ import gc, sys, torch sys.path.insert(0, sys.argv[1]) -from harness.backends.simple_backend import SimpleBackend, empty_cache +from agent.model_backends.simple_backend import SimpleBackend, empty_cache llm = SimpleBackend("Qwen/Qwen3.5-0.8B") llm.warmup() loaded = torch.cuda.memory_reserved() diff --git a/tests/test_render.py b/tests/test_render.py index df9ccf6..524c3bb 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -5,36 +5,40 @@ import dataclasses import json import re +import tempfile import threading import xml.etree.ElementTree as ET from pathlib import Path import pytest +from agent import model_families +from agent.harnesses import model_endpoint, pi_harness, simple_harness +from agent.harnesses.simple_harness import RunSettings, SimpleHarness +from agent.i18n import LANGS +from agent.media import Attachment +from agent.model_backends.backend import TOKEN_DETAILS, Sampling +from agent.model_families import GEMMA4, QWEN3_5 +from agent.tools import TOOLS, ToolResult from app import render +from fake_pi import FakePi from fakes import DRAFTER, INFO, MEDIA_TOKENS, QWEN_INFO, FakeLLM, qwen_tool_call, record_run, tool_call -from harness import agent, families -from harness.agent import Run, RunSettings -from harness.backends.backend import TOKEN_DETAILS, Sampling -from harness.families import GEMMA4, QWEN3_5 -from harness.i18n import LANGS -from harness.media import Attachment -from harness.tools import TOOLS, ToolResult from html_check import assert_balanced ANSWER = "現在是下午三點。" def svg_of(diagram: str) -> str: - src = re.search(r'src="data:image/svg\+xml;base64,([^"]+)"', diagram).group(1) - return base64.b64decode(src).decode() + src = re.search(r'src="data:image/svg\+xml;base64,([^"]+)"', diagram) + assert src, diagram + return base64.b64decode(src.group(1)).decode() def approved_run_events(): """A thinking run that asks for approval, gets it, calls a tool and answers.""" fake = FakeLLM(["要查時間" + tool_call("get_current_time"), "好" + ANSWER]) events = [] - run = Run(fake, [], "台北現在幾點？", RunSettings(thinking=True, approve_tools=True), events.append) + run = SimpleHarness(fake, [], "台北現在幾點？", RunSettings(thinking=True, approve_tools=True), events.append) thread = threading.Thread(target=run.run, daemon=True) thread.start() for _ in range(500): @@ -58,6 +62,36 @@ def token(event): LOGPROBS = ("p", "logprob", "top") # what an OpenAI-style API's log probabilities give +BASH = {"tool": "bash", "args": {"command": "date"}, "result": "Mon Oct 26 14:00:00 CST 2026\n"} + + +def pi_page_events(outputs: list, text: str = "現在幾點？", **settings) -> list[dict]: + """A turn pi (its stand-in) runs on the page's own model (a FakeLLM) through model_endpoint, as its events.""" + with pytest.MonkeyPatch.context() as mp, tempfile.TemporaryDirectory() as folder: + FakePi(mp, Path(folder)).say(results={"bash": BASH["result"]}) + llm, events = FakeLLM(list(outputs)), list[dict]() + pi_harness.PiHarness(pi_harness.PiSettings(model_endpoint.PROVIDER, llm.model_id, **settings), [], text, events.append, + folder=Path(folder) / "chat", session="c", page=pi_harness.PageModel(llm, Sampling())).run() + return events + + +def pi_events(*turns, approve: bool | None = None, text: str = "現在幾點？", **settings) -> list[dict]: + """A turn run by pi (its stand-in, tests/fake_pi.py), as its events; with `approve`, the user answers the first call so.""" + with pytest.MonkeyPatch.context() as mp, tempfile.TemporaryDirectory() as folder: + FakePi(mp, Path(folder)).say(*turns) + events: list[dict] = [] + run = pi_harness.PiHarness(pi_harness.PiSettings("fake", "fake-model", approve_tools=approve is not None, **settings), [], text, events.append, + folder=Path(folder) / "chat", session="c") + thread = threading.Thread(target=run.run, daemon=True) + thread.start() + if approve is not None: + for _ in range(1000): + if request := next((e for e in events if e["type"] == "approval_request"), None): + run.decide(request["call_id"], approve) + break + threading.Event().wait(0.01) + thread.join(20) + return events PHOTO = Attachment("photo.png", "image", "/chat/photo.png", 2048) CLIP = Attachment("clip.mp4", "video", "/chat/clip.mp4", 90_000) SALES = Attachment("sales.csv", "file", str(Path(__file__).resolve().parent.parent / "app/assets/examples/sales.csv"), 942) @@ -82,6 +116,11 @@ def token(event): "qwen_thinking": lambda: record_run( ["要算一下\n\n\n" + qwen_tool_call("calculator", expression="1"), "想好了\n\n\n好。<|im_end|>"], family=QWEN3_5, thinking=True)[1], + "pi": lambda: pi_events({"say": [{"thinking": "用 date。"}, {"text": "我查一下。"}, BASH]}, {"say": [{"text": "下午兩點。"}]}), + "pi_rejected": lambda: pi_events({"say": [BASH]}, {"say": [{"text": "好吧。"}]}, approve=False), + "pi_max_steps": lambda: pi_events(*[{"say": [BASH]}] * 3, max_steps=2), + "pi_error": lambda: pi_events({"say": [], "stop": "error", "error": "Connection error."}), + "pi_page_model": lambda: pi_page_events([tool_call("bash", command="date"), ANSWER]), } @@ -100,7 +139,10 @@ def render_everything(data: dict, live: bool, lang: str = "zh") -> list[str]: out.append(render.context_html(step, data["family"], lang=lang)) if data["end"]: out.append(render.stop_html(data, lang=lang)) - if data["settings"]: + if data["settings"] and data["harness"] == "pi": + out.append(render.pi_settings_html(data["settings"], lang=lang)) + out.append(render.pi_tools_html(pi_harness.TOOLS, data["settings"]["tools"], lang=lang)) + elif data["settings"]: out.append(render.knobs_html(data["settings"], INFO["defaults"], lang=lang)) out.append(render.tools_html(TOOLS, data["settings"]["use_tools"], data["family"], lang=lang, offered=data["settings"]["tools"])) return [piece for piece in out if piece] @@ -149,6 +191,11 @@ def test_every_prefix_of_a_run_renders(scenario, lang): "no_details": lambda: as_reported(record_run([tool_call("calculator", expression="6*7"), "It is 42."], user="6*7?", lang="en")[1]), "logprobs": lambda: as_reported( record_run([tool_call("calculator", expression="6*7"), "It is 42."], user="6*7?", lang="en")[1], LOGPROBS), + "pi": lambda: pi_events({"say": [{"thinking": "Use date."}, BASH]}, {"say": [{"text": "It is 2 pm."}]}, text="Time?", lang="en"), + "pi_rejected": lambda: pi_events({"say": [BASH]}, {"say": [{"text": "OK."}]}, approve=False, text="Time?", lang="en"), + "pi_max_steps": lambda: pi_events(*[{"say": [BASH]}] * 3, max_steps=2, text="Time?", lang="en"), + "pi_error": lambda: pi_events({"say": [], "stop": "error", "error": "Connection error."}, text="Time?", lang="en"), + "pi_page_model": lambda: pi_page_events([tool_call("bash", command="date"), "It is 2 pm."], text="Time?", lang="en"), } @@ -166,7 +213,7 @@ def test_an_english_page_has_no_chinese_left(scenario): @pytest.mark.parametrize("info", [INFO, {**INFO, "drafter_id": DRAFTER["model_id"]}, {**INFO, "drafter_id": DRAFTER["model_id"], "drafter": DRAFTER}, QWEN_INFO]) def test_the_model_tab_in_english_has_no_chinese(info): - family = families.get(info["family"]) + family = model_families.get(info["family"]) pieces = [ render.model_html(info, lang="en"), render.special_tokens_html(info, lang="en"), render.folder_html(info, lang="en"), render.INTRO["en"], render.CONTEXT_LEGEND["en"], @@ -194,7 +241,7 @@ def test_switching_language_redraws_a_finished_run(): def test_build_of_nothing(): data = render.build([]) - assert data == {"model": None, "family": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} + assert data == {"model": None, "family": None, "harness": None, "settings": {}, "steps": [], "end": None, "error": None, "last": None, "t": 0} def test_build_folds_a_tool_run_into_steps(): @@ -407,9 +454,9 @@ def test_step_card_approval_states(): def test_approval_timeout_is_shown(monkeypatch): - monkeypatch.setattr(agent, "APPROVAL_TIMEOUT", 0.05) + monkeypatch.setattr(simple_harness, "APPROVAL_TIMEOUT", 0.05) events = [] - Run(FakeLLM([tool_call("get_current_time"), ANSWER]), [], "幾點？", RunSettings(approve_tools=True), events.append).run() + SimpleHarness(FakeLLM([tool_call("get_current_time"), ANSWER]), [], "幾點？", RunSettings(approve_tools=True), events.append).run() data = render.build(events) assert list(data["steps"][0]["calls"].values())[0]["approval"] == "timeout" assert "等待確認逾時，harness 當作拒絕並停止迴圈" in render.step_html(data["steps"][0], data, live=False, lang="zh") @@ -424,7 +471,7 @@ def test_approval_timeout_is_shown(monkeypatch): def test_stopping_is_not_shown_as_rejecting(answer, label, not_label): fake = FakeLLM([tool_call("get_current_time"), ANSWER]) events = [] - run = Run(fake, [], "幾點？", RunSettings(approve_tools=True), events.append) + run = SimpleHarness(fake, [], "幾點？", RunSettings(approve_tools=True), events.append) thread = threading.Thread(target=run.run, daemon=True) thread.start() for _ in range(500): @@ -630,7 +677,7 @@ def test_the_model_folder_lists_each_file_with_its_size_what_it_is_and_what_it_h assert "git settings" in html and "Gemma 4: " not in html.split(".gitattributes")[1] # no family note on it assert "gloss-head" not in html # no drafter loaded, so one folder assert "a later download may get a newer one" in html # fake/gemma-test is not pinned - pinned = render.folder_html({**INFO, "model_id": "google/gemma-4-E2B-it", "folder": {**INFO["folder"], "revision": families.REVISIONS["google/gemma-4-E2B-it"]}}, "en") + pinned = render.folder_html({**INFO, "model_id": "google/gemma-4-E2B-it", "folder": {**INFO["folder"], "revision": model_families.REVISIONS["google/gemma-4-E2B-it"]}}, "en") assert "revision 3e22461, the commit the app pins it to" in pinned assert_balanced(html) @@ -680,7 +727,7 @@ def test_the_model_folder_escapes_what_the_files_hold(): def test_every_file_a_family_notes_is_a_kind_the_page_knows(): - for family in families.FAMILIES.values(): + for family in model_families.FAMILIES.values(): for name in (*family.file_notes, *family.drafter_file_notes): assert render.file_kind(name) == name @@ -843,7 +890,7 @@ def test_every_explained_token_becomes_a_chip_with_a_tip(piece): assert GEMMA4.special_re.fullmatch(piece) html = render.special_html(f"x{piece}y", GEMMA4, lang="zh") name, _ = GEMMA4.special_tokens[piece]["zh"] - assert html == f"x{render.chip(piece, GEMMA4, lang="zh")}y" + assert html == f"x{render.chip(piece, GEMMA4, lang='zh')}y" assert f'{render.esc(name)}' in html assert_balanced(html) @@ -972,3 +1019,125 @@ def test_the_model_tab_lists_what_the_model_reads(): def test_a_tool_left_out_of_the_turn_is_marked(): html = render.tools_html(TOOLS, True, GEMMA4, "en", offered=["calculator"]) assert html.count("not offered to the model in this turn") == len(TOOLS) - 1 + + +# ---------- a run pi ran ---------- + +def test_a_pi_step_shows_the_text_it_streamed_by_channel_and_no_token_numbers(): + data = render.build(SCENARIOS["pi"]()) + assert data["harness"] == "pi" and data["family"] is None + step = data["steps"][0] + assert step["text"] == [{"channel": "thought", "text": "用 date。"}, {"channel": "content", "text": "我查一下。"}, + {"channel": "tool_call", "text": '\nbash {"command": "date"}'}] + card = render.step_html(step, data, lang="en", live=False) + assert '用 date。' in card and 'class="run ch-tool_call"' in card + assert "wrote 20 tokens" in card and "read 1,200 tokens, 200 from the cache" in card and "at pi's prices" not in card + assert "about $0.0021 at pi's prices" in card + assert "the API returned it as JSON" in card and "shell on this computer · returned" in card + assert "Context the request pi sends the API" in card and "system 1 · user 1" in card # the request, not a context + assert 'class="tok ' not in card and "token id" not in card # and no tokens + + +def test_what_pi_streams_is_escaped(): + data = render.build(pi_events({"say": [{"text": ""}]})) + card = render.step_html(data["steps"][0], data, lang="en", live=False) + assert "