From 69770119fff65bf604e23146f92bf73580c03cdc Mon Sep 17 00:00:00 2001 From: Rahul Sethuram Date: Fri, 21 Aug 2026 12:15:13 +0400 Subject: [PATCH] Tell a built-in Bot about its computer The instructions that make the computer usable live in shared/bot-prompt.ts: navigate rather than claiming you cannot browse, snapshot before acting, and at a sign-in call computer_request_help so a person can take the wheel rather than reporting the task as impossible. Two files imported it, both of them shipped Bots. A built-in agent knew only the role its package gave it, while the surface offered it the same computer tools as everything else. Asked to file an issue on a site it was not signed in to, General Assistant browsed to the page, said it could not, and never asked for help, so the wheel was never offered. That is not a Bot declining; it is a Bot that was never told. Renamed to COMPUTER_GUIDANCE, because instructions about the computer belong to the computer rather than to one implementation, and appended to the package role for built-in agents. Only where a computer is configured: a deployment with the browser routes unmounted should not promise a Bot hands it has not got. --- CHANGELOG.md | 7 +++++ agent-bot/src/index.ts | 4 +-- agent-langgraph/src/index.ts | 4 +-- server/src/copilot.ts | 51 +++++++++++++++++++++++++++++++++--- shared/bot-prompt.ts | 11 +++++--- 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 945ff421..be900d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,13 @@ Sessions survive and nobody signs in again. laptop `http://localhost` counts as one, so this never showed up in development; on a real address it does not, and the surface did nothing at all when you pressed send. No message, no error. Ids now come from an API with no such restriction. +- **A package Bot did not know it had a computer.** The instructions that make the computer usable — + snapshot before acting, and ask a person to take the wheel at a sign-in rather than reporting the + task as impossible — were imported by the two shipped Bots and by nothing else, so a built-in agent + knew only the role its package gave it. The tools were on offer to it the whole time. Asked to file + an issue on a site it was not signed in to, it browsed to the page, said it could not, and never + called `computer_request_help`, so nobody was ever offered the wheel. Built-in agents are now told + the same thing the shipped Bots are told, wherever a computer is configured. ### Changed diff --git a/agent-bot/src/index.ts b/agent-bot/src/index.ts index 3fecd861..31fdf208 100644 --- a/agent-bot/src/index.ts +++ b/agent-bot/src/index.ts @@ -3,7 +3,7 @@ import { EventEncoder } from "@ag-ui/encoder"; import { serve } from "bun"; import OpenAI from "openai"; import { hasManagedAgentToken } from "../../shared/agent-authorisation"; -import { SYSTEM_PROMPT } from "../../shared/bot-prompt"; +import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; /** * The built-in Bot is an AG-UI HTTP service registered the same way as any customer-provided Bot. @@ -53,7 +53,7 @@ const openai = new OpenAI({ /** Translate the conversation AG-UI carries into the shape the model provider expects. */ function toProviderMessages(input: RunAgentInput) { const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [ - { role: "system", content: SYSTEM_PROMPT }, + { role: "system", content: COMPUTER_GUIDANCE }, ]; for (const message of input.messages) { diff --git a/agent-langgraph/src/index.ts b/agent-langgraph/src/index.ts index c524fb0b..1c144081 100644 --- a/agent-langgraph/src/index.ts +++ b/agent-langgraph/src/index.ts @@ -18,7 +18,7 @@ import { import { ChatOpenAI } from "@langchain/openai"; import { serve } from "bun"; import { hasManagedAgentToken } from "../../shared/agent-authorisation"; -import { SYSTEM_PROMPT } from "../../shared/bot-prompt"; +import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; /** * The same Bot, on a framework. @@ -129,7 +129,7 @@ if (!API_KEY) { /** Translate the conversation AG-UI carries into LangChain's message classes. */ function toLangChainMessages(input: RunAgentInput): BaseMessage[] { - const messages: BaseMessage[] = [new SystemMessage(SYSTEM_PROMPT)]; + const messages: BaseMessage[] = [new SystemMessage(COMPUTER_GUIDANCE)]; for (const message of input.messages) { if (message.role === "user") { diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 02297684..766ce4f2 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -6,10 +6,11 @@ import { CopilotRuntime, } from "@copilotkit/runtime/v2"; import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono"; +import { z } from "zod"; +import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; import type { AgentActor } from "./agents/profile-types"; import type { StallGuard } from "./channels/stall-guard"; import type { DeploymentConfig } from "./config"; -import { z } from "zod"; import type { GrantedTool } from "./plugins/tools"; /** @@ -175,6 +176,15 @@ export function builtInAgentConfiguration( * let the agent reach a vendor directly and walk around all three. */ tools: GrantedTool[] = [], + /** + * What this Bot should know about the computer, when this deployment has one. + * + * Appended to the role rather than replacing it: the package says what the Bot is for, this says + * what its hands are. Absent leaves the role alone, which is right for a deployment with no + * computer configured, where the browser routes are not mounted and a Bot promised a browser would + * be promising something that does not exist. + */ + computerGuidance?: string, ): BuiltInAgentConfiguration { if (!apiKey) { return { @@ -190,7 +200,9 @@ export function builtInAgentConfiguration( return { model: `${model.provider}/${model.defaultModel}`, - prompt: agent.systemPrompt, + prompt: computerGuidance + ? `${agent.systemPrompt}\n\n${computerGuidance}` + : agent.systemPrompt, apiKey, /* * A run stops after one step unless told otherwise, which for a Bot with tools means it calls @@ -228,12 +240,22 @@ export async function buildAgents( /** Absent leaves every Bot with no tools, which is the correct answer when nothing is granted. */ loadTools: LoadToolsForBot = async () => [], signRun?: SignRun, + /** What every built-in Bot is told about the computer. Absent means this deployment has none. */ + computerGuidance?: string, ): Promise> { return Object.fromEntries( await Promise.all( agents.map(async (agent) => [ agent.id, - await buildAgent(agent, model, apiKey, stallGuard, loadTools, signRun), + await buildAgent( + agent, + model, + apiKey, + stallGuard, + loadTools, + signRun, + computerGuidance, + ), ]), ), ); @@ -246,6 +268,7 @@ async function buildAgent( stallGuard: StallGuard | undefined, loadTools: LoadToolsForBot, signRun?: SignRun, + computerGuidance?: string, ): Promise { if (agent.type === "built_in") { return new BuiltInAgent( @@ -254,6 +277,7 @@ async function buildAgent( model, apiKey, await loadTools(agent.id), + computerGuidance, ), ); } @@ -389,6 +413,7 @@ export async function resolveRuntimeAgents( stallGuard?: StallGuard, loadTools?: LoadToolsForBot, signRun?: SignRun, + computerGuidance?: string, ): Promise> { const registered = await loadAgents(); if (registered.length === 0) { @@ -400,7 +425,15 @@ export async function resolveRuntimeAgents( const apiKey = registered.some((agent) => agent.type === "built_in") ? await resolveModelApiKey() : null; - return buildAgents(registered, model, apiKey, stallGuard, loadTools, signRun); + return buildAgents( + registered, + model, + apiKey, + stallGuard, + loadTools, + signRun, + computerGuidance, + ); } /** What one Bot may call, for the person whose request this is. */ @@ -445,6 +478,8 @@ export function createRequestAgents( loadToolsForActor?: (actorId: string) => LoadToolsForBot, /** Resolved per request, because what it signs is who this request turned out to be. */ signRunForActor?: (actorId: string) => SignRun, + /** What every built-in Bot is told about the computer. Absent means this deployment has none. */ + computerGuidance?: string, ) { return async ({ request }: { request: Request }) => { const actor = await identifyActor(request); @@ -455,6 +490,7 @@ export function createRequestAgents( stallGuard, loadToolsForActor?.(actor.id), signRunForActor?.(actor.id), + computerGuidance, ); }; } @@ -513,6 +549,13 @@ export function mountCopilotRuntime( stallGuard, loadToolsForActor, signRunForActor, + /* + * Only when a computer exists. The tools themselves are registered by the surface, so a Bot is + * offered them without this and the guidance is what tells it how they go together: snapshot + * before acting, and ask a person to take the wheel at a sign-in rather than reporting the task + * as impossible. Absent computer, absent guidance: a Bot is not told about hands it has not got. + */ + config.computer ? COMPUTER_GUIDANCE : undefined, ) as never, }); diff --git a/shared/bot-prompt.ts b/shared/bot-prompt.ts index 8e924902..fc5b14af 100644 --- a/shared/bot-prompt.ts +++ b/shared/bot-prompt.ts @@ -1,8 +1,13 @@ /** * What a Bot in this box knows about its own hands. * - * Shared by `agent-bot` and `agent-langgraph` so the implementations differ by framework, not by - * instructions or available computer behavior. + * Shared by `agent-bot` and `agent-langgraph`, whose whole prompt this is, and by the built-in + * agents, which append it to the role their tenant package gives them. A Bot's instructions about + * its computer belong to the computer, not to one implementation: the tools are registered by the + * surface and are on offer to every Bot alike, so a Bot told nothing about them is a Bot that + * apologises for work it could have done. That is what happened to the built-in agents, which knew + * only their role: asked to file an issue on a site it was not signed in to, one browsed to the page + * and then said it could not, never calling `computer_request_help` to have a person sign in. */ /** * The order of operations that makes the computer tools usable. @@ -10,7 +15,7 @@ * The prompt requires snapshot-first computer use. Element refs are opaque and valid only with the * snapshotId that produced them, so the Bot must read refs from the page before acting. */ -export const SYSTEM_PROMPT = [ +export const COMPUTER_GUIDANCE = [ "You are a Bot with your own computer, a real web browser the person can watch you use.", "When you are asked to look at, open, visit, check or read a web page, call computer_navigate.", "Never claim you cannot browse: opening a page is something you can actually do.",