From cc8097dc20e726f24720764305132c7edc93efab Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 13:26:05 -0800 Subject: [PATCH 1/7] LangGraph TypeScript engine: a Claude plan's query is named per organization too One Claude engine serves every organization of an OpenBot. The engine named a Claude plan's query by (Bot, thread) alone, so the same thread id for a Bot of the same id in two organizations went to one query: one transcript and one set of turns for both. The server now names the run's organization in forwardedProps.openbotOrganization, and the engine joins it to the query's name. A run with no organization keeps the name it always had, so a thread that already has a query keeps it. --- agent-langgraph-ts/src/models.ts | 10 ++++- agent-langgraph-ts/src/plan-model.ts | 34 ++++++++++++--- agent-langgraph-ts/src/run.ts | 4 ++ agent-langgraph-ts/test/claude-plan.test.ts | 48 +++++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/agent-langgraph-ts/src/models.ts b/agent-langgraph-ts/src/models.ts index 503ee54d46..a3ddc07f7b 100644 --- a/agent-langgraph-ts/src/models.ts +++ b/agent-langgraph-ts/src/models.ts @@ -72,6 +72,8 @@ export type RunModel = { route?: ModelRoute; /** The run's thread, which a Claude plan's query is kept under. */ threadId?: string; + /** The run's organization, when the server names one, which joins that query's name. */ + organization?: string; /** The run's own stop, which cancels a Claude plan's query. */ signal?: AbortSignal; }; @@ -399,7 +401,13 @@ function routedModel(run: RunModel, env: Env): BaseChatModel | undefined { model: run.model, effort: run.effort, botId: run.botId, - ...(claude ? { threadId: run.threadId ?? "", signal: run.signal } : {}), + ...(claude + ? { + threadId: run.threadId ?? "", + ...(run.organization ? { organization: run.organization } : {}), + signal: run.signal, + } + : {}), }); } if (!route.token) diff --git a/agent-langgraph-ts/src/plan-model.ts b/agent-langgraph-ts/src/plan-model.ts index 4fa7508803..2cd69d5a18 100644 --- a/agent-langgraph-ts/src/plan-model.ts +++ b/agent-langgraph-ts/src/plan-model.ts @@ -5,7 +5,7 @@ * A CLAUDE PLAN, as plan-model.ts reaches it for built-in Bots. The plan token never leaves the * Claude engine; the Claude Agent SDK makes every call. Its query is kept between a step's tool call * and that call's result, so every call of a run, and the next run on the same thread, goes under - * one thread id per (Bot, thread) (`claudeThreadId`). A stopped run whose query is answering or + * one thread id per (Bot, thread, organization) (`claudeThreadId`). A stopped run whose query is answering or * waiting on a call cancels it at `/model/cancel`; a run that hands a call to the surface does not, * since the query waits for that result. * @@ -64,16 +64,36 @@ export type PlanHarnessFields = { * (`claudeThreadId`); a ChatGPT plan's calls each go under a fresh thread. */ threadId?: string; + /** + * The organization the run is in, as the server names it (`forwardedProps.openbotOrganization`). + * Absent in the organization every OpenBot starts with, and from a server that sends none. + */ + organization?: string; /** The run's stop, which cancels a Claude plan's query. */ signal?: AbortSignal; /** For tests: the fetch the AG-UI client uses. */ fetch?: typeof fetch; }; -/** One Claude query per Bot and thread, as plan-model.ts names it on the server. */ -export function claudeThreadId(botId: string, threadId: string): string { +/** + * One Claude query per Bot and thread, as plan-model.ts names it on the server, and per + * organization where the server names one. Two organizations that use the same thread id for a + * Bot of the same id then never share a query, its transcript or its turns. + * + * Without an organization the name is the one it always was, so a thread that already has a + * query keeps it. + */ +export function claudeThreadId( + botId: string, + threadId: string, + organization?: string, +): string { return createHash("sha256") - .update(JSON.stringify([botId, threadId])) + .update( + JSON.stringify( + organization ? [botId, threadId, organization] : [botId, threadId], + ), + ) .digest("hex"); } @@ -221,7 +241,11 @@ export class PlanHarnessChatModel extends BaseChatModel super({}); this.fields = fields; if (fields.plan !== "claude" || fields.threadId === undefined) return; - const query = claudeThreadId(fields.botId ?? "", fields.threadId); + const query = claudeThreadId( + fields.botId ?? "", + fields.threadId, + fields.organization, + ); this.query = query; fields.signal?.addEventListener("abort", () => void this.cancel(query), { once: true, diff --git a/agent-langgraph-ts/src/run.ts b/agent-langgraph-ts/src/run.ts index 422078102c..99e9e47d5e 100644 --- a/agent-langgraph-ts/src/run.ts +++ b/agent-langgraph-ts/src/run.ts @@ -555,6 +555,10 @@ export async function runAgent( botId: botIdOf(input.messages ?? []), route: runRoute(props), threadId: input.threadId, + ...(typeof props.openbotOrganization === "string" && + props.openbotOrganization + ? { organization: props.openbotOrganization } + : {}), signal: options.signal, }; const emit = options.emit; diff --git a/agent-langgraph-ts/test/claude-plan.test.ts b/agent-langgraph-ts/test/claude-plan.test.ts index 2774dec28f..5d883173e0 100644 --- a/agent-langgraph-ts/test/claude-plan.test.ts +++ b/agent-langgraph-ts/test/claude-plan.test.ts @@ -267,6 +267,54 @@ test("A step's tool call and the next step stay on one Claude query: the same th } }); +/* + * One Claude engine serves every organization. The same thread id from two organizations, for a + * Bot of the same id, has to be two Claude queries: one query is one transcript and one set of + * turns, and one organization would answer inside the other's. A run with no organization keeps + * the name it always had, so a thread that already has a query keeps it. + */ +test("The same thread in two organizations is two Claude queries; with no organization the name is unchanged", async () => { + const claude = await claudeEngine(() => fixture("text")); + const inOrganization = (organization?: string) => + conversation({ + forwardedProps: { + openbotModelRoute: claudeRoute, + ...(organization ? { openbotOrganization: organization } : {}), + }, + }); + try { + await withEngine({ env: env(claude.url) }, async (base) => { + await run(base, inOrganization("org-a")); + await run(base, inOrganization("org-b")); + await run(base, inOrganization()); + }); + const [a, b, none] = claude.requests.map( + (request) => request.body.threadId, + ); + assert.equal(claude.requests.length, 3); + assert.notEqual(a, b); + assert.notEqual(a, none); + assert.notEqual(b, none); + assert.equal( + a, + createHash("sha256") + .update(JSON.stringify(["test-bot", "thread-1", "org-a"])) + .digest("hex"), + ); + // The name a thread's query had before organizations joined it. + assert.equal(none, threadOf("test-bot", "thread-1")); + for (const request of claude.requests) + assert.equal( + (request.body.forwardedProps as Record) + .openbotOrganization, + undefined, + "the organization is not sent on to the Claude engine", + ); + } finally { + await claude.close(); + } +}); + test("A run that hands a call back to the surface keeps its Claude query for the result", async () => { const claude = await claudeEngine(() => fixture("tool")); try { From 95cdff180ba125b879a65795bec4d0dd863c2719 Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 14:14:58 -0800 Subject: [PATCH 2/7] LangGraph harness: call the model with the run's own gateway token when the server sends one One harness serves every organization of an OpenBot, and the key in its environment is the deployment's gateway token, which means one organization. Where more than one organization is allowed the server refuses that token, so every model call has to say which organization it is for. The server now sends the harness it runs the token it mints for each run, in the run's organization (forwardedProps.openbotModelRoute), as it already does for the LangGraph TypeScript engine. The harness calls an OpenAI or Anthropic model with that token as its key; a plan's route carries none, and a run without a route keeps the key in the environment. The route and openbotOrganization are server props, kept out of graph state and never passed to the model. fake_provider can now write down the key each request presented. --- agent-langgraph-agui/src/main.py | 16 +++++++ agent-langgraph-agui/src/tool_runtime.py | 22 +++++++++ .../tests/test_model_choice.py | 45 +++++++++++++++++++ shared/harness_contract.py | 16 ++++++- 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/agent-langgraph-agui/src/main.py b/agent-langgraph-agui/src/main.py index fafe9cf2be..4a236a8f9b 100644 --- a/agent-langgraph-agui/src/main.py +++ b/agent-langgraph-agui/src/main.py @@ -260,6 +260,7 @@ def _model(plan_models=None): model, **_google_genai_kwargs(prefix), **_provider_kwargs(prefix, named, run.effort), + **_gateway_key(run, prefix), **_retries(run), ) return init_chat_model( @@ -267,10 +268,25 @@ def _model(plan_models=None): model_provider=provider, **_google_genai_kwargs(provider), **_provider_kwargs(provider, model, run.effort), + **_gateway_key(run, provider), **_retries(run), ) +def _gateway_key(run, provider: str) -> dict: + """The run's own model gateway token as the key, when the server sent one. + + One harness serves every organization of an OpenBot, and the deployment's gateway token in the + environment means one organization only. The server sends each run a token for its own + organization (`openbotModelRoute`), so the call goes out on that organization's AI connection. + The address stays the configured one: the server sends a token only to a harness it runs, which + reaches the model through its gateway. + """ + if not run.gateway_token or provider not in ("openai", "anthropic"): + return {} + return {"api_key": run.gateway_token} + + async def _plan_model(): """`_model`, given the plan's listing when it has to choose a plan's model by itself.""" store = (os.environ.get("CHATGPT_AUTH_FILE") or "").strip() diff --git a/agent-langgraph-agui/src/tool_runtime.py b/agent-langgraph-agui/src/tool_runtime.py index 05bcd4e9d6..5eda2e6d3f 100644 --- a/agent-langgraph-agui/src/tool_runtime.py +++ b/agent-langgraph-agui/src/tool_runtime.py @@ -44,6 +44,9 @@ class RunTools: # The server retries this run itself (`openbotSendOnce`): the model is asked once per step and # a failure is handed back, where the model's own client would have retried it twice first. send_once: bool = False + # The model gateway's token for this run, in its organization (`openbotModelRoute.token`); + # empty when the server sent none, and the deployment's own key applies. + gateway_token: str = field(default="", repr=False) # The reasoning efforts the server sends, in its own spelling. Anything else is not an effort. @@ -56,13 +59,31 @@ class RunTools: "openbotDeploymentTools", "openbotModel", "openbotSendOnce", + "openbotModelRoute", + "openbotOrganization", "openbot_run", "openbot_deployment_tools", "openbot_model", "openbot_send_once", + "openbot_model_route", + "openbot_organization", } +def run_gateway_token(props) -> str: + """The model gateway's token the server minted for this run, or an empty string. + + One engine serves every organization of an OpenBot, and the gateway answers a call on the AI + connection of the organization its token names. The server sends a run's own token as + `openbotModelRoute: {"via": "openai" | "anthropic", "token": ...}`; a plan's route carries none. + """ + route = props.get("openbotModelRoute") + if not isinstance(route, dict) or route.get("via") not in ("openai", "anthropic"): + return "" + token = route.get("token") + return token.strip() if isinstance(token, str) else "" + + def run_model(props) -> tuple[str, str]: """The `(model, effort)` the server chose for this run, or empty strings. @@ -376,6 +397,7 @@ async def run(self, input: RunAgentInput): model=model, effort=effort, send_once=props.get("openbotSendOnce") is True, + gateway_token=run_gateway_token(props), ) # The maintained endpoint clones this subclass per request. ContextVar # also isolates graph tasks across concurrent requests and resets on diff --git a/agent-langgraph-agui/tests/test_model_choice.py b/agent-langgraph-agui/tests/test_model_choice.py index 9e9fa4e63b..e56311f4fb 100644 --- a/agent-langgraph-agui/tests/test_model_choice.py +++ b/agent-langgraph-agui/tests/test_model_choice.py @@ -799,6 +799,51 @@ async def test_without_a_server_choice_the_deployment_model_answers(monkeypatch) assert "reasoning_effort" not in body +# One harness serves every organization of an OpenBot, and the key in its environment is the +# deployment's gateway token, which means one organization. A run the server sends its own token +# (`openbotModelRoute`) calls the model with that token, so the gateway answers it on the AI +# connection of the run's own organization. A run with none keeps the deployment's key. +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "via", "key_name", "base_name", "base_suffix"), + [ + ("openai", "openai", "OPENAI_API_KEY", "OPENAI_BASE_URL", "/v1"), + ("anthropic", "anthropic", "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", ""), + ], +) +async def test_a_runs_own_gateway_token_is_the_key_it_calls_the_model_with( + monkeypatch, provider, via, key_name, base_name, base_suffix +): + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + monkeypatch.setenv("BOT_PROVIDER", provider) + monkeypatch.setenv(key_name, "deployment-gateway-token") + monkeypatch.setenv(base_name, f"{base}{base_suffix}") + monkeypatch.setenv("BOT_MODEL", "configured-model") + body = run_body() + body["forwardedProps"]["openbotModelRoute"] = { + "via": via, + "token": "obgw_run-token-org-b", + } + body["forwardedProps"]["openbotOrganization"] = "org-b" + finished(await run(body)) + # A plan's route carries no token, and a run without a route keeps the deployment's key. + plan = run_body() + plan["forwardedProps"]["openbotModelRoute"] = {"via": "claude-plan"} + finished(await run(plan)) + finished(await run(run_body())) + + assert keys == [ + "obgw_run-token-org-b", + "deployment-gateway-token", + "deployment-gateway-token", + ] + assert len(bodies) == 3 + # The route and the organization are the server's, never passed on to the model. + assert "obgw_run-token-org-b" not in json.dumps(bodies) + assert "org-b" not in json.dumps(bodies) + + @pytest.mark.parametrize( ("model", "expected"), [ diff --git a/shared/harness_contract.py b/shared/harness_contract.py index ef54cad350..c13a918a0f 100644 --- a/shared/harness_contract.py +++ b/shared/harness_contract.py @@ -107,11 +107,13 @@ def anthropic_environment(base: str, model: str = "claude-sonnet-5-5") -> dict[s @contextmanager -def fake_provider() -> Iterator[tuple[str, list[dict]]]: +def fake_provider(keys: list[str] | None = None) -> Iterator[tuple[str, list[dict]]]: """An OpenAI- and Anthropic-compatible model on localhost that answers "ok" and records bodies. Yields `(base_url, bodies)`: every request body in arrival order. Serves Chat Completions, Responses and Messages, streamed and not, since which one a framework uses is its own choice. + Given `keys`, also writes down the key each request presented, as OpenAI's SDK sends it + (`Authorization: Bearer`) or Anthropic's (`x-api-key`). """ import uvicorn from fastapi import FastAPI, Request @@ -120,6 +122,18 @@ def fake_provider() -> Iterator[tuple[str, list[dict]]]: bodies: list[dict] = [] app = FastAPI() + if keys is not None: + + @app.middleware("http") + async def presented(request: Request, call_next): + bearer = request.headers.get("authorization", "") + keys.append( + bearer[len("Bearer ") :] + if bearer.startswith("Bearer ") + else request.headers.get("x-api-key", "") + ) + return await call_next(request) + def sse(lines): async def stream(): for line in lines: From 9c6cca982ee113431a08fa1140425e77f7b27c78 Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 14:48:11 -0800 Subject: [PATCH 3/7] LangGraph harness: read the run's gateway token where a run's settings may not carry one The compose probe stands in for the run's settings with an object that has only a model and an effort. Read the token as _retries reads send_once, so a settings object without it means no token. --- agent-langgraph-agui/src/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/agent-langgraph-agui/src/main.py b/agent-langgraph-agui/src/main.py index 4a236a8f9b..be41dad66a 100644 --- a/agent-langgraph-agui/src/main.py +++ b/agent-langgraph-agui/src/main.py @@ -282,9 +282,10 @@ def _gateway_key(run, provider: str) -> dict: The address stays the configured one: the server sends a token only to a harness it runs, which reaches the model through its gateway. """ - if not run.gateway_token or provider not in ("openai", "anthropic"): + token = getattr(run, "gateway_token", "") + if not isinstance(token, str) or not token or provider not in ("openai", "anthropic"): return {} - return {"api_key": run.gateway_token} + return {"api_key": token} async def _plan_model(): From e3f413284271c3c9a8f4ad5ea389b805b8aaf342 Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 16:20:47 -0800 Subject: [PATCH 4/7] Every harness calls the model gateway with its run's own token, in its organization One harness serves every organization of an OpenBot, and the key in its environment is the deployment's gateway token, which names one organization. Where more than one organization is allowed the gateway refuses that token, so the server sends each run a token of its own (forwardedProps.openbotModelRoute), and every harness now calls the gateway with it for that run. - Python harnesses (CrewAI, LlamaIndex, Agno, LangChain, Google ADK, Pydantic AI, Microsoft Agent Framework, Strands, AG2, Langroid): shared/run_model_key.py. RunModelKey reads the token from the run's body, takes the server's props out so no framework holds the token, and keeps it for the run; every httpx and httpx2 request to the gateway's path then carries it in place of the environment's key. The frameworks build their model clients once, at start, so the HTTP request is the one place they all meet. - Claude Agent SDK: Claude Code makes the calls from a process that lives across runs, so a query on the gateway is pointed back at the harness on a secret loopback path, which passes each call on with the newest token a run of that thread brought (src/run_model_proxy.py). - Mastra: the token arrives as the OpenBot model route context entry, and a run with one answers on a model that calls with it. - Antigravity reaches Gemini with Google's own key, never the gateway; its server props are still taken out. Only to the gateway: a request to any other address keeps its own key, and a run with no token keeps the environment's. This replaces the LangGraph harness's own reading of the token in the previous commit. Each harness has a test that calls a local stand-in for the gateway, at the gateway's paths, with a run that brings a token and one that does not, on both wires; each fails on main. --- agent-adk/Dockerfile | 2 +- agent-adk/src/main.py | 3 + agent-adk/tests/test_conversation_contract.py | 21 ++ agent-ag2/Dockerfile | 2 +- agent-ag2/src/main.py | 3 + agent-ag2/tests/test_conversation_contract.py | 21 ++ agent-agno/Dockerfile | 2 +- agent-agno/src/main.py | 3 + .../tests/test_conversation_contract.py | 21 ++ agent-antigravity/Dockerfile | 2 +- agent-antigravity/src/main.py | 4 + .../tests/test_conversation_contract.py | 14 ++ agent-claude-sdk/Dockerfile | 2 + agent-claude-sdk/src/adapter.py | 11 +- agent-claude-sdk/src/main.py | 13 +- agent-claude-sdk/src/run_model_proxy.py | 144 ++++++++++++++ agent-claude-sdk/tests/test_run_model_key.py | 141 ++++++++++++++ agent-crewai/Dockerfile | 2 +- agent-crewai/src/main.py | 3 + .../tests/test_conversation_contract.py | 23 +++ agent-langgraph-agui/Dockerfile | 2 +- agent-langgraph-agui/src/main.py | 20 +- agent-langgraph-agui/src/tool_runtime.py | 22 --- .../tests/test_conversation_contract.py | 21 ++ .../tests/test_model_choice.py | 45 ----- agent-langroid/Dockerfile | 2 +- agent-langroid/src/main.py | 3 + .../tests/test_conversation_contract.py | 21 ++ agent-llamaindex/Dockerfile | 2 +- agent-llamaindex/src/main.py | 3 + .../tests/test_conversation_contract.py | 21 ++ agent-mastra/src/mastra/index.test.ts | 106 +++++++++- agent-mastra/src/mastra/index.ts | 63 +++++- agent-microsoft/Dockerfile | 2 +- agent-microsoft/src/main.py | 3 + .../tests/test_conversation_contract.py | 30 +++ agent-pydantic-ai/Dockerfile | 2 +- agent-pydantic-ai/src/main.py | 3 + .../tests/test_conversation_contract.py | 22 +++ agent-strands/Dockerfile | 2 +- agent-strands/src/main.py | 3 + .../tests/test_conversation_contract.py | 18 ++ shared/harness_contract.py | 98 +++++++++- shared/run_model_key.py | 181 ++++++++++++++++++ 44 files changed, 1027 insertions(+), 105 deletions(-) create mode 100644 agent-claude-sdk/src/run_model_proxy.py create mode 100644 agent-claude-sdk/tests/test_run_model_key.py create mode 100644 shared/run_model_key.py diff --git a/agent-adk/Dockerfile b/agent-adk/Dockerfile index fb990dbb85..65266eee30 100644 --- a/agent-adk/Dockerfile +++ b/agent-adk/Dockerfile @@ -9,7 +9,7 @@ COPY agent-adk/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-adk/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4208 EXPOSE 4208 diff --git a/agent-adk/src/main.py b/agent-adk/src/main.py index 32b4c335d7..af1ac2e007 100644 --- a/agent-adk/src/main.py +++ b/agent-adk/src/main.py @@ -15,6 +15,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui.core import ( AssistantMessage, @@ -343,6 +344,8 @@ def _model_id() -> str: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-adk/tests/test_conversation_contract.py b/agent-adk/tests/test_conversation_contract.py index 67d8ed9acc..19969878e8 100644 --- a/agent-adk/tests/test_conversation_contract.py +++ b/agent-adk/tests/test_conversation_contract.py @@ -29,6 +29,7 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -119,6 +120,26 @@ async def turns(): return asyncio.run(asyncio.wait_for(turns(), timeout=60)) +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. Gemini goes to Google with its own key, never the gateway. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + monkeypatch.setenv("OTEL_SDK_DISABLED", "true") + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): with fake_provider() as (base, bodies): diff --git a/agent-ag2/Dockerfile b/agent-ag2/Dockerfile index df952e81a6..fbce0838fa 100644 --- a/agent-ag2/Dockerfile +++ b/agent-ag2/Dockerfile @@ -9,7 +9,7 @@ COPY agent-ag2/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-ag2/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4210 EXPOSE 4210 diff --git a/agent-ag2/src/main.py b/agent-ag2/src/main.py index 81da87903b..0d705c2560 100644 --- a/agent-ag2/src/main.py +++ b/agent-ag2/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings, openai_api +from run_model_key import RunModelKey from ag2 import Agent from ag2.ag_ui import AGUIStream @@ -55,6 +56,8 @@ def _config() -> AnthropicConfig | OpenAICompletionsConfig | OpenAIResponsesConf stream = AGUIStream(agent) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-ag2/tests/test_conversation_contract.py b/agent-ag2/tests/test_conversation_contract.py index 09e40c0290..13dd3878f2 100644 --- a/agent-ag2/tests/test_conversation_contract.py +++ b/agent-ag2/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-agno/Dockerfile b/agent-agno/Dockerfile index 9e97f4343d..e92b4dad67 100644 --- a/agent-agno/Dockerfile +++ b/agent-agno/Dockerfile @@ -9,7 +9,7 @@ COPY agent-agno/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-agno/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4203 EXPOSE 4203 diff --git a/agent-agno/src/main.py b/agent-agno/src/main.py index 145f905487..d6b167042a 100644 --- a/agent-agno/src/main.py +++ b/agent-agno/src/main.py @@ -14,6 +14,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from agno.agent import Agent from agno.models.litellm import LiteLLM @@ -81,6 +82,8 @@ def _sampling() -> dict: ) app = AgentOS(agents=[agent], interfaces=[AGUI(agent=agent)]).get_app() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-agno/tests/test_conversation_contract.py b/agent-agno/tests/test_conversation_contract.py index 5ded715a18..f46f875766 100644 --- a/agent-agno/tests/test_conversation_contract.py +++ b/agent-agno/tests/test_conversation_contract.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -48,6 +49,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/agui", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-antigravity/Dockerfile b/agent-antigravity/Dockerfile index cd96befd7d..83e3583e90 100644 --- a/agent-antigravity/Dockerfile +++ b/agent-antigravity/Dockerfile @@ -12,7 +12,7 @@ COPY agent-antigravity/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-antigravity/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4214 EXPOSE 4214 diff --git a/agent-antigravity/src/main.py b/agent-antigravity/src/main.py index 28f453da01..33a1bced38 100644 --- a/agent-antigravity/src/main.py +++ b/agent-antigravity/src/main.py @@ -36,6 +36,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import SPEC, bot_settings +from run_model_key import RunModelKey from ag_ui_antigravity import create_antigravity_app from fastapi import Request @@ -95,6 +96,9 @@ def _agent() -> OpenBotAntigravityAgent: app = create_antigravity_app(_agent(), path="/") +# Gemini is reached with Google's own key, never OpenBot's model gateway, so no run's token is ever +# sent anywhere from here; the server's props still never reach the agent. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-antigravity/tests/test_conversation_contract.py b/agent-antigravity/tests/test_conversation_contract.py index d54bb504d1..50170d674c 100644 --- a/agent-antigravity/tests/test_conversation_contract.py +++ b/agent-antigravity/tests/test_conversation_contract.py @@ -30,10 +30,12 @@ from harness_contract import ( EARLIER_QUESTION, LATEST_QUESTION, + RUN_KEY, STANDING_ROLE, conversation, missing_from, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -177,6 +179,18 @@ def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, assert STANDING_ROLE in _system_instruction(body), "the standing role is a system instruction" +def test_a_runs_gateway_token_never_reaches_gemini(monkeypatch, tmp_path): + """Gemini is reached with Google's own key: the run's token for OpenBot's gateway goes nowhere. + + See `shared/run_model_key.py`. + """ + with fresh_bot(monkeypatch, tmp_path) as run: + body = run(with_run_key(conversation(), "openai")) + assert missing_from(body) == [] + assert RUN_KEY not in json.dumps(run.bodies) + assert "contract-organization" not in json.dumps(run.bodies) + + def test_a_live_session_is_not_sent_the_history_twice(monkeypatch, tmp_path): first = conversation() with fresh_bot(monkeypatch, tmp_path) as run: diff --git a/agent-claude-sdk/Dockerfile b/agent-claude-sdk/Dockerfile index 6ace4d928d..90134366a6 100644 --- a/agent-claude-sdk/Dockerfile +++ b/agent-claude-sdk/Dockerfile @@ -9,6 +9,8 @@ COPY agent-claude-sdk/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-claude-sdk/src ./src +# A run's own key to OpenBot's model gateway, shared by every Python harness (src/run_model_proxy.py). +COPY shared/run_model_key.py /shared/ ENV PORT=4212 EXPOSE 4212 diff --git a/agent-claude-sdk/src/adapter.py b/agent-claude-sdk/src/adapter.py index b909bcd0a9..6794b89cd3 100644 --- a/agent-claude-sdk/src/adapter.py +++ b/agent-claude-sdk/src/adapter.py @@ -53,6 +53,7 @@ from claude_agent_sdk.types import StreamEvent from .replay import RESULT_CHARS, cut_note, run_replayed_query, system_instructions +from .run_model_proxy import query_environment, refresh logger = logging.getLogger(__name__) @@ -827,7 +828,13 @@ def build_options(self, input_data=None, thread_id=None): if chosen: # After the upstream merge, so the server's choice wins over a forwarded `model`. options.model, options.effort = chosen - options.env = {**(options.env or {}), UNKNOWN_MODEL_WINDOW: "1"} + # On the gateway, the query's calls come back through this harness with the newest run + # token of its thread. See run_model_proxy.py. + options.env = { + **(options.env or {}), + UNKNOWN_MODEL_WINDOW: "1", + **query_environment(thread_id), + } if once: options.env[MAX_RETRIES] = "0" options.max_buffer_size = MESSAGE_BUFFER_BYTES @@ -1291,6 +1298,8 @@ async def _pump_query(self, input_data, turn, fit_results=False): async def run(self, input_data): thread_id, run_id = input_data.thread_id, input_data.run_id + # A query this run carries on sends its calls with this run's token from now. + refresh(thread_id) lock = self._segment_locks.setdefault(thread_id, asyncio.Lock()) async with lock: answer_only = ( diff --git a/agent-claude-sdk/src/main.py b/agent-claude-sdk/src/main.py index dced048ac1..a5490a02ef 100644 --- a/agent-claude-sdk/src/main.py +++ b/agent-claude-sdk/src/main.py @@ -17,6 +17,8 @@ from pydantic import BaseModel, Field from .adapter import OpenBotClaudeAgentAdapter +from .run_model_proxy import PROXY_PATH, forward +from run_model_key import RunModelKey # On the path run_model_proxy puts shared/ on. TOKEN_HEADER = "x-openbot-agent-token" @@ -44,11 +46,15 @@ def _refuse_both_credentials() -> None: _refuse_both_credentials() app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_proxy.py. +app.add_middleware(RunModelKey) @app.middleware("http") async def refuse_without_the_server_token(request: Request, call_next): - if request.url.path != "/health": + # Claude Code's own model calls come back here on a secret path of their query's, from this + # container's loopback, and carry no server token. See run_model_proxy.py. + if request.url.path != "/health" and not request.url.path.startswith(PROXY_PATH): expected = (os.environ.get("MANAGED_AGENT_TOKEN") or "").strip() offered = (request.headers.get(TOKEN_HEADER) or "").strip() if not expected or not hmac.compare_digest(offered.encode("utf-8"), expected.encode("utf-8")): @@ -61,6 +67,11 @@ async def health(): return {"ok": True, "harness": "claude-agent-sdk"} +@app.api_route(PROXY_PATH + "{secret}/{path:path}", methods=["GET", "POST"]) +async def run_model(request: Request, secret: str, path: str): + return await forward(request, secret, path) + + add_claude_fastapi_endpoint( app=app, adapter=OpenBotClaudeAgentAdapter(name="openbot"), diff --git a/agent-claude-sdk/src/run_model_proxy.py b/agent-claude-sdk/src/run_model_proxy.py new file mode 100644 index 0000000000..68e24b149e --- /dev/null +++ b/agent-claude-sdk/src/run_model_proxy.py @@ -0,0 +1,144 @@ +"""Claude Code's model calls on a run's own gateway token, in its organization. + +One harness serves every organization of an OpenBot. Its model calls go to the server's model +gateway, which answers each on the AI connection of the organization the caller's key names, and +the key in this container's environment names one organization. So the server sends each run a +token of its own (`openbotModelRoute`), which `RunModelKey` keeps for the run +(`shared/run_model_key.py`). + +The Python harnesses send their model calls themselves, and that module puts the token on each. +Here Claude Code sends them, from a process of its own that lives across runs: a query waits on a +tool's result while the person decides, and the next run carries it on. A key fixed in its +environment would be the first run's, past its ten minutes. So a query on the gateway is pointed at +this harness instead, on a path only it knows (`/_run-model/`), and every call it makes is +passed on to the gateway with the newest token any run of that thread brought. + +Only where the configured address is the gateway's (`/api/model-provider/`) and a run brought a +token. Otherwise Claude Code is given what it always was. +""" + +import os +import secrets +import sys +import time +from pathlib import Path +from urllib.parse import urlsplit + +# The module every Python harness shares: beside this harness in the repository, at /shared in the +# image the Dockerfile builds. +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) +from run_model_key import GATEWAY_PATH, run_key # noqa: E402 + +PROXY_PATH = "/_run-model/" + +# A thread's slot outlives a run: its query may wait on a person for hours. One a day old is gone. +SLOT_SECONDS = 24 * 60 * 60 + + +class _Slot: + __slots__ = ("secret", "key", "at") + + def __init__(self, secret: str, key: str): + self.secret = secret + self.key = key + self.at = time.monotonic() + + +_by_thread: dict[str, _Slot] = {} +_by_secret: dict[str, _Slot] = {} + + +def _gateway_base() -> str: + base = (os.environ.get("ANTHROPIC_BASE_URL") or "").strip().rstrip("/") + return base if base and GATEWAY_PATH in urlsplit(base).path + "/" else "" + + +def _forget_old(now: float) -> None: + for thread, slot in list(_by_thread.items()): + if now - slot.at > SLOT_SECONDS: + del _by_thread[thread] + _by_secret.pop(slot.secret, None) + + +def query_environment(thread_id: str | None) -> dict[str, str]: + """What a new query of `thread_id` is started with, so its calls come back through here. + + Empty where the run brought no token or the configured address is not the gateway's. + """ + key = run_key() + if not key or not thread_id or not _gateway_base(): + return {} + now = time.monotonic() + _forget_old(now) + slot = _by_thread.get(thread_id) + if slot is None: + slot = _Slot(secrets.token_urlsafe(32), key) + _by_thread[thread_id] = slot + _by_secret[slot.secret] = slot + slot.key, slot.at = key, now + port = (os.environ.get("PORT") or "4212").strip() + return { + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}{PROXY_PATH}{slot.secret}", + # Claude Code needs a key to start; the one it sends is replaced on the way out. + "ANTHROPIC_API_KEY": "openbot-run-model", + } + + +def refresh(thread_id: str | None) -> None: + """A run of `thread_id` brought a newer token: the query it carries on sends that from now.""" + key = run_key() + slot = _by_thread.get(thread_id or "") + if key and slot is not None: + slot.key, slot.at = key, time.monotonic() + + +def slot_key(secret: str) -> str | None: + slot = _by_secret.get(secret) + return slot.key if slot is not None else None + + +# What is not passed on to the gateway: how this hop was made, and the key Claude Code was given. +_REQUEST_DROPPED = {"host", "content-length", "connection", "authorization", "x-api-key"} +# What is not passed back: how the gateway's answer was framed, which this hop frames again. +_RESPONSE_DROPPED = {"content-length", "transfer-encoding", "connection"} + +_client = None + + +async def forward(request, secret: str, path: str): + """One of Claude Code's model calls, passed on to the gateway with its thread's newest token.""" + import httpx + from fastapi.responses import JSONResponse, StreamingResponse + from starlette.background import BackgroundTask + + global _client + key = slot_key(secret) + base = _gateway_base() + client_host = request.client.host if request.client else "" + if not key or not base or client_host not in ("127.0.0.1", "::1"): + return JSONResponse({"error": "not found"}, status_code=404) + if _client is None: + _client = httpx.AsyncClient(timeout=httpx.Timeout(None, connect=30.0)) + target = f"{base}/{path}" + if request.url.query: + target = f"{target}?{request.url.query}" + headers = { + name: value + for name, value in request.headers.items() + if name.lower() not in _REQUEST_DROPPED + } + headers["x-api-key"] = key + upstream = _client.build_request( + request.method, target, headers=headers, content=await request.body() + ) + response = await _client.send(upstream, stream=True) + return StreamingResponse( + response.aiter_raw(), + status_code=response.status_code, + headers={ + name: value + for name, value in response.headers.items() + if name.lower() not in _RESPONSE_DROPPED + }, + background=BackgroundTask(response.aclose), + ) diff --git a/agent-claude-sdk/tests/test_run_model_key.py b/agent-claude-sdk/tests/test_run_model_key.py new file mode 100644 index 0000000000..49dbc00597 --- /dev/null +++ b/agent-claude-sdk/tests/test_run_model_key.py @@ -0,0 +1,141 @@ +"""Claude Code calls OpenBot's model gateway with the token of the run, in its organization. + +One harness serves every organization of an OpenBot, and the key in its environment is the +deployment's gateway token, which names one organization. The server sends each run a token of its +own (`openbotModelRoute`). Claude Code makes the model calls from a process of its own, so a query +on the gateway sends them back through this harness, which passes each on with the newest token a +run of that thread brought (`src/run_model_proxy.py`). A run with no token, and an address that is +not the gateway's, are given what they always were. + +These run the real Claude Code this harness ships, through the harness's own routes on a real port +(Claude Code dials it), against a local stand-in for the gateway's Anthropic wire. +""" + +import importlib +import json +import socket +import sys +import threading +import time +from pathlib import Path + +import httpx +import pytest +import uvicorn + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) +from harness_contract import ( + DEPLOYMENT_KEY, + RUN_KEY, + fake_provider, + gateway_environment, + run_events, +) + +TOKEN = "valid-test-token" +LATER_KEY = "obgw_contract-later-run-token" + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _serve(app, port): + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(0.01) + return server, thread + + +def _run(port, thread, key=None): + props = {"openbotSendOnce": True} + if key: + props["openbotModelRoute"] = {"via": "anthropic", "token": key} + props["openbotOrganization"] = "contract-organization" + response = httpx.post( + f"http://127.0.0.1:{port}/", + headers={"x-openbot-agent-token": TOKEN, "accept": "text/event-stream"}, + json={ + "threadId": thread, + "runId": f"run-{time.monotonic_ns()}", + "state": {}, + "messages": [{"id": f"user-{time.monotonic_ns()}", "role": "user", "content": "Hello?"}], + "tools": [], + "context": [], + "forwardedProps": props, + }, + timeout=120, + ) + assert response.status_code == 200 + events = run_events(response.text) + assert events[-1]["type"] == "RUN_FINISHED", events + return events + + +@pytest.fixture +def harness(monkeypatch, tmp_path): + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + for name, value in gateway_environment("anthropic")(base).items(): + monkeypatch.setenv(name, value) + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / ".claude")) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + port = _free_port() + monkeypatch.setenv("PORT", str(port)) + from src import main + + server, thread = _serve(importlib.reload(main).app, port) + try: + yield port, keys, bodies + finally: + server.should_exit = True + thread.join(timeout=10) + + +def _messages_keys(keys, start): + """The keys the Messages API calls presented since `start` (Claude Code also asks other routes).""" + return [key for key in keys[start:] if key] + + +def test_claude_code_calls_the_gateway_with_the_runs_own_key(harness): + port, keys, bodies = harness + + _run(port, "thread-org-b", RUN_KEY) + with_token = _messages_keys(keys, 0) + assert with_token and set(with_token) == {RUN_KEY}, keys + + # The next run of the thread brings a newer token; the thread's calls carry that from now. + start = len(keys) + _run(port, "thread-org-b", LATER_KEY) + later = _messages_keys(keys, start) + assert later and set(later) == {LATER_KEY}, keys + + # A run with no token: the environment's key, straight to the gateway, as before. + start = len(keys) + _run(port, "thread-without-token") + without = _messages_keys(keys, start) + assert without and set(without) == {DEPLOYMENT_KEY}, keys + + # Neither the token nor the organization is ever said to the model. + text = json.dumps(bodies) + assert RUN_KEY not in text and LATER_KEY not in text and "contract-organization" not in text + + +def test_the_way_back_in_takes_nothing_it_does_not_know(harness): + port, keys, _ = harness + # No secret this process handed out, so nothing is passed on and no key is spent. + response = httpx.post( + f"http://127.0.0.1:{port}/_run-model/made-up-secret/v1/messages", + json={"model": "claude-sonnet-5-5", "messages": []}, + timeout=30, + ) + assert response.status_code == 404 + assert keys == [] diff --git a/agent-crewai/Dockerfile b/agent-crewai/Dockerfile index e2672f4f86..ed67960f20 100644 --- a/agent-crewai/Dockerfile +++ b/agent-crewai/Dockerfile @@ -9,7 +9,7 @@ COPY agent-crewai/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-crewai/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4202 EXPOSE 4202 diff --git a/agent-crewai/src/main.py b/agent-crewai/src/main.py index feb533d7fb..943970295b 100644 --- a/agent-crewai/src/main.py +++ b/agent-crewai/src/main.py @@ -20,6 +20,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey import ag_ui_crewai.endpoint as crewai_endpoint from ag_ui.core import Message, Tool @@ -177,6 +178,8 @@ async def answer(self): app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-crewai/tests/test_conversation_contract.py b/agent-crewai/tests/test_conversation_contract.py index bfdb65eb5c..4e26eb2d21 100644 --- a/agent-crewai/tests/test_conversation_contract.py +++ b/agent-crewai/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -53,6 +54,28 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + import ag_ui_crewai.endpoint + + importlib.reload(ag_ui_crewai.endpoint) + sys.modules.pop("src.main", None) + return importlib.import_module("src.main").app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-langgraph-agui/Dockerfile b/agent-langgraph-agui/Dockerfile index 989639ec09..e5aa276331 100644 --- a/agent-langgraph-agui/Dockerfile +++ b/agent-langgraph-agui/Dockerfile @@ -9,7 +9,7 @@ COPY agent-langgraph-agui/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-langgraph-agui/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4206 EXPOSE 4206 diff --git a/agent-langgraph-agui/src/main.py b/agent-langgraph-agui/src/main.py index be41dad66a..5b81f52524 100644 --- a/agent-langgraph-agui/src/main.py +++ b/agent-langgraph-agui/src/main.py @@ -15,6 +15,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui_langgraph import add_langgraph_fastapi_endpoint from fastapi import FastAPI, Request @@ -260,7 +261,6 @@ def _model(plan_models=None): model, **_google_genai_kwargs(prefix), **_provider_kwargs(prefix, named, run.effort), - **_gateway_key(run, prefix), **_retries(run), ) return init_chat_model( @@ -268,26 +268,10 @@ def _model(plan_models=None): model_provider=provider, **_google_genai_kwargs(provider), **_provider_kwargs(provider, model, run.effort), - **_gateway_key(run, provider), **_retries(run), ) -def _gateway_key(run, provider: str) -> dict: - """The run's own model gateway token as the key, when the server sent one. - - One harness serves every organization of an OpenBot, and the deployment's gateway token in the - environment means one organization only. The server sends each run a token for its own - organization (`openbotModelRoute`), so the call goes out on that organization's AI connection. - The address stays the configured one: the server sends a token only to a harness it runs, which - reaches the model through its gateway. - """ - token = getattr(run, "gateway_token", "") - if not isinstance(token, str) or not token or provider not in ("openai", "anthropic"): - return {} - return {"api_key": token} - - async def _plan_model(): """`_model`, given the plan's listing when it has to choose a plan's model by itself.""" store = (os.environ.get("CHATGPT_AUTH_FILE") or "").strip() @@ -378,6 +362,8 @@ async def answer(state: MessagesState): graph = builder.compile(checkpointer=MemorySaver()) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-langgraph-agui/src/tool_runtime.py b/agent-langgraph-agui/src/tool_runtime.py index 5eda2e6d3f..05bcd4e9d6 100644 --- a/agent-langgraph-agui/src/tool_runtime.py +++ b/agent-langgraph-agui/src/tool_runtime.py @@ -44,9 +44,6 @@ class RunTools: # The server retries this run itself (`openbotSendOnce`): the model is asked once per step and # a failure is handed back, where the model's own client would have retried it twice first. send_once: bool = False - # The model gateway's token for this run, in its organization (`openbotModelRoute.token`); - # empty when the server sent none, and the deployment's own key applies. - gateway_token: str = field(default="", repr=False) # The reasoning efforts the server sends, in its own spelling. Anything else is not an effort. @@ -59,31 +56,13 @@ class RunTools: "openbotDeploymentTools", "openbotModel", "openbotSendOnce", - "openbotModelRoute", - "openbotOrganization", "openbot_run", "openbot_deployment_tools", "openbot_model", "openbot_send_once", - "openbot_model_route", - "openbot_organization", } -def run_gateway_token(props) -> str: - """The model gateway's token the server minted for this run, or an empty string. - - One engine serves every organization of an OpenBot, and the gateway answers a call on the AI - connection of the organization its token names. The server sends a run's own token as - `openbotModelRoute: {"via": "openai" | "anthropic", "token": ...}`; a plan's route carries none. - """ - route = props.get("openbotModelRoute") - if not isinstance(route, dict) or route.get("via") not in ("openai", "anthropic"): - return "" - token = route.get("token") - return token.strip() if isinstance(token, str) else "" - - def run_model(props) -> tuple[str, str]: """The `(model, effort)` the server chose for this run, or empty strings. @@ -397,7 +376,6 @@ async def run(self, input: RunAgentInput): model=model, effort=effort, send_once=props.get("openbotSendOnce") is True, - gateway_token=run_gateway_token(props), ) # The maintained endpoint clones this subclass per request. ContextVar # also isolates graph tasks across concurrent requests and resets on diff --git a/agent-langgraph-agui/tests/test_conversation_contract.py b/agent-langgraph-agui/tests/test_conversation_contract.py index 643dbc4abd..e4443fc691 100644 --- a/agent-langgraph-agui/tests/test_conversation_contract.py +++ b/agent-langgraph-agui/tests/test_conversation_contract.py @@ -24,6 +24,7 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -87,6 +88,26 @@ def _system_text(body: dict) -> str: return "\n".join(parts) +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): with fake_provider() as (base, bodies): diff --git a/agent-langgraph-agui/tests/test_model_choice.py b/agent-langgraph-agui/tests/test_model_choice.py index e56311f4fb..9e9fa4e63b 100644 --- a/agent-langgraph-agui/tests/test_model_choice.py +++ b/agent-langgraph-agui/tests/test_model_choice.py @@ -799,51 +799,6 @@ async def test_without_a_server_choice_the_deployment_model_answers(monkeypatch) assert "reasoning_effort" not in body -# One harness serves every organization of an OpenBot, and the key in its environment is the -# deployment's gateway token, which means one organization. A run the server sends its own token -# (`openbotModelRoute`) calls the model with that token, so the gateway answers it on the AI -# connection of the run's own organization. A run with none keeps the deployment's key. -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("provider", "via", "key_name", "base_name", "base_suffix"), - [ - ("openai", "openai", "OPENAI_API_KEY", "OPENAI_BASE_URL", "/v1"), - ("anthropic", "anthropic", "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", ""), - ], -) -async def test_a_runs_own_gateway_token_is_the_key_it_calls_the_model_with( - monkeypatch, provider, via, key_name, base_name, base_suffix -): - keys: list[str] = [] - with fake_provider(keys) as (base, bodies): - monkeypatch.setenv("BOT_PROVIDER", provider) - monkeypatch.setenv(key_name, "deployment-gateway-token") - monkeypatch.setenv(base_name, f"{base}{base_suffix}") - monkeypatch.setenv("BOT_MODEL", "configured-model") - body = run_body() - body["forwardedProps"]["openbotModelRoute"] = { - "via": via, - "token": "obgw_run-token-org-b", - } - body["forwardedProps"]["openbotOrganization"] = "org-b" - finished(await run(body)) - # A plan's route carries no token, and a run without a route keeps the deployment's key. - plan = run_body() - plan["forwardedProps"]["openbotModelRoute"] = {"via": "claude-plan"} - finished(await run(plan)) - finished(await run(run_body())) - - assert keys == [ - "obgw_run-token-org-b", - "deployment-gateway-token", - "deployment-gateway-token", - ] - assert len(bodies) == 3 - # The route and the organization are the server's, never passed on to the model. - assert "obgw_run-token-org-b" not in json.dumps(bodies) - assert "org-b" not in json.dumps(bodies) - - @pytest.mark.parametrize( ("model", "expected"), [ diff --git a/agent-langroid/Dockerfile b/agent-langroid/Dockerfile index b34f1675db..c6d82df2bc 100644 --- a/agent-langroid/Dockerfile +++ b/agent-langroid/Dockerfile @@ -9,7 +9,7 @@ COPY agent-langroid/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-langroid/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4209 EXPOSE 4209 diff --git a/agent-langroid/src/main.py b/agent-langroid/src/main.py index fbfa838ed8..3655b6a883 100644 --- a/agent-langroid/src/main.py +++ b/agent-langroid/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings, openai_api +from run_model_key import RunModelKey import litellm from ag_ui_langroid import create_langroid_app @@ -90,6 +91,8 @@ def _model_id() -> str: ) app = create_langroid_app(FrontendToolsAgent(name="openbot", agent=agent)) +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-langroid/tests/test_conversation_contract.py b/agent-langroid/tests/test_conversation_contract.py index 8a5e2f45cc..83dc1aa15c 100644 --- a/agent-langroid/tests/test_conversation_contract.py +++ b/agent-langroid/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-llamaindex/Dockerfile b/agent-llamaindex/Dockerfile index 7aca05b66e..7e94d0976f 100644 --- a/agent-llamaindex/Dockerfile +++ b/agent-llamaindex/Dockerfile @@ -9,7 +9,7 @@ COPY agent-llamaindex/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-llamaindex/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4204 EXPOSE 4204 diff --git a/agent-llamaindex/src/main.py b/agent-llamaindex/src/main.py index 48b81e5189..8b618f5419 100644 --- a/agent-llamaindex/src/main.py +++ b/agent-llamaindex/src/main.py @@ -13,6 +13,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey import litellm from fastapi import FastAPI, Request @@ -56,6 +57,8 @@ def _llm() -> LiteLLM: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-llamaindex/tests/test_conversation_contract.py b/agent-llamaindex/tests/test_conversation_contract.py index 334e7d6e20..0d2446c703 100644 --- a/agent-llamaindex/tests/test_conversation_contract.py +++ b/agent-llamaindex/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/run", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-mastra/src/mastra/index.test.ts b/agent-mastra/src/mastra/index.test.ts index 236d813d9e..759f53f395 100644 --- a/agent-mastra/src/mastra/index.test.ts +++ b/agent-mastra/src/mastra/index.test.ts @@ -82,7 +82,7 @@ async function configuredModelId(botModel: string | undefined) { "-e", [ 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', - 'const model = mastra.getAgent("openbot").model;', + 'const model = await mastra.getAgent("openbot").getModel();', "console.log(JSON.stringify({ modelId: model.modelId }));", ].join("\n"), ], @@ -161,7 +161,7 @@ async function providerStartup(botProvider: string) { "-e", [ 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', - 'const model = mastra.getAgent("openbot").model;', + 'const model = await mastra.getAgent("openbot").getModel();', "console.log(JSON.stringify({ modelId: model.modelId }));", ].join("\n"), ], @@ -568,3 +568,105 @@ describe("OpenBot Mastra provider requests", () => { }, 15_000); } }); + +/** + * One harness serves every organization of an OpenBot, and the key in its environment is the + * deployment's gateway token, which names one organization. A run the server sends its own token + * (the `OpenBot model route` context entry) calls OpenBot's model gateway with that token; a run + * without one keeps the environment's key. + */ +describe("OpenBot Mastra on OpenBot's model gateway", () => { + for (const wire of ["openai", "anthropic"] as const) { + test(`a run's own token is the key it calls the ${wire} wire with`, async () => { + const seen: (string | null)[] = []; + const gateway = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const body = await request.json(); + seen.push( + wire === "anthropic" + ? request.headers.get("x-api-key") + : request.headers.get("authorization"), + ); + if (wire === "anthropic") + return Response.json({ + id: "msg", + type: "message", + role: "assistant", + model: body.model, + content: [{ type: "text", text: "hello" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); + return Response.json({ + id: "chat", + object: "chat.completion", + created: 0, + model: body.model, + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "hello" }, + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }); + const base = `${gateway.url.toString().replace(/\/+$/, "")}/api/model-provider${wire === "anthropic" ? "/anthropic" : "/v1"}`; + try { + const { stdout, stderr, exitCode } = await runProbe( + [ + Bun.argv[0], + "--no-env-file", + "-e", + [ + // Mastra's own RequestContext, from the harness's packages, as its server builds one. + 'const { RequestContext } = await import(Bun.resolveSync("@mastra/core/request-context", process.cwd() + "/agent-mastra/src/mastra"));', + 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', + "const requestContext = new RequestContext();", + 'requestContext.set("ag-ui", { context: [{ description: "OpenBot model route", value: JSON.stringify({ via: ' + + JSON.stringify(wire) + + ', token: "obgw_run-token-org-b" }) }] });', + 'const agent = mastra.getAgent("openbot");', + 'const withToken = await agent.generate("Say hello", { requestContext });', + 'const without = await agent.generate("Say hello");', + "console.log(JSON.stringify({ texts: [withToken.text, without.text] }));", + ].join("\n"), + ], + { + PATH: process.env.PATH ?? "/opt/homebrew/bin:/usr/bin:/bin", + MASTRA_TELEMETRY_DISABLED: "true", + DO_NOT_TRACK: "1", + NODE_ENV: "test", + BOT_PROVIDER: wire, + BOT_MODEL: wire === "anthropic" ? "claude-sonnet-5-5" : "gpt-5.5", + ANTHROPIC_API_KEY: + wire === "anthropic" ? "deployment-gateway-token" : "", + OPENAI_API_KEY: + wire === "anthropic" ? "" : "deployment-gateway-token", + ANTHROPIC_BASE_URL: wire === "anthropic" ? base : "", + OPENAI_BASE_URL: wire === "anthropic" ? "" : base, + }, + 10_000, + ); + if (exitCode !== 0) + throw new Error( + `gateway probe exited ${exitCode}\n${stdout}\n${stderr}`, + ); + expect(stdout).toContain('"texts":["hello","hello"]'); + const expected = (key: string) => + wire === "anthropic" ? key : `Bearer ${key}`; + expect(seen).toEqual([ + expected("obgw_run-token-org-b"), + expected("deployment-gateway-token"), + ]); + } finally { + await gateway.stop(true); + } + }, 15_000); + } +}); diff --git a/agent-mastra/src/mastra/index.ts b/agent-mastra/src/mastra/index.ts index ed5625b619..543454a7d5 100644 --- a/agent-mastra/src/mastra/index.ts +++ b/agent-mastra/src/mastra/index.ts @@ -35,7 +35,7 @@ const SUPPORTED_PROVIDERS = new Set(["openai", "anthropic"]); * and one it has that this harness has no module for, used to fall through to the OpenAI branch * below and answer with a model the deployment never chose. */ -async function buildModel() { +async function buildModel(runKey?: string) { const settings = botSettings("agent-mastra"); const providerName = settings.provider; const spec = providerSpec(providerName); @@ -56,12 +56,16 @@ async function buildModel() { // Provider modules create default clients at import, which reject Compose's empty overrides. if (!baseURL) delete process.env[baseVariable]; + // The run's own gateway token, only where the address is OpenBot's model gateway. + const key = runKey && baseURL && onTheGateway(baseURL) ? runKey : undefined; + if (spec.id === "anthropic") { const { createAnthropic } = await import("@ai-sdk/anthropic"); // Other harnesses accept an Anthropic origin; AI SDK expects the /v1 API prefix. const origin = (baseURL || "https://api.anthropic.com").replace(/\/+$/, ""); return createAnthropic({ baseURL: origin.endsWith("/v1") ? origin : `${origin}/v1`, + ...(key ? { apiKey: key } : {}), })(model); } const { createOpenAI } = await import("@ai-sdk/openai"); @@ -70,9 +74,11 @@ async function buildModel() { baseURL?.replace(/\/+$/, "") !== "https://api.openai.com/v1"; const openai = createOpenAI({ baseURL: baseURL || "https://api.openai.com/v1", - apiKey: compatible - ? apiKeyOrPlaceholder(process.env[spec.keyVariable]) - : undefined, + apiKey: + key ?? + (compatible + ? apiKeyOrPlaceholder(process.env[spec.keyVariable]) + : undefined), }); // Compatible endpoints commonly expose Chat Completions; OpenAI keeps its Responses default. return compatible ? openai.chat(model) : openai(model); @@ -161,11 +167,58 @@ export function buildOpenBotInstructions({ return [openbotBaseInstructions, ...instructions].join("\n\n"); } +/** OpenBot's model gateway: its OpenAI and Anthropic wires are under this path on the server. */ +const GATEWAY_PATH = "/api/model-provider/"; + +function onTheGateway(baseURL: string): boolean { + try { + return `${new URL(baseURL.trim()).pathname.replace(/\/+$/, "")}/`.includes( + GATEWAY_PATH, + ); + } catch { + return false; + } +} + +/** + * The run's own gateway token, in its organization, which the server sends a harness it runs as + * the `OpenBot model route` context entry: `{"via": "openai" | "anthropic", "token": ...}`. + * + * One harness serves every organization of an OpenBot, and the key in its environment is the + * deployment's gateway token, which names one organization. A run with none keeps that key. + */ +export function runKeyFrom( + requestContext?: OpenBotInstructionArgs["requestContext"], +): string | undefined { + const entry = agUiContextEntries(requestContext).find( + (candidate): candidate is ContextEntry => + isUsableEntry(candidate) && + candidate.description === "OpenBot model route", + ); + if (!entry) return undefined; + try { + const route = JSON.parse(entry.value) as { via?: unknown; token?: unknown }; + if (route.via !== "openai" && route.via !== "anthropic") return undefined; + return typeof route.token === "string" && route.token.trim() + ? route.token.trim() + : undefined; + } catch { + return undefined; + } +} + +// Built at start, so a provider this Bot cannot drive is refused before it serves anything. +const deploymentModel = await buildModel(); + const openbot = new Agent({ id: "openbot", name: "openbot", instructions: buildOpenBotInstructions, - model: await buildModel(), + // A run that brought its own gateway token answers on a model that calls with it. + model: async ({ requestContext }) => { + const key = runKeyFrom(requestContext); + return key ? buildModel(key) : deploymentModel; + }, }); /** The one header OpenBot's server sends, compared without leaking length through timing. */ diff --git a/agent-microsoft/Dockerfile b/agent-microsoft/Dockerfile index a740f7c78f..a2fee714b4 100644 --- a/agent-microsoft/Dockerfile +++ b/agent-microsoft/Dockerfile @@ -9,7 +9,7 @@ COPY agent-microsoft/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-microsoft/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4211 EXPOSE 4211 diff --git a/agent-microsoft/src/main.py b/agent-microsoft/src/main.py index 5fd4a3e1e8..2465baf5a3 100644 --- a/agent-microsoft/src/main.py +++ b/agent-microsoft/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from agent_framework.anthropic import AnthropicClient from agent_framework.openai import OpenAIChatClient @@ -43,6 +44,8 @@ def _client() -> AnthropicClient | OpenAIChatClient: ) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-microsoft/tests/test_conversation_contract.py b/agent-microsoft/tests/test_conversation_contract.py index f1baa9c859..68a00000bd 100644 --- a/agent-microsoft/tests/test_conversation_contract.py +++ b/agent-microsoft/tests/test_conversation_contract.py @@ -14,11 +14,14 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_run_keys, conversation, fake_provider, + gateway_environment, missing_from, openai_environment, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -50,3 +53,30 @@ def _run(monkeypatch, environment, body): @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] + + +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization, for that run; the environment's otherwise. + + See `shared/run_model_key.py`. + """ + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in gateway_environment(wire)(base).items(): + monkeypatch.setenv(key, value) + + from src import main + + # One event loop for both runs, as the server process has: the SDK keeps its client. + with TestClient(importlib.reload(main).app) as client: + for body in (with_run_key(conversation(), wire), conversation(run_id="contract-run-2")): + response = client.post("/", json=body, headers={"x-openbot-agent-token": TOKEN}) + assert response.status_code == 200 + assert "RUN_FINISHED" in response.text, response.text + if "openbotModelRoute" in body["forwardedProps"]: + ran_with_token = len(keys) + check_run_keys(keys, ran_with_token, bodies) diff --git a/agent-pydantic-ai/Dockerfile b/agent-pydantic-ai/Dockerfile index 7cf7486472..6347fc43a8 100644 --- a/agent-pydantic-ai/Dockerfile +++ b/agent-pydantic-ai/Dockerfile @@ -9,7 +9,7 @@ COPY agent-pydantic-ai/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-pydantic-ai/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4205 EXPOSE 4205 diff --git a/agent-pydantic-ai/src/main.py b/agent-pydantic-ai/src/main.py index c8743daaf4..ea05d9fefc 100644 --- a/agent-pydantic-ai/src/main.py +++ b/agent-pydantic-ai/src/main.py @@ -13,6 +13,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -57,6 +58,8 @@ def _drop_blank_base_urls() -> None: _drop_blank_base_urls() agent = Agent(_model_id()) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-pydantic-ai/tests/test_conversation_contract.py b/agent-pydantic-ai/tests/test_conversation_contract.py index 5e5afbca38..350651c126 100644 --- a/agent-pydantic-ai/tests/test_conversation_contract.py +++ b/agent-pydantic-ai/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -48,6 +49,27 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + monkeypatch.setenv("PYDANTIC_AI_NO_BANNER", "1") + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-strands/Dockerfile b/agent-strands/Dockerfile index 27ac341d83..ab9c308d7d 100644 --- a/agent-strands/Dockerfile +++ b/agent-strands/Dockerfile @@ -9,7 +9,7 @@ COPY agent-strands/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-strands/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4207 EXPOSE 4207 diff --git a/agent-strands/src/main.py b/agent-strands/src/main.py index 038fdb49cf..ecf2c3b0c9 100644 --- a/agent-strands/src/main.py +++ b/agent-strands/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui_strands import StrandsAgent, add_strands_fastapi_endpoint from fastapi import FastAPI, Request @@ -37,6 +38,8 @@ def _model_id() -> str: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) TOKEN_HEADER = "x-openbot-agent-token" diff --git a/agent-strands/tests/test_conversation_contract.py b/agent-strands/tests/test_conversation_contract.py index 3a396bb99d..1def4299de 100644 --- a/agent-strands/tests/test_conversation_contract.py +++ b/agent-strands/tests/test_conversation_contract.py @@ -20,11 +20,14 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_run_keys, conversation, fake_provider, + gateway_environment, missing_from, openai_environment, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -191,6 +194,21 @@ def test_a_row_the_run_also_sends_as_context_is_said_once(monkeypatch): assert json.dumps(bodies[-1]).count(memory) == 1, "the memory reaches the model once" +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization, for that run; the environment's otherwise. + + See `shared/run_model_key.py`. + """ + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + client = TestClient(_app(monkeypatch, gateway_environment(wire), base)) + _post(client, with_run_key(conversation(), wire)) + ran_with_token = len(keys) + _post(client, conversation(run_id="contract-run-2")) + check_run_keys(keys, ran_with_token, bodies) + + def test_a_warm_process_takes_the_role_from_this_run_not_the_first(monkeypatch): first = conversation(run_id="run-1") second = conversation(run_id="run-2") diff --git a/shared/harness_contract.py b/shared/harness_contract.py index c13a918a0f..d000fe0107 100644 --- a/shared/harness_contract.py +++ b/shared/harness_contract.py @@ -106,6 +106,91 @@ def anthropic_environment(base: str, model: str = "claude-sonnet-5-5") -> dict[s } +# OpenBot's model gateway, as a harness it runs reaches it: the deployment's gateway token in the +# environment, and the token the server minted for one run, in its organization, on the run. +DEPLOYMENT_KEY = "contract-deployment-gateway-token" +RUN_KEY = "obgw_contract-run-token" + + +def gateway_environment(wire: str): + """An environment function, like `openai_environment`, for the gateway's `wire` at `base`.""" + + def environment(base: str, model: str = "contract-model") -> dict[str, str]: + gateway = f"{base}/api/model-provider" + if wire == "anthropic": + return { + "BOT_PROVIDER": "anthropic", + "BOT_MODEL": "claude-sonnet-5-5", + "ANTHROPIC_API_KEY": DEPLOYMENT_KEY, + "ANTHROPIC_BASE_URL": f"{gateway}/anthropic", + "OPENAI_API_KEY": "", + "OPENAI_BASE_URL": "", + } + return { + "BOT_PROVIDER": "", + "BOT_MODEL": model, + "OPENAI_API_KEY": DEPLOYMENT_KEY, + "OPENAI_BASE_URL": f"{gateway}/v1", + "ANTHROPIC_API_KEY": "", + } + + environment.__name__ = f"gateway_{wire}_environment" + return environment + + +def with_run_key(body: dict, wire: str) -> dict: + """`body` as the server sends a harness it runs: with the run's own gateway token.""" + return { + **body, + "forwardedProps": { + **(body.get("forwardedProps") or {}), + "openbotModelRoute": {"via": wire, "token": RUN_KEY}, + "openbotOrganization": "contract-organization", + }, + } + + +def check_run_keys(keys: list[str], ran_with_token: int, bodies: list[dict]) -> None: + """The first `ran_with_token` model requests carried the run's token, the rest the environment's. + + And neither the token nor the organization reached the model in a request body. + """ + assert ran_with_token > 0, "the run with a token never reached the model" + assert set(keys[:ran_with_token]) == {RUN_KEY}, keys + assert keys[ran_with_token:] and set(keys[ran_with_token:]) == {DEPLOYMENT_KEY}, keys + text = json.dumps(bodies) + assert RUN_KEY not in text and "contract-organization" not in text + + +def check_gateway_keys(app_for, wire: str, *, path: str = "/", token: str = "contract-token") -> None: + """A harness calls the gateway with its run's token, and with the environment's key without one. + + `app_for(environment)` sets `environment` (a dict) for the harness and answers its app on a + fresh import. Two runs on one process, as the server sends them: the first with a token, the + second without. + """ + from fastapi.testclient import TestClient + + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + with TestClient(app_for(gateway_environment(wire)(base))) as client: + # What a framework asks at start (a model listing) is no run's, and keeps its key. + started = len(keys) + ran_with_token = 0 + runs = ( + with_run_key(conversation(), wire), + conversation(thread_id="contract-thread-2", run_id="contract-run-2"), + ) + for body in runs: + response = client.post(path, json=body, headers={"x-openbot-agent-token": token}) + assert response.status_code == 200, response.text + types = [e["type"] for e in run_events(response.text)] + assert "RUN_FINISHED" in types, response.text + if not ran_with_token: + ran_with_token = len(keys) - started + check_run_keys(keys[started:], ran_with_token, bodies) + + @contextmanager def fake_provider(keys: list[str] | None = None) -> Iterator[tuple[str, list[dict]]]: """An OpenAI- and Anthropic-compatible model on localhost that answers "ok" and records bodies. @@ -250,7 +335,18 @@ async def messages(request: Request): with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] - server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) + async def served(scope, receive, send): + # Also at OpenBot's model gateway addresses (`gateway_environment`), as the gateway serves + # each wire: `/api/model-provider/v1/...` and `/api/model-provider/anthropic/v1/...`. + path = scope.get("path", "") + for prefix in ("/api/model-provider/anthropic", "/api/model-provider"): + if path.startswith(prefix + "/"): + scope = {key: value for key, value in scope.items() if key != "raw_path"} + scope["path"] = path[len(prefix) :] + break + await app(scope, receive, send) + + server = uvicorn.Server(uvicorn.Config(served, host="127.0.0.1", port=port, log_level="error")) thread = threading.Thread(target=server.run, daemon=True) thread.start() deadline = time.monotonic() + 10 diff --git a/shared/run_model_key.py b/shared/run_model_key.py new file mode 100644 index 0000000000..88255164b3 --- /dev/null +++ b/shared/run_model_key.py @@ -0,0 +1,181 @@ +"""A run's own key to OpenBot's model gateway, for every Python harness OpenBot runs. + +One harness serves every organization of an OpenBot. Its model calls go to the server's model +gateway (`/api/model-provider/...`), which answers each on the AI connection of the organization +the caller's key names. The key in the harness's environment is the deployment's gateway token, +and it names one organization. So the server sends each run a token of its own, minted for that +run in its organization, as `forwardedProps.openbotModelRoute: {"via": ..., "token": ...}`, and the +harness calls the model with it for that run. + +THE FRAMEWORKS BUILD THEIR MODEL CLIENTS ONCE, at start, from the environment, and each has its own +way of doing it. So the key is swapped where every one of them ends up: the HTTP request. A harness +wraps its app in `RunModelKey`, which reads the token from the run's body (and takes the server's +props out of it, so no framework ever holds the token in its state), and keeps it for the run in +a context variable; every request the harness then sends through `httpx` (the OpenAI and Anthropic +SDKs, LiteLLM and the rest) carries it in place of the environment's key. + +ONLY TO THE GATEWAY. The token replaces a key only on a request whose path is the gateway's +(`/api/model-provider/`). A harness given a provider key of its own sends that key to the provider +as before, and a run that brought no token sends what it always sent. +""" + +from __future__ import annotations + +import json +from contextvars import ContextVar +from typing import Any + +GATEWAY_PATH = "/api/model-provider/" + +# The props only the server writes and only this module reads, taken out of every run. +SERVER_PROPS = ( + "openbotModelRoute", + "openbotOrganization", + "openbot_model_route", + "openbot_organization", +) + +_run_key: ContextVar[str | None] = ContextVar("openbot_run_model_key", default=None) + + +def run_key() -> str | None: + """The token of the run in progress, or None.""" + return _run_key.get() + + +def key_from(props: Any) -> str: + """The token the server sent in a run's forwarded props, or an empty string.""" + if not isinstance(props, dict): + return "" + route = props.get("openbotModelRoute") + if not isinstance(route, dict) or route.get("via") not in ("openai", "anthropic"): + return "" + token = route.get("token") + return token.strip() if isinstance(token, str) else "" + + +def with_run_key(headers: Any, path: str) -> None: + """The run's token in place of the key on a request to the gateway; nothing otherwise. + + OpenAI's SDK sends its key as `Authorization: Bearer`, Anthropic's as `x-api-key`. Each is + replaced where it is; a request with neither is given the bearer the gateway reads first. + """ + key = _run_key.get() + if not key or GATEWAY_PATH not in path: + return + had_api_key = "x-api-key" in headers + if had_api_key: + headers["x-api-key"] = key + if "authorization" in headers or not had_api_key: + headers["authorization"] = f"Bearer {key}" + + +_installed = False + + +def install() -> None: + """Every HTTP request of this process carries the run's token to the gateway. Idempotent. + + `httpx` and `httpx2`, the fork the OpenAI and Anthropic SDKs moved to; a framework that uses + either sends through `Client.send` or `AsyncClient.send`, whatever it builds on top. + """ + global _installed + if _installed: + return + import importlib + + for name in ("httpx", "httpx2"): + try: + module = importlib.import_module(name) + except ImportError: + continue + _patch(module.Client, module.AsyncClient) + _installed = True + + +def _patch(client, async_client) -> None: + sync_send = client.send + async_send = async_client.send + + def send(self, request, *args, **kwargs): + with_run_key(request.headers, request.url.path) + return sync_send(self, request, *args, **kwargs) + + async def asend(self, request, *args, **kwargs): + with_run_key(request.headers, request.url.path) + return await async_send(self, request, *args, **kwargs) + + client.send = send + async_client.send = asend + + +class RunModelKey: + """ASGI middleware: the run's token kept for the run, and the server's props taken out. + + Reads a POST's JSON body whole, takes `SERVER_PROPS` out of its `forwardedProps`, and hands the + app the body without them. The token is kept in a context variable for as long as the app + answers, which every task the app starts inherits. + """ + + def __init__(self, app): + self.app = app + install() + + async def __call__(self, scope, receive, send): + if scope.get("type") != "http" or scope.get("method") != "POST": + await self.app(scope, receive, send) + return + chunks = [] + while True: + message = await receive() + if message["type"] != "http.request": + # The client went away before the body was whole: nothing to run. + await self.app(scope, _replay(b"", receive), send) + return + chunks.append(message.get("body", b"")) + if not message.get("more_body", False): + break + body = b"".join(chunks) + token = "" + try: + parsed = json.loads(body) if body else None + except ValueError: + parsed = None + if isinstance(parsed, dict): + props = parsed.get("forwardedProps") + if not isinstance(props, dict): + props = parsed.get("forwarded_props") + if isinstance(props, dict): + token = key_from(props) + if any(name in props for name in SERVER_PROPS): + for name in SERVER_PROPS: + props.pop(name, None) + body = json.dumps(parsed).encode("utf-8") + scope = { + **scope, + "headers": [ + (name, value) + for name, value in scope.get("headers", []) + if name.lower() != b"content-length" + ] + + [(b"content-length", str(len(body)).encode("ascii"))], + } + reset = _run_key.set(token or None) + try: + await self.app(scope, _replay(body, receive), send) + finally: + _run_key.reset(reset) + + +def _replay(body: bytes, receive): + """`receive`, answering the body once and then whatever the client sends next.""" + sent = False + + async def replay(): + nonlocal sent + if not sent: + sent = True + return {"type": "http.request", "body": body, "more_body": False} + return await receive() + + return replay From 64215693ca20f4e54a69f2fb4c04db5764c9ea3e Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 16:30:14 -0800 Subject: [PATCH 5/7] Claude Agent SDK harness: install httpx, which passes Claude Code's calls on to the gateway The image had no httpx: the harness's tests install it for themselves, the image does not. Found by the live run of the built image through the gateway. --- agent-claude-sdk/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent-claude-sdk/requirements.txt b/agent-claude-sdk/requirements.txt index 3ac7d8b8e3..4c890f0af4 100644 --- a/agent-claude-sdk/requirements.txt +++ b/agent-claude-sdk/requirements.txt @@ -1,5 +1,7 @@ ag-ui-claude-sdk==0.1.5 claude-agent-sdk==0.2.182 fastapi +# Passes Claude Code's model calls on to OpenBot's model gateway (src/run_model_proxy.py). +httpx==0.28.1 python-multipart uvicorn[standard] From d0e79e5668e75ded64abd423f02fc8141cd393e5 Mon Sep 17 00:00:00 2001 From: David McKay Date: Thu, 19 Nov 2026 17:45:20 -0800 Subject: [PATCH 6/7] Keep the model lock to model clients, and the compose probe's FastAPI in step httpx in the Claude Agent SDK harness is not a model client, so it is not pinned exactly: the model lock reads every exact pin in a harness's requirements as one. The compose probe's stand-in FastAPI takes add_middleware, which the LangGraph harness now calls. --- agent-claude-sdk/requirements.txt | 2 +- tests/compose.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/agent-claude-sdk/requirements.txt b/agent-claude-sdk/requirements.txt index 4c890f0af4..4ec6dee19e 100644 --- a/agent-claude-sdk/requirements.txt +++ b/agent-claude-sdk/requirements.txt @@ -2,6 +2,6 @@ ag-ui-claude-sdk==0.1.5 claude-agent-sdk==0.2.182 fastapi # Passes Claude Code's model calls on to OpenBot's model gateway (src/run_model_proxy.py). -httpx==0.28.1 +httpx>=0.28.1,<0.29 python-multipart uvicorn[standard] diff --git a/tests/compose.test.ts b/tests/compose.test.ts index 1407394b48..6f16a17173 100644 --- a/tests/compose.test.ts +++ b/tests/compose.test.ts @@ -114,6 +114,8 @@ function runLangGraphAguiModelProbe( join(dir, "fastapi", "__init__.py"), [ "class FastAPI:", + " def add_middleware(self, *_args, **_kwargs):", + " pass", " def middleware(self, *_args, **_kwargs):", " def decorator(fn):", " return fn", From 9e3bc65732c6fb8ffd61dfd8d62c34b15e0d50d7 Mon Sep 17 00:00:00 2001 From: David McKay Date: Fri, 20 Nov 2026 05:39:35 -0800 Subject: [PATCH 7/7] Changelog: engines keep each organization apart --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7e3421f36..8341aa7409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,21 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. backed up (`backup.kind`), the agent refuses every upgrade that changes the database, before touching anything. +### Bots and models + +- **Every harness OpenBot ships calls the model gateway with its run's own token, where the server + sends one.** One harness serves every organization of an OpenBot, and the key in its environment + names one organization. A run that brings its own token (`forwardedProps.openbotModelRoute`; on + Mastra the `OpenBot model route` context entry) calls the gateway with it, so the call is answered + on that run's organization's AI connection. CrewAI, LlamaIndex, Agno, LangChain, Google ADK, + Pydantic AI, Microsoft Agent Framework, Strands, AG2, Langroid, the Claude Agent SDK on a key and + Mastra all do. Only calls to the gateway carry it, and the token never reaches the model or a + framework's own state. A run that brings none keeps the environment's key, so OpenBot Desktop + behaves as before. +- **A Claude plan's query is kept per organization on the LangGraph (TypeScript) engine.** The same + conversation id for a Bot of the same id in two organizations is two Claude queries now, not one + shared transcript. A run with no organization keeps the query name it always had. + ## 1.1.2 **Before upgrading.** Nothing in a Helm values file, a Compose override or a script has to change,