diff --git a/CHANGELOG.md b/CHANGELOG.md index c7e3421f36..8341aa7409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,21 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. backed up (`backup.kind`), the agent refuses every upgrade that changes the database, before touching anything. +### Bots and models + +- **Every harness OpenBot ships calls the model gateway with its run's own token, where the server + sends one.** One harness serves every organization of an OpenBot, and the key in its environment + names one organization. A run that brings its own token (`forwardedProps.openbotModelRoute`; on + Mastra the `OpenBot model route` context entry) calls the gateway with it, so the call is answered + on that run's organization's AI connection. CrewAI, LlamaIndex, Agno, LangChain, Google ADK, + Pydantic AI, Microsoft Agent Framework, Strands, AG2, Langroid, the Claude Agent SDK on a key and + Mastra all do. Only calls to the gateway carry it, and the token never reaches the model or a + framework's own state. A run that brings none keeps the environment's key, so OpenBot Desktop + behaves as before. +- **A Claude plan's query is kept per organization on the LangGraph (TypeScript) engine.** The same + conversation id for a Bot of the same id in two organizations is two Claude queries now, not one + shared transcript. A run with no organization keeps the query name it always had. + ## 1.1.2 **Before upgrading.** Nothing in a Helm values file, a Compose override or a script has to change, diff --git a/agent-adk/Dockerfile b/agent-adk/Dockerfile index fb990dbb85..65266eee30 100644 --- a/agent-adk/Dockerfile +++ b/agent-adk/Dockerfile @@ -9,7 +9,7 @@ COPY agent-adk/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-adk/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4208 EXPOSE 4208 diff --git a/agent-adk/src/main.py b/agent-adk/src/main.py index 32b4c335d7..af1ac2e007 100644 --- a/agent-adk/src/main.py +++ b/agent-adk/src/main.py @@ -15,6 +15,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui.core import ( AssistantMessage, @@ -343,6 +344,8 @@ def _model_id() -> str: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-adk/tests/test_conversation_contract.py b/agent-adk/tests/test_conversation_contract.py index 67d8ed9acc..19969878e8 100644 --- a/agent-adk/tests/test_conversation_contract.py +++ b/agent-adk/tests/test_conversation_contract.py @@ -29,6 +29,7 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -119,6 +120,26 @@ async def turns(): return asyncio.run(asyncio.wait_for(turns(), timeout=60)) +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. Gemini goes to Google with its own key, never the gateway. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + monkeypatch.setenv("OTEL_SDK_DISABLED", "true") + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): with fake_provider() as (base, bodies): diff --git a/agent-ag2/Dockerfile b/agent-ag2/Dockerfile index df952e81a6..fbce0838fa 100644 --- a/agent-ag2/Dockerfile +++ b/agent-ag2/Dockerfile @@ -9,7 +9,7 @@ COPY agent-ag2/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-ag2/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4210 EXPOSE 4210 diff --git a/agent-ag2/src/main.py b/agent-ag2/src/main.py index 81da87903b..0d705c2560 100644 --- a/agent-ag2/src/main.py +++ b/agent-ag2/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings, openai_api +from run_model_key import RunModelKey from ag2 import Agent from ag2.ag_ui import AGUIStream @@ -55,6 +56,8 @@ def _config() -> AnthropicConfig | OpenAICompletionsConfig | OpenAIResponsesConf stream = AGUIStream(agent) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-ag2/tests/test_conversation_contract.py b/agent-ag2/tests/test_conversation_contract.py index 09e40c0290..13dd3878f2 100644 --- a/agent-ag2/tests/test_conversation_contract.py +++ b/agent-ag2/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-agno/Dockerfile b/agent-agno/Dockerfile index 9e97f4343d..e92b4dad67 100644 --- a/agent-agno/Dockerfile +++ b/agent-agno/Dockerfile @@ -9,7 +9,7 @@ COPY agent-agno/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-agno/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4203 EXPOSE 4203 diff --git a/agent-agno/src/main.py b/agent-agno/src/main.py index 145f905487..d6b167042a 100644 --- a/agent-agno/src/main.py +++ b/agent-agno/src/main.py @@ -14,6 +14,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from agno.agent import Agent from agno.models.litellm import LiteLLM @@ -81,6 +82,8 @@ def _sampling() -> dict: ) app = AgentOS(agents=[agent], interfaces=[AGUI(agent=agent)]).get_app() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-agno/tests/test_conversation_contract.py b/agent-agno/tests/test_conversation_contract.py index 5ded715a18..f46f875766 100644 --- a/agent-agno/tests/test_conversation_contract.py +++ b/agent-agno/tests/test_conversation_contract.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -48,6 +49,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/agui", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-antigravity/Dockerfile b/agent-antigravity/Dockerfile index cd96befd7d..83e3583e90 100644 --- a/agent-antigravity/Dockerfile +++ b/agent-antigravity/Dockerfile @@ -12,7 +12,7 @@ COPY agent-antigravity/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-antigravity/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4214 EXPOSE 4214 diff --git a/agent-antigravity/src/main.py b/agent-antigravity/src/main.py index 28f453da01..33a1bced38 100644 --- a/agent-antigravity/src/main.py +++ b/agent-antigravity/src/main.py @@ -36,6 +36,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import SPEC, bot_settings +from run_model_key import RunModelKey from ag_ui_antigravity import create_antigravity_app from fastapi import Request @@ -95,6 +96,9 @@ def _agent() -> OpenBotAntigravityAgent: app = create_antigravity_app(_agent(), path="/") +# Gemini is reached with Google's own key, never OpenBot's model gateway, so no run's token is ever +# sent anywhere from here; the server's props still never reach the agent. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-antigravity/tests/test_conversation_contract.py b/agent-antigravity/tests/test_conversation_contract.py index d54bb504d1..50170d674c 100644 --- a/agent-antigravity/tests/test_conversation_contract.py +++ b/agent-antigravity/tests/test_conversation_contract.py @@ -30,10 +30,12 @@ from harness_contract import ( EARLIER_QUESTION, LATEST_QUESTION, + RUN_KEY, STANDING_ROLE, conversation, missing_from, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -177,6 +179,18 @@ def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, assert STANDING_ROLE in _system_instruction(body), "the standing role is a system instruction" +def test_a_runs_gateway_token_never_reaches_gemini(monkeypatch, tmp_path): + """Gemini is reached with Google's own key: the run's token for OpenBot's gateway goes nowhere. + + See `shared/run_model_key.py`. + """ + with fresh_bot(monkeypatch, tmp_path) as run: + body = run(with_run_key(conversation(), "openai")) + assert missing_from(body) == [] + assert RUN_KEY not in json.dumps(run.bodies) + assert "contract-organization" not in json.dumps(run.bodies) + + def test_a_live_session_is_not_sent_the_history_twice(monkeypatch, tmp_path): first = conversation() with fresh_bot(monkeypatch, tmp_path) as run: diff --git a/agent-claude-sdk/Dockerfile b/agent-claude-sdk/Dockerfile index 6ace4d928d..90134366a6 100644 --- a/agent-claude-sdk/Dockerfile +++ b/agent-claude-sdk/Dockerfile @@ -9,6 +9,8 @@ COPY agent-claude-sdk/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-claude-sdk/src ./src +# A run's own key to OpenBot's model gateway, shared by every Python harness (src/run_model_proxy.py). +COPY shared/run_model_key.py /shared/ ENV PORT=4212 EXPOSE 4212 diff --git a/agent-claude-sdk/requirements.txt b/agent-claude-sdk/requirements.txt index 3ac7d8b8e3..4ec6dee19e 100644 --- a/agent-claude-sdk/requirements.txt +++ b/agent-claude-sdk/requirements.txt @@ -1,5 +1,7 @@ ag-ui-claude-sdk==0.1.5 claude-agent-sdk==0.2.182 fastapi +# Passes Claude Code's model calls on to OpenBot's model gateway (src/run_model_proxy.py). +httpx>=0.28.1,<0.29 python-multipart uvicorn[standard] diff --git a/agent-claude-sdk/src/adapter.py b/agent-claude-sdk/src/adapter.py index b909bcd0a9..6794b89cd3 100644 --- a/agent-claude-sdk/src/adapter.py +++ b/agent-claude-sdk/src/adapter.py @@ -53,6 +53,7 @@ from claude_agent_sdk.types import StreamEvent from .replay import RESULT_CHARS, cut_note, run_replayed_query, system_instructions +from .run_model_proxy import query_environment, refresh logger = logging.getLogger(__name__) @@ -827,7 +828,13 @@ def build_options(self, input_data=None, thread_id=None): if chosen: # After the upstream merge, so the server's choice wins over a forwarded `model`. options.model, options.effort = chosen - options.env = {**(options.env or {}), UNKNOWN_MODEL_WINDOW: "1"} + # On the gateway, the query's calls come back through this harness with the newest run + # token of its thread. See run_model_proxy.py. + options.env = { + **(options.env or {}), + UNKNOWN_MODEL_WINDOW: "1", + **query_environment(thread_id), + } if once: options.env[MAX_RETRIES] = "0" options.max_buffer_size = MESSAGE_BUFFER_BYTES @@ -1291,6 +1298,8 @@ async def _pump_query(self, input_data, turn, fit_results=False): async def run(self, input_data): thread_id, run_id = input_data.thread_id, input_data.run_id + # A query this run carries on sends its calls with this run's token from now. + refresh(thread_id) lock = self._segment_locks.setdefault(thread_id, asyncio.Lock()) async with lock: answer_only = ( diff --git a/agent-claude-sdk/src/main.py b/agent-claude-sdk/src/main.py index dced048ac1..a5490a02ef 100644 --- a/agent-claude-sdk/src/main.py +++ b/agent-claude-sdk/src/main.py @@ -17,6 +17,8 @@ from pydantic import BaseModel, Field from .adapter import OpenBotClaudeAgentAdapter +from .run_model_proxy import PROXY_PATH, forward +from run_model_key import RunModelKey # On the path run_model_proxy puts shared/ on. TOKEN_HEADER = "x-openbot-agent-token" @@ -44,11 +46,15 @@ def _refuse_both_credentials() -> None: _refuse_both_credentials() app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_proxy.py. +app.add_middleware(RunModelKey) @app.middleware("http") async def refuse_without_the_server_token(request: Request, call_next): - if request.url.path != "/health": + # Claude Code's own model calls come back here on a secret path of their query's, from this + # container's loopback, and carry no server token. See run_model_proxy.py. + if request.url.path != "/health" and not request.url.path.startswith(PROXY_PATH): expected = (os.environ.get("MANAGED_AGENT_TOKEN") or "").strip() offered = (request.headers.get(TOKEN_HEADER) or "").strip() if not expected or not hmac.compare_digest(offered.encode("utf-8"), expected.encode("utf-8")): @@ -61,6 +67,11 @@ async def health(): return {"ok": True, "harness": "claude-agent-sdk"} +@app.api_route(PROXY_PATH + "{secret}/{path:path}", methods=["GET", "POST"]) +async def run_model(request: Request, secret: str, path: str): + return await forward(request, secret, path) + + add_claude_fastapi_endpoint( app=app, adapter=OpenBotClaudeAgentAdapter(name="openbot"), diff --git a/agent-claude-sdk/src/run_model_proxy.py b/agent-claude-sdk/src/run_model_proxy.py new file mode 100644 index 0000000000..68e24b149e --- /dev/null +++ b/agent-claude-sdk/src/run_model_proxy.py @@ -0,0 +1,144 @@ +"""Claude Code's model calls on a run's own gateway token, in its organization. + +One harness serves every organization of an OpenBot. Its model calls go to the server's model +gateway, which answers each on the AI connection of the organization the caller's key names, and +the key in this container's environment names one organization. So the server sends each run a +token of its own (`openbotModelRoute`), which `RunModelKey` keeps for the run +(`shared/run_model_key.py`). + +The Python harnesses send their model calls themselves, and that module puts the token on each. +Here Claude Code sends them, from a process of its own that lives across runs: a query waits on a +tool's result while the person decides, and the next run carries it on. A key fixed in its +environment would be the first run's, past its ten minutes. So a query on the gateway is pointed at +this harness instead, on a path only it knows (`/_run-model/`), and every call it makes is +passed on to the gateway with the newest token any run of that thread brought. + +Only where the configured address is the gateway's (`/api/model-provider/`) and a run brought a +token. Otherwise Claude Code is given what it always was. +""" + +import os +import secrets +import sys +import time +from pathlib import Path +from urllib.parse import urlsplit + +# The module every Python harness shares: beside this harness in the repository, at /shared in the +# image the Dockerfile builds. +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) +from run_model_key import GATEWAY_PATH, run_key # noqa: E402 + +PROXY_PATH = "/_run-model/" + +# A thread's slot outlives a run: its query may wait on a person for hours. One a day old is gone. +SLOT_SECONDS = 24 * 60 * 60 + + +class _Slot: + __slots__ = ("secret", "key", "at") + + def __init__(self, secret: str, key: str): + self.secret = secret + self.key = key + self.at = time.monotonic() + + +_by_thread: dict[str, _Slot] = {} +_by_secret: dict[str, _Slot] = {} + + +def _gateway_base() -> str: + base = (os.environ.get("ANTHROPIC_BASE_URL") or "").strip().rstrip("/") + return base if base and GATEWAY_PATH in urlsplit(base).path + "/" else "" + + +def _forget_old(now: float) -> None: + for thread, slot in list(_by_thread.items()): + if now - slot.at > SLOT_SECONDS: + del _by_thread[thread] + _by_secret.pop(slot.secret, None) + + +def query_environment(thread_id: str | None) -> dict[str, str]: + """What a new query of `thread_id` is started with, so its calls come back through here. + + Empty where the run brought no token or the configured address is not the gateway's. + """ + key = run_key() + if not key or not thread_id or not _gateway_base(): + return {} + now = time.monotonic() + _forget_old(now) + slot = _by_thread.get(thread_id) + if slot is None: + slot = _Slot(secrets.token_urlsafe(32), key) + _by_thread[thread_id] = slot + _by_secret[slot.secret] = slot + slot.key, slot.at = key, now + port = (os.environ.get("PORT") or "4212").strip() + return { + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}{PROXY_PATH}{slot.secret}", + # Claude Code needs a key to start; the one it sends is replaced on the way out. + "ANTHROPIC_API_KEY": "openbot-run-model", + } + + +def refresh(thread_id: str | None) -> None: + """A run of `thread_id` brought a newer token: the query it carries on sends that from now.""" + key = run_key() + slot = _by_thread.get(thread_id or "") + if key and slot is not None: + slot.key, slot.at = key, time.monotonic() + + +def slot_key(secret: str) -> str | None: + slot = _by_secret.get(secret) + return slot.key if slot is not None else None + + +# What is not passed on to the gateway: how this hop was made, and the key Claude Code was given. +_REQUEST_DROPPED = {"host", "content-length", "connection", "authorization", "x-api-key"} +# What is not passed back: how the gateway's answer was framed, which this hop frames again. +_RESPONSE_DROPPED = {"content-length", "transfer-encoding", "connection"} + +_client = None + + +async def forward(request, secret: str, path: str): + """One of Claude Code's model calls, passed on to the gateway with its thread's newest token.""" + import httpx + from fastapi.responses import JSONResponse, StreamingResponse + from starlette.background import BackgroundTask + + global _client + key = slot_key(secret) + base = _gateway_base() + client_host = request.client.host if request.client else "" + if not key or not base or client_host not in ("127.0.0.1", "::1"): + return JSONResponse({"error": "not found"}, status_code=404) + if _client is None: + _client = httpx.AsyncClient(timeout=httpx.Timeout(None, connect=30.0)) + target = f"{base}/{path}" + if request.url.query: + target = f"{target}?{request.url.query}" + headers = { + name: value + for name, value in request.headers.items() + if name.lower() not in _REQUEST_DROPPED + } + headers["x-api-key"] = key + upstream = _client.build_request( + request.method, target, headers=headers, content=await request.body() + ) + response = await _client.send(upstream, stream=True) + return StreamingResponse( + response.aiter_raw(), + status_code=response.status_code, + headers={ + name: value + for name, value in response.headers.items() + if name.lower() not in _RESPONSE_DROPPED + }, + background=BackgroundTask(response.aclose), + ) diff --git a/agent-claude-sdk/tests/test_run_model_key.py b/agent-claude-sdk/tests/test_run_model_key.py new file mode 100644 index 0000000000..49dbc00597 --- /dev/null +++ b/agent-claude-sdk/tests/test_run_model_key.py @@ -0,0 +1,141 @@ +"""Claude Code calls OpenBot's model gateway with the token of the run, in its organization. + +One harness serves every organization of an OpenBot, and the key in its environment is the +deployment's gateway token, which names one organization. The server sends each run a token of its +own (`openbotModelRoute`). Claude Code makes the model calls from a process of its own, so a query +on the gateway sends them back through this harness, which passes each on with the newest token a +run of that thread brought (`src/run_model_proxy.py`). A run with no token, and an address that is +not the gateway's, are given what they always were. + +These run the real Claude Code this harness ships, through the harness's own routes on a real port +(Claude Code dials it), against a local stand-in for the gateway's Anthropic wire. +""" + +import importlib +import json +import socket +import sys +import threading +import time +from pathlib import Path + +import httpx +import pytest +import uvicorn + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) +from harness_contract import ( + DEPLOYMENT_KEY, + RUN_KEY, + fake_provider, + gateway_environment, + run_events, +) + +TOKEN = "valid-test-token" +LATER_KEY = "obgw_contract-later-run-token" + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _serve(app, port): + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(0.01) + return server, thread + + +def _run(port, thread, key=None): + props = {"openbotSendOnce": True} + if key: + props["openbotModelRoute"] = {"via": "anthropic", "token": key} + props["openbotOrganization"] = "contract-organization" + response = httpx.post( + f"http://127.0.0.1:{port}/", + headers={"x-openbot-agent-token": TOKEN, "accept": "text/event-stream"}, + json={ + "threadId": thread, + "runId": f"run-{time.monotonic_ns()}", + "state": {}, + "messages": [{"id": f"user-{time.monotonic_ns()}", "role": "user", "content": "Hello?"}], + "tools": [], + "context": [], + "forwardedProps": props, + }, + timeout=120, + ) + assert response.status_code == 200 + events = run_events(response.text) + assert events[-1]["type"] == "RUN_FINISHED", events + return events + + +@pytest.fixture +def harness(monkeypatch, tmp_path): + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + for name, value in gateway_environment("anthropic")(base).items(): + monkeypatch.setenv(name, value) + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / ".claude")) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + port = _free_port() + monkeypatch.setenv("PORT", str(port)) + from src import main + + server, thread = _serve(importlib.reload(main).app, port) + try: + yield port, keys, bodies + finally: + server.should_exit = True + thread.join(timeout=10) + + +def _messages_keys(keys, start): + """The keys the Messages API calls presented since `start` (Claude Code also asks other routes).""" + return [key for key in keys[start:] if key] + + +def test_claude_code_calls_the_gateway_with_the_runs_own_key(harness): + port, keys, bodies = harness + + _run(port, "thread-org-b", RUN_KEY) + with_token = _messages_keys(keys, 0) + assert with_token and set(with_token) == {RUN_KEY}, keys + + # The next run of the thread brings a newer token; the thread's calls carry that from now. + start = len(keys) + _run(port, "thread-org-b", LATER_KEY) + later = _messages_keys(keys, start) + assert later and set(later) == {LATER_KEY}, keys + + # A run with no token: the environment's key, straight to the gateway, as before. + start = len(keys) + _run(port, "thread-without-token") + without = _messages_keys(keys, start) + assert without and set(without) == {DEPLOYMENT_KEY}, keys + + # Neither the token nor the organization is ever said to the model. + text = json.dumps(bodies) + assert RUN_KEY not in text and LATER_KEY not in text and "contract-organization" not in text + + +def test_the_way_back_in_takes_nothing_it_does_not_know(harness): + port, keys, _ = harness + # No secret this process handed out, so nothing is passed on and no key is spent. + response = httpx.post( + f"http://127.0.0.1:{port}/_run-model/made-up-secret/v1/messages", + json={"model": "claude-sonnet-5-5", "messages": []}, + timeout=30, + ) + assert response.status_code == 404 + assert keys == [] diff --git a/agent-crewai/Dockerfile b/agent-crewai/Dockerfile index e2672f4f86..ed67960f20 100644 --- a/agent-crewai/Dockerfile +++ b/agent-crewai/Dockerfile @@ -9,7 +9,7 @@ COPY agent-crewai/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-crewai/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4202 EXPOSE 4202 diff --git a/agent-crewai/src/main.py b/agent-crewai/src/main.py index feb533d7fb..943970295b 100644 --- a/agent-crewai/src/main.py +++ b/agent-crewai/src/main.py @@ -20,6 +20,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey import ag_ui_crewai.endpoint as crewai_endpoint from ag_ui.core import Message, Tool @@ -177,6 +178,8 @@ async def answer(self): app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-crewai/tests/test_conversation_contract.py b/agent-crewai/tests/test_conversation_contract.py index bfdb65eb5c..4e26eb2d21 100644 --- a/agent-crewai/tests/test_conversation_contract.py +++ b/agent-crewai/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -53,6 +54,28 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + import ag_ui_crewai.endpoint + + importlib.reload(ag_ui_crewai.endpoint) + sys.modules.pop("src.main", None) + return importlib.import_module("src.main").app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-langgraph-agui/Dockerfile b/agent-langgraph-agui/Dockerfile index 989639ec09..e5aa276331 100644 --- a/agent-langgraph-agui/Dockerfile +++ b/agent-langgraph-agui/Dockerfile @@ -9,7 +9,7 @@ COPY agent-langgraph-agui/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-langgraph-agui/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4206 EXPOSE 4206 diff --git a/agent-langgraph-agui/src/main.py b/agent-langgraph-agui/src/main.py index fafe9cf2be..5b81f52524 100644 --- a/agent-langgraph-agui/src/main.py +++ b/agent-langgraph-agui/src/main.py @@ -15,6 +15,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui_langgraph import add_langgraph_fastapi_endpoint from fastapi import FastAPI, Request @@ -361,6 +362,8 @@ async def answer(state: MessagesState): graph = builder.compile(checkpointer=MemorySaver()) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-langgraph-agui/tests/test_conversation_contract.py b/agent-langgraph-agui/tests/test_conversation_contract.py index 643dbc4abd..e4443fc691 100644 --- a/agent-langgraph-agui/tests/test_conversation_contract.py +++ b/agent-langgraph-agui/tests/test_conversation_contract.py @@ -24,6 +24,7 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -87,6 +88,26 @@ def _system_text(body: dict) -> str: return "\n".join(parts) +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): with fake_provider() as (base, bodies): diff --git a/agent-langgraph-ts/src/models.ts b/agent-langgraph-ts/src/models.ts index 503ee54d46..a3ddc07f7b 100644 --- a/agent-langgraph-ts/src/models.ts +++ b/agent-langgraph-ts/src/models.ts @@ -72,6 +72,8 @@ export type RunModel = { route?: ModelRoute; /** The run's thread, which a Claude plan's query is kept under. */ threadId?: string; + /** The run's organization, when the server names one, which joins that query's name. */ + organization?: string; /** The run's own stop, which cancels a Claude plan's query. */ signal?: AbortSignal; }; @@ -399,7 +401,13 @@ function routedModel(run: RunModel, env: Env): BaseChatModel | undefined { model: run.model, effort: run.effort, botId: run.botId, - ...(claude ? { threadId: run.threadId ?? "", signal: run.signal } : {}), + ...(claude + ? { + threadId: run.threadId ?? "", + ...(run.organization ? { organization: run.organization } : {}), + signal: run.signal, + } + : {}), }); } if (!route.token) diff --git a/agent-langgraph-ts/src/plan-model.ts b/agent-langgraph-ts/src/plan-model.ts index 4fa7508803..2cd69d5a18 100644 --- a/agent-langgraph-ts/src/plan-model.ts +++ b/agent-langgraph-ts/src/plan-model.ts @@ -5,7 +5,7 @@ * A CLAUDE PLAN, as plan-model.ts reaches it for built-in Bots. The plan token never leaves the * Claude engine; the Claude Agent SDK makes every call. Its query is kept between a step's tool call * and that call's result, so every call of a run, and the next run on the same thread, goes under - * one thread id per (Bot, thread) (`claudeThreadId`). A stopped run whose query is answering or + * one thread id per (Bot, thread, organization) (`claudeThreadId`). A stopped run whose query is answering or * waiting on a call cancels it at `/model/cancel`; a run that hands a call to the surface does not, * since the query waits for that result. * @@ -64,16 +64,36 @@ export type PlanHarnessFields = { * (`claudeThreadId`); a ChatGPT plan's calls each go under a fresh thread. */ threadId?: string; + /** + * The organization the run is in, as the server names it (`forwardedProps.openbotOrganization`). + * Absent in the organization every OpenBot starts with, and from a server that sends none. + */ + organization?: string; /** The run's stop, which cancels a Claude plan's query. */ signal?: AbortSignal; /** For tests: the fetch the AG-UI client uses. */ fetch?: typeof fetch; }; -/** One Claude query per Bot and thread, as plan-model.ts names it on the server. */ -export function claudeThreadId(botId: string, threadId: string): string { +/** + * One Claude query per Bot and thread, as plan-model.ts names it on the server, and per + * organization where the server names one. Two organizations that use the same thread id for a + * Bot of the same id then never share a query, its transcript or its turns. + * + * Without an organization the name is the one it always was, so a thread that already has a + * query keeps it. + */ +export function claudeThreadId( + botId: string, + threadId: string, + organization?: string, +): string { return createHash("sha256") - .update(JSON.stringify([botId, threadId])) + .update( + JSON.stringify( + organization ? [botId, threadId, organization] : [botId, threadId], + ), + ) .digest("hex"); } @@ -221,7 +241,11 @@ export class PlanHarnessChatModel extends BaseChatModel super({}); this.fields = fields; if (fields.plan !== "claude" || fields.threadId === undefined) return; - const query = claudeThreadId(fields.botId ?? "", fields.threadId); + const query = claudeThreadId( + fields.botId ?? "", + fields.threadId, + fields.organization, + ); this.query = query; fields.signal?.addEventListener("abort", () => void this.cancel(query), { once: true, diff --git a/agent-langgraph-ts/src/run.ts b/agent-langgraph-ts/src/run.ts index 422078102c..99e9e47d5e 100644 --- a/agent-langgraph-ts/src/run.ts +++ b/agent-langgraph-ts/src/run.ts @@ -555,6 +555,10 @@ export async function runAgent( botId: botIdOf(input.messages ?? []), route: runRoute(props), threadId: input.threadId, + ...(typeof props.openbotOrganization === "string" && + props.openbotOrganization + ? { organization: props.openbotOrganization } + : {}), signal: options.signal, }; const emit = options.emit; diff --git a/agent-langgraph-ts/test/claude-plan.test.ts b/agent-langgraph-ts/test/claude-plan.test.ts index 2774dec28f..5d883173e0 100644 --- a/agent-langgraph-ts/test/claude-plan.test.ts +++ b/agent-langgraph-ts/test/claude-plan.test.ts @@ -267,6 +267,54 @@ test("A step's tool call and the next step stay on one Claude query: the same th } }); +/* + * One Claude engine serves every organization. The same thread id from two organizations, for a + * Bot of the same id, has to be two Claude queries: one query is one transcript and one set of + * turns, and one organization would answer inside the other's. A run with no organization keeps + * the name it always had, so a thread that already has a query keeps it. + */ +test("The same thread in two organizations is two Claude queries; with no organization the name is unchanged", async () => { + const claude = await claudeEngine(() => fixture("text")); + const inOrganization = (organization?: string) => + conversation({ + forwardedProps: { + openbotModelRoute: claudeRoute, + ...(organization ? { openbotOrganization: organization } : {}), + }, + }); + try { + await withEngine({ env: env(claude.url) }, async (base) => { + await run(base, inOrganization("org-a")); + await run(base, inOrganization("org-b")); + await run(base, inOrganization()); + }); + const [a, b, none] = claude.requests.map( + (request) => request.body.threadId, + ); + assert.equal(claude.requests.length, 3); + assert.notEqual(a, b); + assert.notEqual(a, none); + assert.notEqual(b, none); + assert.equal( + a, + createHash("sha256") + .update(JSON.stringify(["test-bot", "thread-1", "org-a"])) + .digest("hex"), + ); + // The name a thread's query had before organizations joined it. + assert.equal(none, threadOf("test-bot", "thread-1")); + for (const request of claude.requests) + assert.equal( + (request.body.forwardedProps as Record) + .openbotOrganization, + undefined, + "the organization is not sent on to the Claude engine", + ); + } finally { + await claude.close(); + } +}); + test("A run that hands a call back to the surface keeps its Claude query for the result", async () => { const claude = await claudeEngine(() => fixture("tool")); try { diff --git a/agent-langroid/Dockerfile b/agent-langroid/Dockerfile index b34f1675db..c6d82df2bc 100644 --- a/agent-langroid/Dockerfile +++ b/agent-langroid/Dockerfile @@ -9,7 +9,7 @@ COPY agent-langroid/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-langroid/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4209 EXPOSE 4209 diff --git a/agent-langroid/src/main.py b/agent-langroid/src/main.py index fbfa838ed8..3655b6a883 100644 --- a/agent-langroid/src/main.py +++ b/agent-langroid/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings, openai_api +from run_model_key import RunModelKey import litellm from ag_ui_langroid import create_langroid_app @@ -90,6 +91,8 @@ def _model_id() -> str: ) app = create_langroid_app(FrontendToolsAgent(name="openbot", agent=agent)) +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-langroid/tests/test_conversation_contract.py b/agent-langroid/tests/test_conversation_contract.py index 8a5e2f45cc..83dc1aa15c 100644 --- a/agent-langroid/tests/test_conversation_contract.py +++ b/agent-langroid/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-llamaindex/Dockerfile b/agent-llamaindex/Dockerfile index 7aca05b66e..7e94d0976f 100644 --- a/agent-llamaindex/Dockerfile +++ b/agent-llamaindex/Dockerfile @@ -9,7 +9,7 @@ COPY agent-llamaindex/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-llamaindex/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4204 EXPOSE 4204 diff --git a/agent-llamaindex/src/main.py b/agent-llamaindex/src/main.py index 48b81e5189..8b618f5419 100644 --- a/agent-llamaindex/src/main.py +++ b/agent-llamaindex/src/main.py @@ -13,6 +13,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey import litellm from fastapi import FastAPI, Request @@ -56,6 +57,8 @@ def _llm() -> LiteLLM: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-llamaindex/tests/test_conversation_contract.py b/agent-llamaindex/tests/test_conversation_contract.py index 334e7d6e20..0d2446c703 100644 --- a/agent-llamaindex/tests/test_conversation_contract.py +++ b/agent-llamaindex/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -47,6 +48,26 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ['openai', 'anthropic']) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, path="/run", token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-mastra/src/mastra/index.test.ts b/agent-mastra/src/mastra/index.test.ts index 236d813d9e..759f53f395 100644 --- a/agent-mastra/src/mastra/index.test.ts +++ b/agent-mastra/src/mastra/index.test.ts @@ -82,7 +82,7 @@ async function configuredModelId(botModel: string | undefined) { "-e", [ 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', - 'const model = mastra.getAgent("openbot").model;', + 'const model = await mastra.getAgent("openbot").getModel();', "console.log(JSON.stringify({ modelId: model.modelId }));", ].join("\n"), ], @@ -161,7 +161,7 @@ async function providerStartup(botProvider: string) { "-e", [ 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', - 'const model = mastra.getAgent("openbot").model;', + 'const model = await mastra.getAgent("openbot").getModel();', "console.log(JSON.stringify({ modelId: model.modelId }));", ].join("\n"), ], @@ -568,3 +568,105 @@ describe("OpenBot Mastra provider requests", () => { }, 15_000); } }); + +/** + * One harness serves every organization of an OpenBot, and the key in its environment is the + * deployment's gateway token, which names one organization. A run the server sends its own token + * (the `OpenBot model route` context entry) calls OpenBot's model gateway with that token; a run + * without one keeps the environment's key. + */ +describe("OpenBot Mastra on OpenBot's model gateway", () => { + for (const wire of ["openai", "anthropic"] as const) { + test(`a run's own token is the key it calls the ${wire} wire with`, async () => { + const seen: (string | null)[] = []; + const gateway = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const body = await request.json(); + seen.push( + wire === "anthropic" + ? request.headers.get("x-api-key") + : request.headers.get("authorization"), + ); + if (wire === "anthropic") + return Response.json({ + id: "msg", + type: "message", + role: "assistant", + model: body.model, + content: [{ type: "text", text: "hello" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); + return Response.json({ + id: "chat", + object: "chat.completion", + created: 0, + model: body.model, + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "hello" }, + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }, + }); + const base = `${gateway.url.toString().replace(/\/+$/, "")}/api/model-provider${wire === "anthropic" ? "/anthropic" : "/v1"}`; + try { + const { stdout, stderr, exitCode } = await runProbe( + [ + Bun.argv[0], + "--no-env-file", + "-e", + [ + // Mastra's own RequestContext, from the harness's packages, as its server builds one. + 'const { RequestContext } = await import(Bun.resolveSync("@mastra/core/request-context", process.cwd() + "/agent-mastra/src/mastra"));', + 'const { mastra } = await import("./agent-mastra/src/mastra/index.ts");', + "const requestContext = new RequestContext();", + 'requestContext.set("ag-ui", { context: [{ description: "OpenBot model route", value: JSON.stringify({ via: ' + + JSON.stringify(wire) + + ', token: "obgw_run-token-org-b" }) }] });', + 'const agent = mastra.getAgent("openbot");', + 'const withToken = await agent.generate("Say hello", { requestContext });', + 'const without = await agent.generate("Say hello");', + "console.log(JSON.stringify({ texts: [withToken.text, without.text] }));", + ].join("\n"), + ], + { + PATH: process.env.PATH ?? "/opt/homebrew/bin:/usr/bin:/bin", + MASTRA_TELEMETRY_DISABLED: "true", + DO_NOT_TRACK: "1", + NODE_ENV: "test", + BOT_PROVIDER: wire, + BOT_MODEL: wire === "anthropic" ? "claude-sonnet-5-5" : "gpt-5.5", + ANTHROPIC_API_KEY: + wire === "anthropic" ? "deployment-gateway-token" : "", + OPENAI_API_KEY: + wire === "anthropic" ? "" : "deployment-gateway-token", + ANTHROPIC_BASE_URL: wire === "anthropic" ? base : "", + OPENAI_BASE_URL: wire === "anthropic" ? "" : base, + }, + 10_000, + ); + if (exitCode !== 0) + throw new Error( + `gateway probe exited ${exitCode}\n${stdout}\n${stderr}`, + ); + expect(stdout).toContain('"texts":["hello","hello"]'); + const expected = (key: string) => + wire === "anthropic" ? key : `Bearer ${key}`; + expect(seen).toEqual([ + expected("obgw_run-token-org-b"), + expected("deployment-gateway-token"), + ]); + } finally { + await gateway.stop(true); + } + }, 15_000); + } +}); diff --git a/agent-mastra/src/mastra/index.ts b/agent-mastra/src/mastra/index.ts index ed5625b619..543454a7d5 100644 --- a/agent-mastra/src/mastra/index.ts +++ b/agent-mastra/src/mastra/index.ts @@ -35,7 +35,7 @@ const SUPPORTED_PROVIDERS = new Set(["openai", "anthropic"]); * and one it has that this harness has no module for, used to fall through to the OpenAI branch * below and answer with a model the deployment never chose. */ -async function buildModel() { +async function buildModel(runKey?: string) { const settings = botSettings("agent-mastra"); const providerName = settings.provider; const spec = providerSpec(providerName); @@ -56,12 +56,16 @@ async function buildModel() { // Provider modules create default clients at import, which reject Compose's empty overrides. if (!baseURL) delete process.env[baseVariable]; + // The run's own gateway token, only where the address is OpenBot's model gateway. + const key = runKey && baseURL && onTheGateway(baseURL) ? runKey : undefined; + if (spec.id === "anthropic") { const { createAnthropic } = await import("@ai-sdk/anthropic"); // Other harnesses accept an Anthropic origin; AI SDK expects the /v1 API prefix. const origin = (baseURL || "https://api.anthropic.com").replace(/\/+$/, ""); return createAnthropic({ baseURL: origin.endsWith("/v1") ? origin : `${origin}/v1`, + ...(key ? { apiKey: key } : {}), })(model); } const { createOpenAI } = await import("@ai-sdk/openai"); @@ -70,9 +74,11 @@ async function buildModel() { baseURL?.replace(/\/+$/, "") !== "https://api.openai.com/v1"; const openai = createOpenAI({ baseURL: baseURL || "https://api.openai.com/v1", - apiKey: compatible - ? apiKeyOrPlaceholder(process.env[spec.keyVariable]) - : undefined, + apiKey: + key ?? + (compatible + ? apiKeyOrPlaceholder(process.env[spec.keyVariable]) + : undefined), }); // Compatible endpoints commonly expose Chat Completions; OpenAI keeps its Responses default. return compatible ? openai.chat(model) : openai(model); @@ -161,11 +167,58 @@ export function buildOpenBotInstructions({ return [openbotBaseInstructions, ...instructions].join("\n\n"); } +/** OpenBot's model gateway: its OpenAI and Anthropic wires are under this path on the server. */ +const GATEWAY_PATH = "/api/model-provider/"; + +function onTheGateway(baseURL: string): boolean { + try { + return `${new URL(baseURL.trim()).pathname.replace(/\/+$/, "")}/`.includes( + GATEWAY_PATH, + ); + } catch { + return false; + } +} + +/** + * The run's own gateway token, in its organization, which the server sends a harness it runs as + * the `OpenBot model route` context entry: `{"via": "openai" | "anthropic", "token": ...}`. + * + * One harness serves every organization of an OpenBot, and the key in its environment is the + * deployment's gateway token, which names one organization. A run with none keeps that key. + */ +export function runKeyFrom( + requestContext?: OpenBotInstructionArgs["requestContext"], +): string | undefined { + const entry = agUiContextEntries(requestContext).find( + (candidate): candidate is ContextEntry => + isUsableEntry(candidate) && + candidate.description === "OpenBot model route", + ); + if (!entry) return undefined; + try { + const route = JSON.parse(entry.value) as { via?: unknown; token?: unknown }; + if (route.via !== "openai" && route.via !== "anthropic") return undefined; + return typeof route.token === "string" && route.token.trim() + ? route.token.trim() + : undefined; + } catch { + return undefined; + } +} + +// Built at start, so a provider this Bot cannot drive is refused before it serves anything. +const deploymentModel = await buildModel(); + const openbot = new Agent({ id: "openbot", name: "openbot", instructions: buildOpenBotInstructions, - model: await buildModel(), + // A run that brought its own gateway token answers on a model that calls with it. + model: async ({ requestContext }) => { + const key = runKeyFrom(requestContext); + return key ? buildModel(key) : deploymentModel; + }, }); /** The one header OpenBot's server sends, compared without leaking length through timing. */ diff --git a/agent-microsoft/Dockerfile b/agent-microsoft/Dockerfile index a740f7c78f..a2fee714b4 100644 --- a/agent-microsoft/Dockerfile +++ b/agent-microsoft/Dockerfile @@ -9,7 +9,7 @@ COPY agent-microsoft/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-microsoft/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4211 EXPOSE 4211 diff --git a/agent-microsoft/src/main.py b/agent-microsoft/src/main.py index 5fd4a3e1e8..2465baf5a3 100644 --- a/agent-microsoft/src/main.py +++ b/agent-microsoft/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from agent_framework.anthropic import AnthropicClient from agent_framework.openai import OpenAIChatClient @@ -43,6 +44,8 @@ def _client() -> AnthropicClient | OpenAIChatClient: ) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-microsoft/tests/test_conversation_contract.py b/agent-microsoft/tests/test_conversation_contract.py index f1baa9c859..68a00000bd 100644 --- a/agent-microsoft/tests/test_conversation_contract.py +++ b/agent-microsoft/tests/test_conversation_contract.py @@ -14,11 +14,14 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_run_keys, conversation, fake_provider, + gateway_environment, missing_from, openai_environment, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -50,3 +53,30 @@ def _run(monkeypatch, environment, body): @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] + + +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization, for that run; the environment's otherwise. + + See `shared/run_model_key.py`. + """ + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in gateway_environment(wire)(base).items(): + monkeypatch.setenv(key, value) + + from src import main + + # One event loop for both runs, as the server process has: the SDK keeps its client. + with TestClient(importlib.reload(main).app) as client: + for body in (with_run_key(conversation(), wire), conversation(run_id="contract-run-2")): + response = client.post("/", json=body, headers={"x-openbot-agent-token": TOKEN}) + assert response.status_code == 200 + assert "RUN_FINISHED" in response.text, response.text + if "openbotModelRoute" in body["forwardedProps"]: + ran_with_token = len(keys) + check_run_keys(keys, ran_with_token, bodies) diff --git a/agent-pydantic-ai/Dockerfile b/agent-pydantic-ai/Dockerfile index 7cf7486472..6347fc43a8 100644 --- a/agent-pydantic-ai/Dockerfile +++ b/agent-pydantic-ai/Dockerfile @@ -9,7 +9,7 @@ COPY agent-pydantic-ai/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-pydantic-ai/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4205 EXPOSE 4205 diff --git a/agent-pydantic-ai/src/main.py b/agent-pydantic-ai/src/main.py index c8743daaf4..ea05d9fefc 100644 --- a/agent-pydantic-ai/src/main.py +++ b/agent-pydantic-ai/src/main.py @@ -13,6 +13,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -57,6 +58,8 @@ def _drop_blank_base_urls() -> None: _drop_blank_base_urls() agent = Agent(_model_id()) app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) @app.middleware("http") diff --git a/agent-pydantic-ai/tests/test_conversation_contract.py b/agent-pydantic-ai/tests/test_conversation_contract.py index 5e5afbca38..350651c126 100644 --- a/agent-pydantic-ai/tests/test_conversation_contract.py +++ b/agent-pydantic-ai/tests/test_conversation_contract.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from harness_contract import ( anthropic_environment, + check_gateway_keys, conversation, fake_provider, missing_from, @@ -48,6 +49,27 @@ def _run(monkeypatch, environment, body): return bodies[-1] +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization; the environment's without one. + + See `shared/run_model_key.py`. + """ + + def app_for(environment): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + monkeypatch.setenv("PYDANTIC_AI_NO_BANNER", "1") + for key, value in environment.items(): + monkeypatch.setenv(key, value) + from src import main + + return importlib.reload(main).app + + check_gateway_keys(app_for, wire, token=TOKEN) + + @pytest.mark.parametrize("environment", [openai_environment, anthropic_environment]) def test_a_fresh_process_answers_from_the_whole_thread_in_its_role(monkeypatch, environment): assert missing_from(_run(monkeypatch, environment, conversation())) == [] diff --git a/agent-strands/Dockerfile b/agent-strands/Dockerfile index 27ac341d83..ab9c308d7d 100644 --- a/agent-strands/Dockerfile +++ b/agent-strands/Dockerfile @@ -9,7 +9,7 @@ COPY agent-strands/requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY agent-strands/src ./src -COPY shared/model_providers.py shared/model-providers.json /shared/ +COPY shared/model_providers.py shared/model-providers.json shared/run_model_key.py /shared/ ENV PORT=4207 EXPOSE 4207 diff --git a/agent-strands/src/main.py b/agent-strands/src/main.py index 038fdb49cf..ecf2c3b0c9 100644 --- a/agent-strands/src/main.py +++ b/agent-strands/src/main.py @@ -9,6 +9,7 @@ # one level above /app/src in the image the Dockerfile builds. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "shared")) from model_providers import bot_settings +from run_model_key import RunModelKey from ag_ui_strands import StrandsAgent, add_strands_fastapi_endpoint from fastapi import FastAPI, Request @@ -37,6 +38,8 @@ def _model_id() -> str: app = FastAPI() +# Each run's model calls carry its own gateway token, in its organization. See run_model_key.py. +app.add_middleware(RunModelKey) TOKEN_HEADER = "x-openbot-agent-token" diff --git a/agent-strands/tests/test_conversation_contract.py b/agent-strands/tests/test_conversation_contract.py index 3a396bb99d..1def4299de 100644 --- a/agent-strands/tests/test_conversation_contract.py +++ b/agent-strands/tests/test_conversation_contract.py @@ -20,11 +20,14 @@ LATEST_QUESTION, STANDING_ROLE, anthropic_environment, + check_run_keys, conversation, fake_provider, + gateway_environment, missing_from, openai_environment, run_events, + with_run_key, ) TOKEN = "contract-token" @@ -191,6 +194,21 @@ def test_a_row_the_run_also_sends_as_context_is_said_once(monkeypatch): assert json.dumps(bodies[-1]).count(memory) == 1, "the memory reaches the model once" +@pytest.mark.parametrize("wire", ["openai", "anthropic"]) +def test_a_run_calls_the_gateway_with_its_own_key(monkeypatch, wire): + """The run's own gateway token, in its organization, for that run; the environment's otherwise. + + See `shared/run_model_key.py`. + """ + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + client = TestClient(_app(monkeypatch, gateway_environment(wire), base)) + _post(client, with_run_key(conversation(), wire)) + ran_with_token = len(keys) + _post(client, conversation(run_id="contract-run-2")) + check_run_keys(keys, ran_with_token, bodies) + + def test_a_warm_process_takes_the_role_from_this_run_not_the_first(monkeypatch): first = conversation(run_id="run-1") second = conversation(run_id="run-2") diff --git a/shared/harness_contract.py b/shared/harness_contract.py index ef54cad350..d000fe0107 100644 --- a/shared/harness_contract.py +++ b/shared/harness_contract.py @@ -106,12 +106,99 @@ def anthropic_environment(base: str, model: str = "claude-sonnet-5-5") -> dict[s } +# OpenBot's model gateway, as a harness it runs reaches it: the deployment's gateway token in the +# environment, and the token the server minted for one run, in its organization, on the run. +DEPLOYMENT_KEY = "contract-deployment-gateway-token" +RUN_KEY = "obgw_contract-run-token" + + +def gateway_environment(wire: str): + """An environment function, like `openai_environment`, for the gateway's `wire` at `base`.""" + + def environment(base: str, model: str = "contract-model") -> dict[str, str]: + gateway = f"{base}/api/model-provider" + if wire == "anthropic": + return { + "BOT_PROVIDER": "anthropic", + "BOT_MODEL": "claude-sonnet-5-5", + "ANTHROPIC_API_KEY": DEPLOYMENT_KEY, + "ANTHROPIC_BASE_URL": f"{gateway}/anthropic", + "OPENAI_API_KEY": "", + "OPENAI_BASE_URL": "", + } + return { + "BOT_PROVIDER": "", + "BOT_MODEL": model, + "OPENAI_API_KEY": DEPLOYMENT_KEY, + "OPENAI_BASE_URL": f"{gateway}/v1", + "ANTHROPIC_API_KEY": "", + } + + environment.__name__ = f"gateway_{wire}_environment" + return environment + + +def with_run_key(body: dict, wire: str) -> dict: + """`body` as the server sends a harness it runs: with the run's own gateway token.""" + return { + **body, + "forwardedProps": { + **(body.get("forwardedProps") or {}), + "openbotModelRoute": {"via": wire, "token": RUN_KEY}, + "openbotOrganization": "contract-organization", + }, + } + + +def check_run_keys(keys: list[str], ran_with_token: int, bodies: list[dict]) -> None: + """The first `ran_with_token` model requests carried the run's token, the rest the environment's. + + And neither the token nor the organization reached the model in a request body. + """ + assert ran_with_token > 0, "the run with a token never reached the model" + assert set(keys[:ran_with_token]) == {RUN_KEY}, keys + assert keys[ran_with_token:] and set(keys[ran_with_token:]) == {DEPLOYMENT_KEY}, keys + text = json.dumps(bodies) + assert RUN_KEY not in text and "contract-organization" not in text + + +def check_gateway_keys(app_for, wire: str, *, path: str = "/", token: str = "contract-token") -> None: + """A harness calls the gateway with its run's token, and with the environment's key without one. + + `app_for(environment)` sets `environment` (a dict) for the harness and answers its app on a + fresh import. Two runs on one process, as the server sends them: the first with a token, the + second without. + """ + from fastapi.testclient import TestClient + + keys: list[str] = [] + with fake_provider(keys) as (base, bodies): + with TestClient(app_for(gateway_environment(wire)(base))) as client: + # What a framework asks at start (a model listing) is no run's, and keeps its key. + started = len(keys) + ran_with_token = 0 + runs = ( + with_run_key(conversation(), wire), + conversation(thread_id="contract-thread-2", run_id="contract-run-2"), + ) + for body in runs: + response = client.post(path, json=body, headers={"x-openbot-agent-token": token}) + assert response.status_code == 200, response.text + types = [e["type"] for e in run_events(response.text)] + assert "RUN_FINISHED" in types, response.text + if not ran_with_token: + ran_with_token = len(keys) - started + check_run_keys(keys[started:], ran_with_token, bodies) + + @contextmanager -def fake_provider() -> Iterator[tuple[str, list[dict]]]: +def fake_provider(keys: list[str] | None = None) -> Iterator[tuple[str, list[dict]]]: """An OpenAI- and Anthropic-compatible model on localhost that answers "ok" and records bodies. Yields `(base_url, bodies)`: every request body in arrival order. Serves Chat Completions, Responses and Messages, streamed and not, since which one a framework uses is its own choice. + Given `keys`, also writes down the key each request presented, as OpenAI's SDK sends it + (`Authorization: Bearer`) or Anthropic's (`x-api-key`). """ import uvicorn from fastapi import FastAPI, Request @@ -120,6 +207,18 @@ def fake_provider() -> Iterator[tuple[str, list[dict]]]: bodies: list[dict] = [] app = FastAPI() + if keys is not None: + + @app.middleware("http") + async def presented(request: Request, call_next): + bearer = request.headers.get("authorization", "") + keys.append( + bearer[len("Bearer ") :] + if bearer.startswith("Bearer ") + else request.headers.get("x-api-key", "") + ) + return await call_next(request) + def sse(lines): async def stream(): for line in lines: @@ -236,7 +335,18 @@ async def messages(request: Request): with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] - server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) + async def served(scope, receive, send): + # Also at OpenBot's model gateway addresses (`gateway_environment`), as the gateway serves + # each wire: `/api/model-provider/v1/...` and `/api/model-provider/anthropic/v1/...`. + path = scope.get("path", "") + for prefix in ("/api/model-provider/anthropic", "/api/model-provider"): + if path.startswith(prefix + "/"): + scope = {key: value for key, value in scope.items() if key != "raw_path"} + scope["path"] = path[len(prefix) :] + break + await app(scope, receive, send) + + server = uvicorn.Server(uvicorn.Config(served, host="127.0.0.1", port=port, log_level="error")) thread = threading.Thread(target=server.run, daemon=True) thread.start() deadline = time.monotonic() + 10 diff --git a/shared/run_model_key.py b/shared/run_model_key.py new file mode 100644 index 0000000000..88255164b3 --- /dev/null +++ b/shared/run_model_key.py @@ -0,0 +1,181 @@ +"""A run's own key to OpenBot's model gateway, for every Python harness OpenBot runs. + +One harness serves every organization of an OpenBot. Its model calls go to the server's model +gateway (`/api/model-provider/...`), which answers each on the AI connection of the organization +the caller's key names. The key in the harness's environment is the deployment's gateway token, +and it names one organization. So the server sends each run a token of its own, minted for that +run in its organization, as `forwardedProps.openbotModelRoute: {"via": ..., "token": ...}`, and the +harness calls the model with it for that run. + +THE FRAMEWORKS BUILD THEIR MODEL CLIENTS ONCE, at start, from the environment, and each has its own +way of doing it. So the key is swapped where every one of them ends up: the HTTP request. A harness +wraps its app in `RunModelKey`, which reads the token from the run's body (and takes the server's +props out of it, so no framework ever holds the token in its state), and keeps it for the run in +a context variable; every request the harness then sends through `httpx` (the OpenAI and Anthropic +SDKs, LiteLLM and the rest) carries it in place of the environment's key. + +ONLY TO THE GATEWAY. The token replaces a key only on a request whose path is the gateway's +(`/api/model-provider/`). A harness given a provider key of its own sends that key to the provider +as before, and a run that brought no token sends what it always sent. +""" + +from __future__ import annotations + +import json +from contextvars import ContextVar +from typing import Any + +GATEWAY_PATH = "/api/model-provider/" + +# The props only the server writes and only this module reads, taken out of every run. +SERVER_PROPS = ( + "openbotModelRoute", + "openbotOrganization", + "openbot_model_route", + "openbot_organization", +) + +_run_key: ContextVar[str | None] = ContextVar("openbot_run_model_key", default=None) + + +def run_key() -> str | None: + """The token of the run in progress, or None.""" + return _run_key.get() + + +def key_from(props: Any) -> str: + """The token the server sent in a run's forwarded props, or an empty string.""" + if not isinstance(props, dict): + return "" + route = props.get("openbotModelRoute") + if not isinstance(route, dict) or route.get("via") not in ("openai", "anthropic"): + return "" + token = route.get("token") + return token.strip() if isinstance(token, str) else "" + + +def with_run_key(headers: Any, path: str) -> None: + """The run's token in place of the key on a request to the gateway; nothing otherwise. + + OpenAI's SDK sends its key as `Authorization: Bearer`, Anthropic's as `x-api-key`. Each is + replaced where it is; a request with neither is given the bearer the gateway reads first. + """ + key = _run_key.get() + if not key or GATEWAY_PATH not in path: + return + had_api_key = "x-api-key" in headers + if had_api_key: + headers["x-api-key"] = key + if "authorization" in headers or not had_api_key: + headers["authorization"] = f"Bearer {key}" + + +_installed = False + + +def install() -> None: + """Every HTTP request of this process carries the run's token to the gateway. Idempotent. + + `httpx` and `httpx2`, the fork the OpenAI and Anthropic SDKs moved to; a framework that uses + either sends through `Client.send` or `AsyncClient.send`, whatever it builds on top. + """ + global _installed + if _installed: + return + import importlib + + for name in ("httpx", "httpx2"): + try: + module = importlib.import_module(name) + except ImportError: + continue + _patch(module.Client, module.AsyncClient) + _installed = True + + +def _patch(client, async_client) -> None: + sync_send = client.send + async_send = async_client.send + + def send(self, request, *args, **kwargs): + with_run_key(request.headers, request.url.path) + return sync_send(self, request, *args, **kwargs) + + async def asend(self, request, *args, **kwargs): + with_run_key(request.headers, request.url.path) + return await async_send(self, request, *args, **kwargs) + + client.send = send + async_client.send = asend + + +class RunModelKey: + """ASGI middleware: the run's token kept for the run, and the server's props taken out. + + Reads a POST's JSON body whole, takes `SERVER_PROPS` out of its `forwardedProps`, and hands the + app the body without them. The token is kept in a context variable for as long as the app + answers, which every task the app starts inherits. + """ + + def __init__(self, app): + self.app = app + install() + + async def __call__(self, scope, receive, send): + if scope.get("type") != "http" or scope.get("method") != "POST": + await self.app(scope, receive, send) + return + chunks = [] + while True: + message = await receive() + if message["type"] != "http.request": + # The client went away before the body was whole: nothing to run. + await self.app(scope, _replay(b"", receive), send) + return + chunks.append(message.get("body", b"")) + if not message.get("more_body", False): + break + body = b"".join(chunks) + token = "" + try: + parsed = json.loads(body) if body else None + except ValueError: + parsed = None + if isinstance(parsed, dict): + props = parsed.get("forwardedProps") + if not isinstance(props, dict): + props = parsed.get("forwarded_props") + if isinstance(props, dict): + token = key_from(props) + if any(name in props for name in SERVER_PROPS): + for name in SERVER_PROPS: + props.pop(name, None) + body = json.dumps(parsed).encode("utf-8") + scope = { + **scope, + "headers": [ + (name, value) + for name, value in scope.get("headers", []) + if name.lower() != b"content-length" + ] + + [(b"content-length", str(len(body)).encode("ascii"))], + } + reset = _run_key.set(token or None) + try: + await self.app(scope, _replay(body, receive), send) + finally: + _run_key.reset(reset) + + +def _replay(body: bytes, receive): + """`receive`, answering the body once and then whatever the client sends next.""" + sent = False + + async def replay(): + nonlocal sent + if not sent: + sent = True + return {"type": "http.request", "body": body, "more_body": False} + return await receive() + + return replay diff --git a/tests/compose.test.ts b/tests/compose.test.ts index 1407394b48..6f16a17173 100644 --- a/tests/compose.test.ts +++ b/tests/compose.test.ts @@ -114,6 +114,8 @@ function runLangGraphAguiModelProbe( join(dir, "fastapi", "__init__.py"), [ "class FastAPI:", + " def add_middleware(self, *_args, **_kwargs):", + " pass", " def middleware(self, *_args, **_kwargs):", " def decorator(fn):", " return fn",