From a05b073eab9eaf9f87ca28bc29d4de1a98351480 Mon Sep 17 00:00:00 2001 From: Titan-Frank <2434393892@qq.com> Date: Thu, 27 Aug 2026 11:20:39 +0800 Subject: [PATCH] fix: restore Codex MCP compatibility --- .github/workflows/checks.yml | 9 +- Makefile | 1 + README.md | 6 +- backend/app/services/agents/codex_adapter.py | 142 ++++++++++++++++-- backend/requirements.txt | 3 + backend/tests/test_codex_adapter.py | 132 ++++++++++++++++ docs/RELEASE-PACKAGE.md | 15 +- docs/agents/SETUP.md | 4 +- .../adr-003-install-configure-split.md | 7 +- docs/migration/from-setup-scripts.md | 5 +- docs/profiles/harness.md | 6 + docs/profiles/webui.md | 7 + installers/checks/check_codex_config.py | 109 ++++++++++++++ installers/configure_agent.sh | 25 ++- installers/lib/codex_config.py | 118 ++++++++++++--- 15 files changed, 537 insertions(+), 52 deletions(-) create mode 100644 backend/tests/test_codex_adapter.py create mode 100644 installers/checks/check_codex_config.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2ae2f73..acadcba 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -56,6 +56,9 @@ jobs: - name: Documentation links resolve run: python3 installers/checks/check_docs.py + - name: Codex MCP config merge is safe and compatible + run: python3 installers/checks/check_codex_config.py + - name: Installer dry-run for every profile run: | for p in webui harness skills; do @@ -100,12 +103,12 @@ print('TOML parser available') printf 'this is [not valid toml\n' > /tmp/bad.toml before=$(md5sum /tmp/bad.toml | cut -d' ' -f1) - python3 installers/lib/codex_config.py /tmp/bad.toml http://localhost:8000/mcp >/dev/null 2>&1 + python3 installers/lib/codex_config.py /tmp/bad.toml http://localhost:8000/mcp /usr/bin/python3 >/dev/null 2>&1 [ $? -eq 0 ] && { echo "::error::accepted invalid TOML"; fail=1; } [ "$(md5sum /tmp/bad.toml | cut -d' ' -f1)" = "$before" ] || { echo "::error::modified invalid TOML"; fail=1; } printf '[mcp_servers.dataflow]\ncommand = "python"\n' > /tmp/stdio.toml - python3 installers/lib/codex_config.py /tmp/stdio.toml http://localhost:8000/mcp >/dev/null 2>&1 + python3 installers/lib/codex_config.py /tmp/stdio.toml http://localhost:8000/mcp /usr/bin/python3 >/dev/null 2>&1 [ $? -eq 4 ] || { echo "::error::did not refuse a command/stdio TOML entry"; fail=1; } printf '{"mcpServers":{"dataflow":{"command":"python"}}}\n' > /tmp/stdio.json @@ -118,7 +121,7 @@ print('TOML parser available') # A quoted table name must be updated in place, not duplicated. printf '[mcp_servers."dataflow"]\nurl = "http://localhost:8000/mcp"\n' > /tmp/quoted.toml - out=$(python3 installers/lib/codex_config.py /tmp/quoted.toml http://localhost:8000/mcp 2>/dev/null) + out=$(python3 installers/lib/codex_config.py /tmp/quoted.toml http://localhost:8000/mcp /usr/bin/python3 2>/dev/null) n=$(printf '%s\n' "$out" | grep -c 'mcp_servers') [ "$n" -eq 1 ] || { echo "::error::quoted table produced $n dataflow tables"; fail=1; } diff --git a/Makefile b/Makefile index fa660b2..e371fdc 100644 --- a/Makefile +++ b/Makefile @@ -32,5 +32,6 @@ check: skills @$(PYTHON) installers/checks/check_mcp_whitelist.py @$(PYTHON) installers/checks/check_profiles.py @$(PYTHON) installers/checks/check_docs.py + @$(PYTHON) installers/checks/check_codex_config.py checks: check diff --git a/README.md b/README.md index 218cf4e..29c8f84 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ export ANTHROPIC_API_KEY=sk-ant-... # Codex (API key or `codex login` OAuth) npm install --global @openai/codex codex login # OAuth, or export OPENAI_API_KEY=sk-... -./install.sh configure-agent --agent codex +./install.sh configure-agent --agent codex --scope user # Cursor: install the IDE, open this repository, then: ./install.sh configure-agent --agent cursor @@ -167,6 +167,10 @@ Agent configuration is intentionally a separate step from installation. The installer does not write API keys. See [Agent setup](docs/agents/SETUP.md) for the exact authorization boundaries and verification steps. +For Codex, the configurator writes a stdio bridge backed by the WebUI Python +environment's `mcp-proxy`. This is required because the bundled DataFlow MCP +endpoint uses legacy SSE while current Codex URL entries expect streamable HTTP. + ## Installing never writes agent configuration Installing and configuring an agent are two separate commands, on purpose: diff --git a/backend/app/services/agents/codex_adapter.py b/backend/app/services/agents/codex_adapter.py index d91820f..74818f9 100644 --- a/backend/app/services/agents/codex_adapter.py +++ b/backend/app/services/agents/codex_adapter.py @@ -10,7 +10,9 @@ 2. **System prompt delivery** — ``codex exec`` does not accept a system prompt flag in headless mode. We prepend the harness rules to the user message, same approach as Cursor. -3. **Tool approval** — ``--sandbox workspace-write`` grants full auto-approve. +3. **Tool approval** — ``--approve-for-me`` reviews headless tool calls + automatically and applies Codex's workspace-write sandbox. Current Codex + releases reject combining this flag with an explicit ``--sandbox`` value. 4. **Session resume** — ``codex exec`` itself does not accept ``--resume``; the top-level ``codex resume`` subcommand is for interactive sessions. We treat each turn as stateless and rely on Codex's own conversation @@ -26,6 +28,7 @@ import asyncio import json +from collections import deque from typing import AsyncGenerator, Optional from app.core.logger_setup import get_logger @@ -44,6 +47,34 @@ def _format_user_message(system_prompt: str, message: str) -> str: ) +def _build_exec_command( + cli_path: str, + webui_root: str, + system_prompt: str, + message: str, +) -> list[str]: + """Build a command compatible with current headless Codex CLI releases.""" + return [ + cli_path, + "exec", + "--json", + "--approve-for-me", + "--cd", + webui_root, + _format_user_message(system_prompt, message), + ] + + +def _format_process_error(returncode: int, stderr: str) -> str: + """Turn a failed Codex invocation into a useful WebUI error.""" + detail = "\n".join(line for line in stderr.splitlines() if line.strip()) + if len(detail) > 1200: + detail = detail[-1200:] + if detail: + return f"Codex exited with status {returncode}: {detail}" + return f"Codex exited with status {returncode} without an error message." + + class CodexAdapter(AgentAdapter): kind = "codex" @@ -73,13 +104,12 @@ async def chat_stream( import os # Codex's `exec` does not resume; session_id is ignored. - cmd = [ - self.cli_path, "exec", - "--json", - "--sandbox", "workspace-write", - "--cwd", str(self.webui_root), - _format_user_message(self.system_prompt, message), - ] + cmd = _build_exec_command( + self.cli_path, + str(self.webui_root), + self.system_prompt, + message, + ) # Forward auth-related env vars. Codex config.toml may reference # CODEX_API_KEY, OPENAI_API_KEY, or OPENAI_BASE_URL depending on @@ -96,6 +126,17 @@ async def chat_stream( ) self._process = process emitted_session = False + emitted_error = False + stderr_lines: deque[str] = deque(maxlen=40) + + async def drain_stderr() -> None: + assert process.stderr is not None + async for raw in process.stderr: + line = raw.decode("utf-8", errors="replace").rstrip() + if line: + stderr_lines.append(line) + + stderr_task = asyncio.create_task(drain_stderr()) try: assert process.stdout is not None @@ -108,12 +149,12 @@ async def chat_stream( except json.JSONDecodeError: continue - # Codex emits a session-id at the top of the stream when - # available. Tolerate either a top-level field or a typed - # `session_started` event. + # Current Codex emits ``thread.started`` with ``thread_id``; + # older builds used session_id in one of two shapes. if not emitted_session: sid = ( - chunk.get("session_id") + chunk.get("thread_id") + or chunk.get("session_id") or (chunk.get("msg") or {}).get("session_id") ) if sid: @@ -121,8 +162,19 @@ async def chat_stream( yield {"type": "session", "session_id": sid} async for evt in self._translate(chunk): + if evt.get("type") == "error": + emitted_error = True yield evt + returncode = await process.wait() + await stderr_task + if returncode != 0 and not emitted_error: + yield { + "type": "error", + "message": _format_process_error( + returncode, "\n".join(stderr_lines) + ), + } yield {"type": "done"} except asyncio.CancelledError: yield {"type": "done"} @@ -137,10 +189,74 @@ async def chat_stream( process.kill() except Exception: pass + if not stderr_task.done(): + stderr_task.cancel() self._process = None async def _translate(self, chunk: dict) -> AsyncGenerator[NormalizedEvent, None]: - # Codex events are typed via msg.type per the docs. + # Codex 0.149+ uses top-level lifecycle events with an ``item`` payload. + ctype = chunk.get("type", "") + item = chunk.get("item") if isinstance(chunk.get("item"), dict) else {} + item_type = item.get("type", "") + + if ctype == "item.completed" and item_type == "agent_message": + text = item.get("text", "") + if text: + yield {"type": "text_chunk", "content": str(text)} + return + + if ctype == "item.started" and item_type == "mcp_tool_call": + server = item.get("server", "") + tool = item.get("tool", "") + name = f"mcp__{server}__{tool}" if server and tool else tool + yield { + "type": "tool_call_start", + "tool_use_id": item.get("id", ""), + "name": name, + "input_preview": truncate_preview(item.get("arguments") or {}), + } + return + + if ctype == "item.completed" and item_type == "mcp_tool_call": + error = item.get("error") + yield { + "type": "tool_call_end", + "tool_use_id": item.get("id", ""), + "is_error": bool(error), + "output_preview": truncate_preview(error or item.get("result") or ""), + } + return + + if ctype == "item.started" and item_type == "command_execution": + yield { + "type": "tool_call_start", + "tool_use_id": item.get("id", ""), + "name": "command_execution", + "input_preview": truncate_preview(item.get("command") or ""), + } + return + + if ctype == "item.completed" and item_type == "command_execution": + exit_code = item.get("exit_code") + yield { + "type": "tool_call_end", + "tool_use_id": item.get("id", ""), + "is_error": exit_code not in (None, 0), + "output_preview": truncate_preview(item.get("aggregated_output") or ""), + } + return + + if ctype in ("turn.failed", "error"): + error = chunk.get("error") + if isinstance(error, dict): + message = error.get("message") or str(error) + else: + message = chunk.get("message") or str(error or chunk) + yield {"type": "error", "message": message} + return + + # Older Codex events were typed via msg.type. Keep accepting them so + # the WebUI does not become tied to one narrow CLI release. msg = chunk.get("msg") if isinstance(chunk.get("msg"), dict) else chunk mtype = msg.get("type", "") diff --git a/backend/requirements.txt b/backend/requirements.txt index ace8277..a11df4a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,6 +3,9 @@ fastapi[standard] # MCP 2.x made description keyword-only, which raises a TypeError at startup. fastapi-mcp==0.4.0 mcp>=1.12,<2 +# Codex supports streamable HTTP, while fastapi-mcp 0.4.0 exposes legacy SSE. +# Run the remote SSE endpoint through a local stdio bridge for Codex. +mcp-proxy==0.12.0 uvicorn[standard] pydantic_settings pandas diff --git a/backend/tests/test_codex_adapter.py b/backend/tests/test_codex_adapter.py new file mode 100644 index 0000000..4ab81c7 --- /dev/null +++ b/backend/tests/test_codex_adapter.py @@ -0,0 +1,132 @@ +"""Compatibility checks for the headless Codex CLI adapter.""" + +import asyncio + +from app.services.agents.codex_adapter import ( + CodexAdapter, + _build_exec_command, + _format_process_error, +) + + +def test_build_exec_command_uses_current_cli_flags(tmp_path): + command = _build_exec_command( + "codex", + str(tmp_path), + "system rules", + "user request", + ) + + assert command[:6] == [ + "codex", + "exec", + "--json", + "--approve-for-me", + "--cd", + str(tmp_path), + ] + assert "--sandbox" not in command + assert command[6] == ( + "[DataFlow-WebUI harness rules — these override your defaults]\n" + "system rules\n\n" + "[User message]\n" + "user request" + ) + + +def _translate(adapter, chunk): + async def collect(): + return [event async for event in adapter._translate(chunk)] + + return asyncio.run(collect()) + + +def test_translate_current_agent_message(tmp_path): + adapter = CodexAdapter( + cli_path="codex", + webui_root=tmp_path, + mcp_config_path=tmp_path / ".mcp.json", + system_prompt="rules", + allowed_tools="", + ) + + assert _translate(adapter, { + "type": "item.completed", + "item": {"id": "item_1", "type": "agent_message", "text": "你好!"}, + }) == [{"type": "text_chunk", "content": "你好!"}] + + +def test_translate_current_mcp_tool_lifecycle(tmp_path): + adapter = CodexAdapter( + cli_path="codex", + webui_root=tmp_path, + mcp_config_path=tmp_path / ".mcp.json", + system_prompt="rules", + allowed_tools="", + ) + + started = _translate(adapter, { + "type": "item.started", + "item": { + "id": "item_2", + "type": "mcp_tool_call", + "server": "dataflow", + "tool": "list_operator_categories", + "arguments": {}, + }, + }) + completed = _translate(adapter, { + "type": "item.completed", + "item": { + "id": "item_2", + "type": "mcp_tool_call", + "result": {"content": [{"type": "text", "text": "ok"}]}, + "error": None, + }, + }) + + assert started == [{ + "type": "tool_call_start", + "tool_use_id": "item_2", + "name": "mcp__dataflow__list_operator_categories", + "input_preview": "{}", + }] + assert completed[0]["type"] == "tool_call_end" + assert completed[0]["tool_use_id"] == "item_2" + assert completed[0]["is_error"] is False + assert '"text": "ok"' in completed[0]["output_preview"] + + +def test_format_process_error_includes_exit_status_and_stderr(): + message = _format_process_error(2, "first line\nargument conflict\n") + + assert "status 2" in message + assert "argument conflict" in message + + +def test_chat_stream_surfaces_nonzero_exit_stderr(tmp_path): + fake_codex = tmp_path / "fake-codex" + fake_codex.write_text( + "#!/bin/sh\n" + "echo 'simulated CLI argument failure' >&2\n" + "exit 2\n" + ) + fake_codex.chmod(0o755) + adapter = CodexAdapter( + cli_path=str(fake_codex), + webui_root=tmp_path, + mcp_config_path=tmp_path / ".mcp.json", + system_prompt="rules", + allowed_tools="", + ) + + async def collect(): + return [event async for event in adapter.chat_stream("hello")] + + events = asyncio.run(collect()) + + assert events[-1] == {"type": "done"} + errors = [event for event in events if event["type"] == "error"] + assert len(errors) == 1 + assert "status 2" in errors[0]["message"] + assert "simulated CLI argument failure" in errors[0]["message"] diff --git a/docs/RELEASE-PACKAGE.md b/docs/RELEASE-PACKAGE.md index e8ac351..da5953d 100644 --- a/docs/RELEASE-PACKAGE.md +++ b/docs/RELEASE-PACKAGE.md @@ -66,9 +66,14 @@ cd .. Codex 则在 `~/.codex/config.toml` 追加: +先确认发布包的 Python 环境已安装 `mcp-proxy`(后端依赖默认会安装),并取得 +解释器绝对路径:`python -c "import sys; print(sys.executable)"`。然后把下方 +`command` 替换为该路径: + ```toml [mcp_servers.dataflow] -url = "http://localhost:8000/mcp" +command = "/absolute/path/to/python" +args = ["-m", "mcp_proxy", "http://localhost:8000/mcp"] enabled = true tool_timeout_sec = 120 ``` @@ -126,9 +131,15 @@ by hand: For Codex, append to `~/.codex/config.toml`: +First confirm that `mcp-proxy` is installed in the release package's Python +environment (the backend requirements install it), then get the interpreter's +absolute path with `python -c "import sys; print(sys.executable)"`. Replace the +`command` value below with that path: + ```toml [mcp_servers.dataflow] -url = "http://localhost:8000/mcp" +command = "/absolute/path/to/python" +args = ["-m", "mcp_proxy", "http://localhost:8000/mcp"] enabled = true tool_timeout_sec = 120 ``` diff --git a/docs/agents/SETUP.md b/docs/agents/SETUP.md index 5802ce5..2a0e253 100644 --- a/docs/agents/SETUP.md +++ b/docs/agents/SETUP.md @@ -71,7 +71,9 @@ Rules you must follow: - **Codex is the exception:** it reads only `~/.codex/config.toml` and has no flag to point elsewhere, so it has no project scope. `--scope user` is required, and the command refuses project scope rather than writing a file Codex ignores. Ask - the user before configuring Codex. + the user before configuring Codex. Its entry launches the active backend Python + as a stdio `mcp-proxy` bridge because the DataFlow endpoint uses legacy SSE and + current Codex direct URL entries expect streamable HTTP. - Never write API keys anywhere. Agents read credentials from environment variables at run time. If auth is missing, tell the user which variable to export — do not ask them to paste a key into the chat, and do not put one in a diff --git a/docs/architecture/adr-003-install-configure-split.md b/docs/architecture/adr-003-install-configure-split.md index adab80c..ac679a4 100644 --- a/docs/architecture/adr-003-install-configure-split.md +++ b/docs/architecture/adr-003-install-configure-split.md @@ -48,9 +48,10 @@ Rules for `configure-agent`: cannot validate. Other servers and unmanaged keys inside the `dataflow` entry are preserved. 4. **Refuse conflicts instead of resolving them.** An existing `dataflow` entry - pointing at a different URL, or configured as a `command`/stdio server rather - than SSE, requires `--force`. Silently merging a URL into a stdio definition - would produce a hybrid that is valid as neither. + pointing at a different endpoint, or launching an unrelated command, requires + `--force`. A matching legacy Codex URL entry is migrated automatically to a + stdio `mcp-proxy` bridge because current Codex treats direct URLs as streamable + HTTP while this backend exposes legacy SSE. 5. **Never touch credentials.** No key is read, written or logged. Agents get credentials from the environment at run time. 6. **Idempotent.** Re-running reports "no change" instead of rewriting. diff --git a/docs/migration/from-setup-scripts.md b/docs/migration/from-setup-scripts.md index 217b633..4448630 100644 --- a/docs/migration/from-setup-scripts.md +++ b/docs/migration/from-setup-scripts.md @@ -54,8 +54,9 @@ three config files, two of them in `$HOME`, without asking. ## If the old script already modified your config -Nothing needs undoing. The new configurator is idempotent and merges: it will -report "no change" if your existing `dataflow` entry already matches. +Nothing needs undoing. The new configurator is idempotent and merges. A matching +legacy Codex `url` entry is automatically migrated to the required stdio +`mcp-proxy` bridge; an already matching bridge reports "no change". Worth checking once, if you had other MCP servers configured before running the old script — its Cursor path could overwrite `~/.cursor/mcp.json`: diff --git a/docs/profiles/harness.md b/docs/profiles/harness.md index c797f6e..fa6b15a 100644 --- a/docs/profiles/harness.md +++ b/docs/profiles/harness.md @@ -98,6 +98,12 @@ pip install -r installers/requirements-configure.txt Without it, `configure-agent --agent codex` refuses and prints the block to add by hand. Claude and Cursor are unaffected — their configs are JSON. +The harness requirements include `mcp-proxy`. The Codex configurator records +the active Python interpreter as a stdio bridge to the backend's legacy SSE MCP +endpoint. Run it from the same activated environment used to install the +harness; a direct `url = "http://localhost:8000/mcp"` entry is not compatible +with current Codex releases. + ## Minimal verification With the backend running: diff --git a/docs/profiles/webui.md b/docs/profiles/webui.md index c525689..54ef8e0 100644 --- a/docs/profiles/webui.md +++ b/docs/profiles/webui.md @@ -81,6 +81,13 @@ pip install -r installers/requirements-configure.txt Without it, `configure-agent --agent codex` refuses and prints the block to add by hand. Claude and Cursor are unaffected — their configs are JSON. +The WebUI requirements include `mcp-proxy`. The Codex configurator resolves the +active Python interpreter to an absolute path and writes a stdio bridge that +forwards to `http://localhost:8000/mcp`. This bridges the backend's legacy SSE +transport to current Codex releases, whose direct URL entries use streamable +HTTP. Run the configurator from the same activated environment used to install +the backend. + ### Auth ```bash diff --git a/installers/checks/check_codex_config.py b/installers/checks/check_codex_config.py new file mode 100644 index 0000000..9b2ac6e --- /dev/null +++ b/installers/checks/check_codex_config.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Regression checks for the conservative Codex MCP config merger.""" + +from __future__ import annotations + +import io +import sys +from contextlib import redirect_stderr +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from installers.lib.codex_config import load_parser, merge + + +URL = "http://localhost:8000/mcp" +PYTHON = "/opt/dataflow/bin/python" + + +def parsed(text: str) -> dict: + loads = load_parser() + assert loads is not None, "a real TOML parser is required" + return loads(text) + + +def check_new_bridge() -> None: + out, code = merge('[mcp_servers.other]\nurl = "https://example.test/mcp"\n', URL, PYTHON, False) + assert code == 0 + config = parsed(out) + assert config["mcp_servers"]["other"]["url"] == "https://example.test/mcp" + dataflow = config["mcp_servers"]["dataflow"] + assert dataflow["command"] == PYTHON + assert dataflow["args"] == ["-m", "mcp_proxy", URL] + assert "url" not in dataflow + + +def check_legacy_url_migration() -> None: + source = ( + '[mcp_servers.dataflow]\n' + 'url = "http://127.0.0.1:8000/mcp"\n' + 'enabled = false\n' + 'custom = "keep-me"\n' + ) + out, code = merge(source, URL, PYTHON, False) + assert code == 0 + dataflow = parsed(out)["mcp_servers"]["dataflow"] + assert dataflow["command"] == PYTHON + assert dataflow["args"] == ["-m", "mcp_proxy", URL] + assert dataflow["enabled"] is False + assert dataflow["custom"] == "keep-me" + assert "url" not in dataflow + + +def check_equivalent_binary_bridge_is_preserved() -> None: + source = ( + '[mcp_servers."dataflow"]\n' + 'command = "/custom/bin/mcp-proxy"\n' + 'args = ["http://127.0.0.1:8000/mcp"]\n' + 'tool_timeout_sec = 30\n' + ) + out, code = merge(source, URL, PYTHON, False) + assert code == 0 + dataflow = parsed(out)["mcp_servers"]["dataflow"] + assert dataflow["command"] == "/custom/bin/mcp-proxy" + assert dataflow["args"] == ["http://127.0.0.1:8000/mcp"] + assert dataflow["tool_timeout_sec"] == 30 + + +def check_conflict_and_force() -> None: + source = ( + '[mcp_servers.dataflow]\n' + 'command = "foreign-server"\n' + 'args = ["--serve"]\n' + 'env = { TOKEN = "do-not-forward" }\n' + ) + with redirect_stderr(io.StringIO()): + _, code = merge(source, URL, PYTHON, False) + assert code == 4 + + out, code = merge(source, URL, PYTHON, True) + assert code == 0 + dataflow = parsed(out)["mcp_servers"]["dataflow"] + assert dataflow["command"] == PYTHON + assert dataflow["args"] == ["-m", "mcp_proxy", URL] + assert "env" not in dataflow + + +def check_invalid_and_inline_toml_are_refused() -> None: + with redirect_stderr(io.StringIO()): + _, code = merge("this is [not toml", URL, PYTHON, False) + assert code == 3 + + inline = f'mcp_servers = {{ dataflow = {{ url = "{URL}" }} }}\n' + with redirect_stderr(io.StringIO()): + _, code = merge(inline, URL, PYTHON, False) + assert code == 4 + + +def main() -> None: + check_new_bridge() + check_legacy_url_migration() + check_equivalent_binary_bridge_is_preserved() + check_conflict_and_force() + check_invalid_and_inline_toml_are_refused() + print("Codex MCP config checks passed") + + +if __name__ == "__main__": + main() diff --git a/installers/configure_agent.sh b/installers/configure_agent.sh index b5034f2..21ed0d4 100755 --- a/installers/configure_agent.sh +++ b/installers/configure_agent.sh @@ -42,8 +42,8 @@ Options: user → writes into your home directory, after showing a diff --dry-run Print the exact file contents that would be written; write nothing --yes Skip the confirmation prompt for --scope user - --force Redirect an existing 'dataflow' MCP entry that points at a - different URL. Without this, such a conflict is refused. + --force Replace an existing 'dataflow' MCP entry that points at a + different endpoint/command. Without this, conflicts are refused. --verbose Show extra detail -h, --help This message @@ -186,12 +186,24 @@ merge_codex_toml() { # Delegates to installers/lib/codex_config.py, which parses the file as TOML # before touching it, preserves keys we do not manage, and refuses to redirect # an existing dataflow entry that points somewhere else. - local path="$1" rc=0 out="" + local path="$1" rc=0 out="" python_exec="" local force_arg="" [[ "$FORCE" -eq 1 ]] && force_arg="--force" + python_exec=$("$DF_PYTHON" -c 'import sys; print(sys.executable)') || { + err "cannot resolve the Python interpreter used for the Codex MCP bridge." + return 1 + } + if ! "$python_exec" -c 'import mcp_proxy' >/dev/null 2>&1; then + err "mcp-proxy is not installed in $python_exec." + err "Install the WebUI backend requirements first:" + err " $python_exec -m pip install -r backend/requirements.txt" + return 1 + fi + out=$("$DF_PYTHON" "$DF_REPO_ROOT/installers/lib/codex_config.py" \ - "$path" "$DF_MCP_URL" $force_arg 2>/tmp/df_codex_err.$$) || rc=$? + "$path" "$DF_MCP_URL" "$python_exec" $force_arg \ + 2>/tmp/df_codex_err.$$) || rc=$? case "$rc" in 0) printf '%s' "$out" ;; @@ -203,7 +215,8 @@ merge_codex_toml() { err " - add the block by hand:" err "" err " [mcp_servers.dataflow]" - err " url = \"$DF_MCP_URL\"" + err " command = \"$python_exec\"" + err " args = [\"-m\", \"mcp_proxy\", \"$DF_MCP_URL\"]" err " enabled = true" err " tool_timeout_sec = 120" rm -f /tmp/df_codex_err.$$ @@ -286,7 +299,7 @@ case "$AGENT" in info "" info "Codex reads its config at startup — restart the session to pick this up." info "Verify:" - info " codex exec --json --full-auto \\" + info " codex exec --json --approve-for-me \\" info " \"call the dataflow MCP tool list_operator_categories and report the result\"" ;; esac diff --git a/installers/lib/codex_config.py b/installers/lib/codex_config.py index dc33e28..47cd7ef 100644 --- a/installers/lib/codex_config.py +++ b/installers/lib/codex_config.py @@ -10,11 +10,16 @@ and silently corrupts them. * **Never drop a key.** Keys inside the managed table that we do not own are preserved verbatim. +* **Bridge the legacy SSE endpoint.** Current Codex releases accept streamable + HTTP URLs, while ``fastapi-mcp==0.4.0`` serves legacy SSE. The managed entry + therefore launches ``mcp-proxy`` through the backend Python interpreter and + talks to Codex over stdio. * **Refuse ambiguity.** An existing ``dataflow`` server that points elsewhere, - or that is configured as a ``command``/stdio server rather than SSE, is a real - conflict. Redirecting it requires ``--force``. + or that launches an unrelated command, is a real conflict. Redirecting it + requires ``--force``. The URL form written by older DataFlow installers is + migrated automatically when it points at the same endpoint. -Usage: codex_config.py [--force] +Usage: codex_config.py [--force] Exit codes: 0 merged content written to stdout @@ -25,9 +30,11 @@ from __future__ import annotations +import json import re import sys from pathlib import Path +from urllib.parse import urlsplit MANAGED_KEY = "dataflow" MANAGED_PREFIX = "mcp_servers" @@ -56,7 +63,54 @@ def load_parser(): return None -def merge(text: str, url: str, force: bool) -> tuple[str, int]: +def _bridge_values(python_command: str, url: str) -> tuple[str, list[str]]: + return python_command, ["-m", "mcp_proxy", url] + + +def _urls_equivalent(left: str, right: str) -> bool: + if left == right: + return True + try: + a = urlsplit(left) + b = urlsplit(right) + loopback = {"localhost", "127.0.0.1", "::1"} + return ( + a.scheme == b.scheme + and a.hostname in loopback + and b.hostname in loopback + and a.port == b.port + and a.path == b.path + and a.query == b.query + and a.fragment == b.fragment + ) + except ValueError: + return False + + +def _is_equivalent_bridge(existing: dict, python_command: str, url: str) -> bool: + command = existing.get("command") + args = existing.get("args") + expected_command, expected_args = _bridge_values(python_command, url) + if ( + command == expected_command + and isinstance(args, list) + and len(args) == 3 + and args[:2] == expected_args[:2] + and isinstance(args[2], str) + and _urls_equivalent(args[2], url) + ): + return True + if isinstance(command, str) and Path(command).name in {"mcp-proxy", "mcp-proxy.exe"}: + return ( + isinstance(args, list) + and len(args) == 1 + and isinstance(args[0], str) + and _urls_equivalent(args[0], url) + ) + return False + + +def merge(text: str, url: str, python_command: str, force: bool) -> tuple[str, int]: loads = load_parser() if loads is None: sys.stderr.write( @@ -77,23 +131,29 @@ def merge(text: str, url: str, force: bool) -> tuple[str, int]: servers = parsed.get(MANAGED_PREFIX) existing = servers.get(MANAGED_KEY) if isinstance(servers, dict) else None + preserve_existing_bridge = False + replacing_foreign_command = False if existing is not None: if not isinstance(existing, dict): sys.stderr.write(f"[{MANAGED_PREFIX}.{MANAGED_KEY}] is not a table\n") return "", 3 current_url = existing.get("url") - # A command/stdio server is a different transport, not a stale URL. - # Merging our url into it would produce a hybrid config that is not a - # valid server definition of either kind. if "command" in existing: - if not force: + if _is_equivalent_bridge(existing, python_command, url): + preserve_existing_bridge = True + elif not force: sys.stderr.write( f"existing [{MANAGED_PREFIX}.{MANAGED_KEY}] is a command/stdio server " - f"(command = {existing['command']!r}), not an SSE server.\n" - "Converting it would produce a mixed command+url definition.\n" + f"(command = {existing['command']!r}), not the DataFlow MCP bridge.\n" ) return "", 4 - elif isinstance(current_url, str) and current_url != url and not force: + else: + replacing_foreign_command = True + elif ( + isinstance(current_url, str) + and not _urls_equivalent(current_url, url) + and not force + ): sys.stderr.write( f"existing [{MANAGED_PREFIX}.{MANAGED_KEY}] points at {current_url!r}, " f"not {url!r}.\n" @@ -116,13 +176,21 @@ def merge(text: str, url: str, force: bool) -> tuple[str, int]: end = len(lines) if start is None: + if existing is not None: + sys.stderr.write( + f"existing [{MANAGED_PREFIX}.{MANAGED_KEY}] uses an inline or " + "otherwise unsupported TOML form; refusing to append a duplicate table.\n" + ) + return "", 4 out = list(lines) while out and not out[-1].strip(): out.pop() if out: out.append("") out.append(f"[{MANAGED_PREFIX}.{MANAGED_KEY}]") - out.append(f'url = "{url}"') + command, args = _bridge_values(python_command, url) + out.append(f"command = {json.dumps(command)}") + out.append(f"args = {json.dumps(args)}") for key, value in DEFAULTS.items(): out.append(f"{key} = {value}") return "\n".join(out) + "\n", 0 @@ -135,9 +203,11 @@ def merge(text: str, url: str, force: bool) -> tuple[str, int]: body.pop() trailing_blanks += 1 - # With --force on a command/stdio entry, the transport keys must go: leaving - # them beside url is the hybrid config we refuse to create. - drop_keys = {"command", "args", "env"} if (existing and "command" in existing) else set() + if preserve_existing_bridge: + bridge_command = existing["command"] + bridge_args = existing["args"] + else: + bridge_command, bridge_args = _bridge_values(python_command, url) seen: set[str] = set() new_body: list[str] = [] @@ -147,12 +217,14 @@ def merge(text: str, url: str, force: bool) -> tuple[str, int]: new_body.append(raw) continue key = m.group(1).strip("\"'") - if key in drop_keys: + if key in {"url", "command", "args"}: + continue + if replacing_foreign_command and key == "env": continue seen.add(key) - new_body.append(f'url = "{url}"' if key == "url" else raw) - if "url" not in seen: - new_body.insert(0, f'url = "{url}"') + new_body.append(raw) + new_body.insert(0, f"args = {json.dumps(bridge_args)}") + new_body.insert(0, f"command = {json.dumps(bridge_command)}") for key, value in DEFAULTS.items(): if key not in seen: new_body.append(f"{key} = {value}") @@ -172,12 +244,16 @@ def merge(text: str, url: str, force: bool) -> tuple[str, int]: def main() -> int: + if len(sys.argv) < 4: + sys.stderr.write("usage: codex_config.py [--force]\n") + return 1 path = Path(sys.argv[1]) url = sys.argv[2] - force = len(sys.argv) > 3 and sys.argv[3] == "--force" + python_command = sys.argv[3] + force = len(sys.argv) > 4 and sys.argv[4] == "--force" text = path.read_text(encoding="utf-8") if path.is_file() else "" - out, code = merge(text, url, force) + out, code = merge(text, url, python_command, force) if code != 0: return code sys.stdout.write(out)