From 8a9ee0d8af51fe5acf477fb12741f9b29e1e523e Mon Sep 17 00:00:00 2001 From: David Liu Date: Thu, 17 Sep 2026 03:10:59 +0000 Subject: [PATCH 1/9] [AIGTWY-4759] Managed integration tests: green lanes + coverage The managed integration lanes were failing (the managed workspace IP-blocked CI runners), so they were non-blocking and thin. This makes them work and adds real coverage. - Install both agent CLIs in each managed lane: a managed config enables both agents and `ug configure` applies it to every enabled agent, so a lane with only one agent failed configuring the other. - Live `managed` tests against ca-central's published config: per-agent apply and an idempotent re-configure. - New `managed_fixture` layer: inject the admin CodingAgentConfig via the built-in UCODE_MANAGED_CONFIG_STUB hook so the real `ug configure` path is exercised across shapes the live workspace does not publish (enabled-agents allowlist per agent, config-driven model list, tracing OTEL env). Each case differs from the published config in what it asserts, so a pass proves the injected config drove configure. Auth, config writers, and binaries stay real; the fetch/wire contract stays covered by the un-stubbed `managed` tests. - Sanction the input-only stub in tests/AGENTS.md and tests/CLAUDE.md; run both markers in the existing managed lanes. Lanes stay non-blocking until proven stable. Co-authored-by: Isaac --- .github/workflows/integration.yml | 15 +- tests/AGENTS.md | 8 +- tests/CLAUDE.md | 7 + tests/integration/README.md | 19 ++- tests/integration/pytest.ini | 1 + .../integration/test_ug_configure_managed.py | 24 ++++ .../test_ug_configure_managed_fixtures.py | 131 ++++++++++++++++++ 7 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_ug_configure_managed_fixtures.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cf2fbe13..7a2fcafd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -191,9 +191,9 @@ jobs: name: Managed config ยท ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 - # Temporarily non-blocking: the managed e2e workspace is not yet reachable from CI runners - # (token mint returns 403). Keep the job running for signal, but do not fail the workflow or - # gate merges on it until that workspace access is sorted. + # Kept non-blocking for now: the managed workspace (E2E_ADMIN_WORKSPACE) is repointed to + # runner-reachable ca-central, but leave these lanes for signal until the managed-config apply + # path is proven stable. Then drop this and add `managed` to the required set. continue-on-error: true timeout-minutes: 30 strategy: @@ -225,17 +225,16 @@ jobs: DATABRICKS_CLIENT_ID: ${{ secrets.E2E_ADMIN_SP_CLIENT_ID }} DATABRICKS_CLIENT_SECRET: ${{ secrets.E2E_ADMIN_SP_CLIENT_SECRET }} run: | - case "$AGENT" in - claude) args=(--claude-version "$CLAUDE_VERSION") ;; - codex) args=(--codex-version "$CODEX_VERSION") ;; - esac + # A managed config enables both agents and `ug configure` applies it to every enabled + # agent, so both CLIs must be installed even though this lane asserts only one agent. + args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION") if [[ -n "$DEPENDENCY" ]]; then args+=(--dependency "$DEPENDENCY") fi uv run --no-project --python 3.12 python scripts/run_integration.py \ --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ - "${args[@]}" -- -m "managed and $AGENT" + "${args[@]}" -- -m "(managed or managed_fixture) and $AGENT" - name: Upload managed test evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/tests/AGENTS.md b/tests/AGENTS.md index dd9d9ab4..6cc382d3 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -26,7 +26,13 @@ tests. Keep work scoped to the behavior requested by the user. 4. **No production changes just to make tests pass.** No test-only environment switches, special server branches, disabled validation, privileged-path overrides, or hardcoded success. Real bugs require normal production fixes - and regression coverage. Report failures instead of concealing them. + and regression coverage. Report failures instead of concealing them. The sole + exception is the `managed_fixture` marker: it uses the built-in + `UCODE_MANAGED_CONFIG_STUB` hook to inject the admin CodingAgentConfig INPUT so the + real `ug configure` path can be exercised across config shapes the live workspace does + not publish. The gateway, agent binaries, ug internals, and ug state stay real; the + config fetch/wire contract stays covered by the un-stubbed `managed` tests; and the + hook must never be used to disable validation or conceal a failure. 5. **Real responses and binaries.** Pin requested ug and agent versions. Never substitute a missing binary/service. Reuse explicit e2e workspace/auth settings; never pick a developer's Databricks profile automatically. diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index de5260a4..5714c263 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -10,3 +10,10 @@ calls, fake binaries/services, fabricated ug state, or test-only production behavior. Do not hide failures with skips, xfails, retries or weaker assertions. Update the matrix when coverage changes and distinguish automated coverage from gaps and checks that were not executed. + +The one carve-out is the `managed_fixture` marker: those tests inject the admin +CodingAgentConfig through the built-in `UCODE_MANAGED_CONFIG_STUB` hook so the real +`ug configure` path can be exercised across managed-config shapes the live workspace does +not publish. That controls only the admin-authored INPUT; the gateway, agent binaries, ug +internals, and ug state stay real, and the config fetch/wire contract itself stays covered +by the un-stubbed `managed` tests. Faking any of those remains banned. diff --git a/tests/integration/README.md b/tests/integration/README.md index d9056b2c..43c825bf 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -139,9 +139,12 @@ Those choices are recorded in `versions.json`. No service is created or modified A missing service or permission fails the selected CUJ, rather than skipping it. There are **39 live cases** (including 4 TUI journeys) and **5 installation -checks** with both agents. A separate **2 managed-workspace cases** (one per agent, -marker `managed`) run against a workspace that publishes a CodingAgentConfig; see -"Managed-workspace journeys" below. See the named coverage and gaps matrix in +checks** with both agents. A separate **3 managed-workspace cases** (one per agent +plus an idempotent re-configure, marker `managed`) run against a workspace that publishes a CodingAgentConfig; see +"Managed-workspace journeys" below. A further **4 `managed_fixture` cases** inject the admin config +locally (via `UCODE_MANAGED_CONFIG_STUB`) to cover shapes the live workspace does not publish; each +differs from the published config in what it asserts so it proves the injected config drove configure. +See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -237,8 +240,9 @@ compensate for capacity failures. Both matrices use `fail-fast: false` and uploa uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, and both full lanes to pass. The **Managed config** lanes run for signal but are temporarily -non-blocking (`continue-on-error`), because the managed e2e workspace is not yet reachable from -CI runners; they neither fail the workflow nor gate merges until that access is sorted. The +non-blocking (`continue-on-error`): the managed workspace is now runner-reachable, but the lanes +stay non-blocking until the managed-config apply path is proven stable. They neither fail the +workflow nor gate merges until then. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -252,6 +256,11 @@ each agent's generated config exposes exactly the admin's static `model_services (Claude's `availableModels`/`modelPicker`, Codex's model catalog). The expected model ids live in the test and mirror the published config; update them there if the admin list changes. +Treat that published CodingAgentConfig as shared CI fixture state. The managed lanes assert its +exact model ids and its both-agent enablement, so editing the managed workspace's config (models, +enabled agents, or defaults) breaks these lanes until the constants in `test_ug_configure_managed.py` +are updated to match. Do not change it casually. + That workspace authenticates as a service principal, so CI mints a short-lived token per run from these same-repository secrets rather than storing a long-lived bearer: diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 4b10e140..7ba34491 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -5,6 +5,7 @@ markers = installation: installed-package checks that require no workspace live: requires the real workspace used by the existing e2e suite managed: requires the managed e2e workspace that publishes a CodingAgentConfig + managed_fixture: real ug/TUI against a real workspace, but the managed CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB smoke: Databricks Hosted configure/TUI and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen claude: only runs when Claude Code is explicitly selected diff --git a/tests/integration/test_ug_configure_managed.py b/tests/integration/test_ug_configure_managed.py index ef1b2c2b..4909bf5c 100644 --- a/tests/integration/test_ug_configure_managed.py +++ b/tests/integration/test_ug_configure_managed.py @@ -74,3 +74,27 @@ def test_ug_configure_managed_codex(live_session, workspace): with AgentTerminal(session, "codex", [str(session.binary), "codex"], "managed-codex") as tui: tui.boot() tui.check_input_and_exit() + + +@pytest.mark.managed +@pytest.mark.claude +def test_ug_configure_managed_is_idempotent(live_session, workspace): + """Scenario: run the managed `ug configure` twice in the same session. + + Expected: each run, with no personal agent selector, applies the admin config to both enabled + agents identically, so a repeat configure neither duplicates, drops, nor rewrites any entry. + """ + session = live_session + runs = [] + for _ in range(2): + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "Select coding agents to configure:" not in result.stdout, result.stdout + assert "managed config is published" in result.stdout, result.stdout + settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) + catalog = json.loads((session.home / ".ucode" / "codex-model-catalog.json").read_text()) + picker = [o.get("model") for o in (settings.get("modelPicker") or {}).get("options", [])] + listed = [m.get("slug") for m in catalog.get("models", []) if m.get("visibility") == "list"] + runs.append((settings.get("availableModels"), picker, listed)) + + expected = (MANAGED_CLAUDE_MODELS, MANAGED_CLAUDE_MODELS, [MANAGED_CODEX_MODEL]) + assert runs == [expected, expected], runs diff --git a/tests/integration/test_ug_configure_managed_fixtures.py b/tests/integration/test_ug_configure_managed_fixtures.py new file mode 100644 index 00000000..84a1c21b --- /dev/null +++ b/tests/integration/test_ug_configure_managed_fixtures.py @@ -0,0 +1,131 @@ +"""Managed-config CUJs driven by an injected admin config (UCODE_MANAGED_CONFIG_STUB). + +These exercise the real installed `ug configure` against a real workspace, but the admin +CodingAgentConfig is injected locally so we can cover shapes the live workspace does not publish. +Only the config INPUT is stubbed; auth, normalization, and the config writers stay real. See +tests/AGENTS.md rule 4. + +Each case deliberately differs from ca-central's published config in the dimension it asserts +(agent count, model list, or tracing), so a pass proves the injected config drove configure rather +than a live fetch. Model ids are real ca-central model services so configure does not reject them. +""" + +import json + +import pytest + +CLAUDE_MODELS = [ + "system.ai.claude-opus-4-8", + "system.ai.claude-sonnet-4-6", + "system.ai.claude-haiku-4-5", +] +CODEX_MODEL = "system.ai.gpt-5-6-sol" + + +def _claude_agent(models: list[str], *, tracing: bool | None = None) -> dict: + config = { + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, + } + if tracing is not None: + config["tracing"] = {"enabled": tracing} + return {"agent": "CODING_AGENT_CLAUDE_CODE", "config": config} + + +def _codex_agent() -> dict: + return { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": {"model_services": [CODEX_MODEL]}, + "default_models": {"default_model": CODEX_MODEL}, + }, + } + + +def _config(default_agent: str, *agents: dict) -> dict: + return {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} + + +def _apply(session, tmp_path, workspace, config: dict): + stub = tmp_path / "managed-config.json" + stub.write_text(json.dumps(config)) + session.env["UCODE_MANAGED_CONFIG_STUB"] = str(stub) + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "managed config is published" in result.stdout, result.stdout + return result + + +def _workspace_state(session) -> dict: + state = json.loads((session.home / ".ucode" / "state.json").read_text()) + return state["workspaces"][state["current_workspace"]] + + +def _claude_settings(session) -> dict: + return json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_single_agent_claude_is_an_allowlist(live_session, workspace, tmp_path): + """Scenario: an injected config enables only Claude, though both agent CLIs are installed. + + Expected: configure applies Claude alone. enabled_agents is an allowlist, and ["claude"] can + only come from the injected config (the live workspace enables both agents). + """ + session = live_session + _apply( + session, + tmp_path, + workspace, + _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(CLAUDE_MODELS)), + ) + assert _workspace_state(session).get("available_tools") == ["claude"], _workspace_state(session) + + +@pytest.mark.managed_fixture +@pytest.mark.codex +def test_managed_fixture_single_agent_codex_is_an_allowlist(live_session, workspace, tmp_path): + """Scenario: an injected config enables only Codex, though both agent CLIs are installed. + + Expected: configure applies Codex alone (the live workspace enables both agents). + """ + session = live_session + _apply(session, tmp_path, workspace, _config("CODING_AGENT_CODEX", _codex_agent())) + assert _workspace_state(session).get("available_tools") == ["codex"], _workspace_state(session) + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_static_model_list_comes_from_the_config(live_session, workspace, tmp_path): + """Scenario: an injected config pins a Claude model list distinct from the live workspace's. + + Expected: Claude's availableModels and picker equal exactly the injected two-model list, proving + the model allow-list is driven by the config (the live workspace publishes three Claude models). + """ + session = live_session + models = [CLAUDE_MODELS[0], CLAUDE_MODELS[2]] + _apply(session, tmp_path, workspace, _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(models))) + + settings = _claude_settings(session) + assert settings.get("availableModels") == models, settings + options = (settings.get("modelPicker") or {}).get("options", []) + assert [option.get("model") for option in options] == models, settings + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_tracing_enabled_writes_otel(live_session, workspace, tmp_path): + """Scenario: an injected config enables tracing for Claude (the live workspace does not). + + Expected: configure writes the OTLP telemetry env pointing at the workspace gateway endpoint. + """ + session = live_session + config = _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(CLAUDE_MODELS, tracing=True)) + _apply(session, tmp_path, workspace, config) + + env = _claude_settings(session).get("env") or {} + assert env.get("CLAUDE_CODE_ENABLE_TELEMETRY") == "1", env + assert env.get("OTEL_TRACES_EXPORTER") == "otlp", env + assert ( + env.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") == f"{workspace}/ai-gateway/otel/v1/traces" + ), env From 6a74de3e60329bf329cd4b2559efd84f0ad66ace Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:15:57 +0000 Subject: [PATCH 2/9] [AIGTWY-4759] Drive the TUI in the managed fixture tests Replace the config-file assertions with tests that launch the real agent and drive /model and /mcp, so they verify what the agent actually presents, not the generated config (asserting config is unit tests' job). Co-authored-by: Isaac --- .../test_ug_configure_managed_fixtures.py | 144 +++++++----------- 1 file changed, 52 insertions(+), 92 deletions(-) diff --git a/tests/integration/test_ug_configure_managed_fixtures.py b/tests/integration/test_ug_configure_managed_fixtures.py index 84a1c21b..6a569586 100644 --- a/tests/integration/test_ug_configure_managed_fixtures.py +++ b/tests/integration/test_ug_configure_managed_fixtures.py @@ -1,49 +1,38 @@ -"""Managed-config CUJs driven by an injected admin config (UCODE_MANAGED_CONFIG_STUB). +"""Managed-config CUJs that drive the real agent TUI under an injected admin config. -These exercise the real installed `ug configure` against a real workspace, but the admin -CodingAgentConfig is injected locally so we can cover shapes the live workspace does not publish. -Only the config INPUT is stubbed; auth, normalization, and the config writers stay real. See -tests/AGENTS.md rule 4. - -Each case deliberately differs from ca-central's published config in the dimension it asserts -(agent count, model list, or tracing), so a pass proves the injected config drove configure rather -than a live fetch. Model ids are real ca-central model services so configure does not reject them. +The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so we can exercise shapes the +live workspace does not publish; only that INPUT is stubbed (auth, the config writers, and the agent +binaries stay real). These assert what the launched agent actually presents in its TUI, the /model +picker and the /mcp list, not the generated config files (asserting the files is unit tests' job). +See tests/AGENTS.md rule 4. """ import json import pytest +from utils.terminal import AgentTerminal -CLAUDE_MODELS = [ - "system.ai.claude-opus-4-8", - "system.ai.claude-sonnet-4-6", - "system.ai.claude-haiku-4-5", -] -CODEX_MODEL = "system.ai.gpt-5-6-sol" - - -def _claude_agent(models: list[str], *, tracing: bool | None = None) -> dict: - config = { - "models": {"model_services": models}, - "default_models": {"default_model": models[0]}, - } - if tracing is not None: - config["tracing"] = {"enabled": tracing} - return {"agent": "CODING_AGENT_CLAUDE_CODE", "config": config} +CLAUDE_OPUS = "system.ai.claude-opus-4-8" +CLAUDE_SONNET = "system.ai.claude-sonnet-4-6" +CLAUDE_HAIKU = "system.ai.claude-haiku-4-5" +MCP_SERVICE = "system.ai.github" -def _codex_agent() -> dict: +def _claude_agent(models: list[str]) -> dict: return { - "agent": "CODING_AGENT_CODEX", + "agent": "CODING_AGENT_CLAUDE_CODE", "config": { - "models": {"model_services": [CODEX_MODEL]}, - "default_models": {"default_model": CODEX_MODEL}, + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, }, } -def _config(default_agent: str, *agents: dict) -> dict: - return {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} +def _config(default_agent: str, *agents: dict, mcp_names: list[str] | None = None) -> dict: + cfg = {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} + if mcp_names is not None: + cfg["mcp_servers"] = {"names": mcp_names} + return cfg def _apply(session, tmp_path, workspace, config: dict): @@ -52,80 +41,51 @@ def _apply(session, tmp_path, workspace, config: dict): session.env["UCODE_MANAGED_CONFIG_STUB"] = str(stub) result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) assert "managed config is published" in result.stdout, result.stdout - return result - - -def _workspace_state(session) -> dict: - state = json.loads((session.home / ".ucode" / "state.json").read_text()) - return state["workspaces"][state["current_workspace"]] - - -def _claude_settings(session) -> dict: - return json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) @pytest.mark.managed_fixture @pytest.mark.claude -def test_managed_fixture_single_agent_claude_is_an_allowlist(live_session, workspace, tmp_path): - """Scenario: an injected config enables only Claude, though both agent CLIs are installed. +def test_managed_fixture_claude_model_picker_lists_managed_models( + live_session, workspace, tmp_path +): + """Scenario: launch Claude under a managed config and open the /model picker. - Expected: configure applies Claude alone. enabled_agents is an allowlist, and ["claude"] can - only come from the injected config (the live workspace enables both agents). + Expected: the picker offers the admin's managed models and not a model outside that list. """ session = live_session - _apply( - session, - tmp_path, - workspace, - _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(CLAUDE_MODELS)), - ) - assert _workspace_state(session).get("available_tools") == ["claude"], _workspace_state(session) - - -@pytest.mark.managed_fixture -@pytest.mark.codex -def test_managed_fixture_single_agent_codex_is_an_allowlist(live_session, workspace, tmp_path): - """Scenario: an injected config enables only Codex, though both agent CLIs are installed. - - Expected: configure applies Codex alone (the live workspace enables both agents). - """ - session = live_session - _apply(session, tmp_path, workspace, _config("CODING_AGENT_CODEX", _codex_agent())) - assert _workspace_state(session).get("available_tools") == ["codex"], _workspace_state(session) - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_managed_fixture_static_model_list_comes_from_the_config(live_session, workspace, tmp_path): - """Scenario: an injected config pins a Claude model list distinct from the live workspace's. - - Expected: Claude's availableModels and picker equal exactly the injected two-model list, proving - the model allow-list is driven by the config (the live workspace publishes three Claude models). - """ - session = live_session - models = [CLAUDE_MODELS[0], CLAUDE_MODELS[2]] + models = [CLAUDE_OPUS, CLAUDE_HAIKU] # distinct from the live config's three models _apply(session, tmp_path, workspace, _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(models))) - - settings = _claude_settings(session) - assert settings.get("availableModels") == models, settings - options = (settings.get("modelPicker") or {}).get("options", []) - assert [option.get("model") for option in options] == models, settings + with AgentTerminal( + session, "claude", [str(session.binary), "claude"], "managed-fixture-model" + ) as tui: + tui.boot() + tui.send("/model", "type the /model command") + tui.send("\r", "open the model picker") + tui.wait_for( + lambda s: "opus-4-8" in s and "haiku-4-5" in s, + "the /model picker to list the managed models", + ) + assert "sonnet-4-6" not in tui.visible, tui.visible @pytest.mark.managed_fixture @pytest.mark.claude -def test_managed_fixture_tracing_enabled_writes_otel(live_session, workspace, tmp_path): - """Scenario: an injected config enables tracing for Claude (the live workspace does not). +def test_managed_fixture_claude_mcp_lists_configured_server(live_session, workspace, tmp_path): + """Scenario: launch Claude under a managed config with an MCP server and open /mcp. - Expected: configure writes the OTLP telemetry env pointing at the workspace gateway endpoint. + Expected: the managed MCP server is listed in the agent's /mcp view. """ session = live_session - config = _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(CLAUDE_MODELS, tracing=True)) + config = _config( + "CODING_AGENT_CLAUDE_CODE", _claude_agent([CLAUDE_OPUS]), mcp_names=[MCP_SERVICE] + ) _apply(session, tmp_path, workspace, config) - - env = _claude_settings(session).get("env") or {} - assert env.get("CLAUDE_CODE_ENABLE_TELEMETRY") == "1", env - assert env.get("OTEL_TRACES_EXPORTER") == "otlp", env - assert ( - env.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") == f"{workspace}/ai-gateway/otel/v1/traces" - ), env + with AgentTerminal( + session, "claude", [str(session.binary), "claude"], "managed-fixture-mcp" + ) as tui: + tui.boot() + tui.send("/mcp", "type the /mcp command") + tui.send("\r", "open the MCP list") + tui.wait_for( + lambda s: "github" in s.lower(), "the /mcp view to list the managed MCP server" + ) From a5784cbf2f937d80bf4034011aa5aee2a397fa77 Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:34:43 +0000 Subject: [PATCH 3/9] [AIGTWY-4759] Add shared managed-config test helpers (utils/managed.py) Co-authored-by: Isaac --- tests/integration/utils/managed.py | 46 ++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/integration/utils/managed.py diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py new file mode 100644 index 00000000..af5e7885 --- /dev/null +++ b/tests/integration/utils/managed.py @@ -0,0 +1,46 @@ +"""Shared builders and stub injection for the managed-config integration suites. + +Pure CodingAgentConfig construction plus the ``UCODE_MANAGED_CONFIG_STUB`` filesystem/env mechanics. +The configure invocation, launch, and assertions stay visible in each test (tests/AGENTS.md), and +this module imports nothing from the ``ucode`` application package (tests/test_integration_contract.py +enforces that boundary). +""" + +import json +from pathlib import Path + + +def set_managed_config_stub(session, tmp_path, config: dict) -> None: + """Write ``config`` to a file and point ``UCODE_MANAGED_CONFIG_STUB`` at it for this session.""" + stub = Path(tmp_path) / "managed-config.json" + stub.write_text(json.dumps(config)) + session.env["UCODE_MANAGED_CONFIG_STUB"] = str(stub) + + +def build_coding_agent_config( + default_agent: str, *agents: dict, mcp_names: list[str] | None = None +) -> dict: + config = {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} + if mcp_names is not None: + config["mcp_servers"] = {"names": mcp_names} + return config + + +def build_claude_agent_config(models: list[str]) -> dict: + return { + "agent": "CODING_AGENT_CLAUDE_CODE", + "config": { + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, + }, + } + + +def build_codex_agent_config(models: list[str]) -> dict: + return { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, + }, + } From 465586a4c24b7a2c06b7766e48572e892650d9f0 Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:34:44 +0000 Subject: [PATCH 4/9] [AIGTWY-4759] Managed /model TUI CUJ suite Co-authored-by: Isaac --- .../test_ug_configure_managed_models.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/integration/test_ug_configure_managed_models.py diff --git a/tests/integration/test_ug_configure_managed_models.py b/tests/integration/test_ug_configure_managed_models.py new file mode 100644 index 00000000..48f5cabd --- /dev/null +++ b/tests/integration/test_ug_configure_managed_models.py @@ -0,0 +1,47 @@ +"""Managed-config CUJ: the launched agent's /model picker reflects the admin's model list. + +The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so the real /model TUI can be +driven against a model list the live workspace does not publish; only the config INPUT is stubbed +(auth, the config writers, and the agent binary stay real). These assert what the agent presents, +not the generated config files (that is unit tests' job). See tests/AGENTS.md rule 4. +""" + +import pytest +from utils.managed import ( + build_claude_agent_config, + build_coding_agent_config, + set_managed_config_stub, +) +from utils.terminal import AgentTerminal + +CLAUDE_OPUS = "system.ai.claude-opus-4-8" +# A real ca-central model absent from the live published config: its presence in the picker can +# only come from the injected config, which the live workspace's model list cannot produce. +CLAUDE_OFF_MENU = "system.ai.claude-sonnet-5" +LIVE_ONLY = "haiku-4-5" # published live, but not in the injected list below + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_claude_model_picker_reflects_the_config(live_session, workspace, tmp_path): + """Scenario: launch Claude under an injected managed config and open the /model picker. + + Expected: the picker offers the injected models (including one the live workspace does not + publish) and omits a model the live workspace does publish. + """ + session = live_session + config = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", build_claude_agent_config([CLAUDE_OPUS, CLAUDE_OFF_MENU]) + ) + set_managed_config_stub(session, tmp_path, config) + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "managed config is published" in result.stdout, result.stdout + + with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-model") as tui: + tui.boot() + tui.send("/model", "type the /model command") + tui.send("\r", "open the model picker") + tui.wait_for( + lambda s: "sonnet-5" in s, "the /model picker to list the injected model", timeout=60 + ) + assert LIVE_ONLY not in tui.visible, tui.visible From 390e0f478b60a8ae5f1fc9cae19b668e8580b100 Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:34:46 +0000 Subject: [PATCH 5/9] [AIGTWY-4759] Managed /mcp TUI CUJ suite Co-authored-by: Isaac --- .../test_ug_configure_managed_mcp.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/integration/test_ug_configure_managed_mcp.py diff --git a/tests/integration/test_ug_configure_managed_mcp.py b/tests/integration/test_ug_configure_managed_mcp.py new file mode 100644 index 00000000..81874bed --- /dev/null +++ b/tests/integration/test_ug_configure_managed_mcp.py @@ -0,0 +1,48 @@ +"""Managed-config CUJ: the launched agent's /mcp view lists the admin's MCP servers. + +The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so the real /mcp TUI can be +driven against an MCP list the live workspace does not publish; only the config INPUT is stubbed +(auth, the config writers, and the agent binary stay real). These assert what the agent presents, +not the generated config files (that is unit tests' job). See tests/AGENTS.md rule 4. +""" + +import pytest +from utils.managed import ( + build_claude_agent_config, + build_coding_agent_config, + set_managed_config_stub, +) +from utils.terminal import AgentTerminal + +CLAUDE_OPUS = "system.ai.claude-opus-4-8" +# A real ca-central MCP service; the live published config lists no MCP servers, so its appearance +# in the /mcp view can only come from the injected config. +MCP_SERVICE = "system.ai.github" + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_claude_mcp_lists_configured_server(live_session, workspace, tmp_path): + """Scenario: launch Claude under an injected config with a managed MCP server and open /mcp. + + Expected: the injected MCP server appears in the agent's /mcp view. + """ + session = live_session + config = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", + build_claude_agent_config([CLAUDE_OPUS]), + mcp_names=[MCP_SERVICE], + ) + set_managed_config_stub(session, tmp_path, config) + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "managed config is published" in result.stdout, result.stdout + + with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-mcp") as tui: + tui.boot() + tui.send("/mcp", "type the /mcp command") + tui.send("\r", "open the MCP list") + tui.wait_for( + lambda s: "github" in s.lower(), + "the /mcp view to list the managed MCP server", + timeout=60, + ) From a1af77f520b1f31ca1a30b2c945cf7158859cf74 Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:34:48 +0000 Subject: [PATCH 6/9] [AIGTWY-4759] Split managed fixtures into per-CUJ suites Co-authored-by: Isaac --- .../test_ug_configure_managed_fixtures.py | 91 ------------------- 1 file changed, 91 deletions(-) delete mode 100644 tests/integration/test_ug_configure_managed_fixtures.py diff --git a/tests/integration/test_ug_configure_managed_fixtures.py b/tests/integration/test_ug_configure_managed_fixtures.py deleted file mode 100644 index 6a569586..00000000 --- a/tests/integration/test_ug_configure_managed_fixtures.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Managed-config CUJs that drive the real agent TUI under an injected admin config. - -The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so we can exercise shapes the -live workspace does not publish; only that INPUT is stubbed (auth, the config writers, and the agent -binaries stay real). These assert what the launched agent actually presents in its TUI, the /model -picker and the /mcp list, not the generated config files (asserting the files is unit tests' job). -See tests/AGENTS.md rule 4. -""" - -import json - -import pytest -from utils.terminal import AgentTerminal - -CLAUDE_OPUS = "system.ai.claude-opus-4-8" -CLAUDE_SONNET = "system.ai.claude-sonnet-4-6" -CLAUDE_HAIKU = "system.ai.claude-haiku-4-5" -MCP_SERVICE = "system.ai.github" - - -def _claude_agent(models: list[str]) -> dict: - return { - "agent": "CODING_AGENT_CLAUDE_CODE", - "config": { - "models": {"model_services": models}, - "default_models": {"default_model": models[0]}, - }, - } - - -def _config(default_agent: str, *agents: dict, mcp_names: list[str] | None = None) -> dict: - cfg = {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} - if mcp_names is not None: - cfg["mcp_servers"] = {"names": mcp_names} - return cfg - - -def _apply(session, tmp_path, workspace, config: dict): - stub = tmp_path / "managed-config.json" - stub.write_text(json.dumps(config)) - session.env["UCODE_MANAGED_CONFIG_STUB"] = str(stub) - result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) - assert "managed config is published" in result.stdout, result.stdout - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_managed_fixture_claude_model_picker_lists_managed_models( - live_session, workspace, tmp_path -): - """Scenario: launch Claude under a managed config and open the /model picker. - - Expected: the picker offers the admin's managed models and not a model outside that list. - """ - session = live_session - models = [CLAUDE_OPUS, CLAUDE_HAIKU] # distinct from the live config's three models - _apply(session, tmp_path, workspace, _config("CODING_AGENT_CLAUDE_CODE", _claude_agent(models))) - with AgentTerminal( - session, "claude", [str(session.binary), "claude"], "managed-fixture-model" - ) as tui: - tui.boot() - tui.send("/model", "type the /model command") - tui.send("\r", "open the model picker") - tui.wait_for( - lambda s: "opus-4-8" in s and "haiku-4-5" in s, - "the /model picker to list the managed models", - ) - assert "sonnet-4-6" not in tui.visible, tui.visible - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_managed_fixture_claude_mcp_lists_configured_server(live_session, workspace, tmp_path): - """Scenario: launch Claude under a managed config with an MCP server and open /mcp. - - Expected: the managed MCP server is listed in the agent's /mcp view. - """ - session = live_session - config = _config( - "CODING_AGENT_CLAUDE_CODE", _claude_agent([CLAUDE_OPUS]), mcp_names=[MCP_SERVICE] - ) - _apply(session, tmp_path, workspace, config) - with AgentTerminal( - session, "claude", [str(session.binary), "claude"], "managed-fixture-mcp" - ) as tui: - tui.boot() - tui.send("/mcp", "type the /mcp command") - tui.send("\r", "open the MCP list") - tui.wait_for( - lambda s: "github" in s.lower(), "the /mcp view to list the managed MCP server" - ) From 962ed3c1ffa511e08679c9af71b8a2e1c6dd729f Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:09:28 +0000 Subject: [PATCH 7/9] [AIGTWY-4759] TEMP diagnostics to confirm suspected managed bugs (will be removed) --- .../test_zz_managed_diagnostics.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tests/integration/test_zz_managed_diagnostics.py diff --git a/tests/integration/test_zz_managed_diagnostics.py b/tests/integration/test_zz_managed_diagnostics.py new file mode 100644 index 00000000..dea00fa0 --- /dev/null +++ b/tests/integration/test_zz_managed_diagnostics.py @@ -0,0 +1,93 @@ +"""THROWAWAY diagnostics: confirm or refute the suspected managed-config bugs against real ug. + +Each probe asserts the CORRECT behavior; a failure in CI confirms the bug (with the real value in +the message). This file is not shipped, it exists only to make the bug list definite. It asserts +config artifacts on purpose (that is how we witness the apply-layer behavior for confirmation). +""" + +import json + +import pytest +from utils.managed import ( + build_claude_agent_config, + build_coding_agent_config, + set_managed_config_stub, +) + +OPUS = "system.ai.claude-opus-4-8" +SONNET = "system.ai.claude-sonnet-4-6" + + +def _configure(session, workspace, ok=True): + return session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240, ok=ok) + + +def _state_tools(session): + state = json.loads((session.home / ".ucode" / "state.json").read_text()) + return state["workspaces"][state["current_workspace"]].get("available_tools") + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_probe_family_default_is_honored(live_session, workspace, tmp_path): + """CORRECT: an admin default_sonnet_model in the allow-list becomes the sonnet default.""" + session = live_session + config = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", + { + "agent": "CODING_AGENT_CLAUDE_CODE", + "config": { + "models": {"model_services": [OPUS, SONNET]}, + "default_models": {"default_model": OPUS, "default_sonnet_model": SONNET}, + }, + }, + ) + set_managed_config_stub(session, tmp_path, config) + result = _configure(session, workspace) + assert "managed config is published" in result.stdout, result.stdout + settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) + # Consumption proof: the injected two-model list drove availableModels (live has three). + assert settings.get("availableModels") == [OPUS, SONNET], settings + env = settings.get("env") or {} + written = env.get("ANTHROPIC_DEFAULT_SONNET_MODEL", "") + base = written[: -len("[1m]")] if written.endswith("[1m]") else written + assert base == SONNET, f"sonnet default was {written!r}, expected {SONNET}" + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_probe_invalid_default_agent_is_rejected(live_session, workspace, tmp_path): + """CORRECT: configure rejects a config whose default_agent is not in enabled_agents.""" + session = live_session + config = build_coding_agent_config("CODING_AGENT_CODEX", build_claude_agent_config([OPUS])) + set_managed_config_stub(session, tmp_path, config) + result = _configure(session, workspace, ok=False) + assert result.returncode != 0, f"configure accepted an invalid config:\n{result.stdout}" + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_probe_deenabling_an_agent_removes_it(live_session, workspace, tmp_path): + """CORRECT: re-configuring with a claude-only config removes a previously-configured codex.""" + session = live_session + both = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", + build_claude_agent_config([OPUS]), + { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": {"model_services": ["system.ai.gpt-5-6-sol"]}, + "default_models": {"default_model": "system.ai.gpt-5-6-sol"}, + }, + }, + ) + set_managed_config_stub(session, tmp_path, both) + _configure(session, workspace) + assert set(_state_tools(session)) == {"claude", "codex"}, _state_tools(session) + + claude_only = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", build_claude_agent_config([OPUS]) + ) + set_managed_config_stub(session, tmp_path, claude_only) + _configure(session, workspace) + assert _state_tools(session) == ["claude"], _state_tools(session) From 71992997482132dbe87002628676ede8ec57971f Mon Sep 17 00:00:00 2001 From: David Siqi Liu <45125958+david-siqi-liu@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:22:18 +0000 Subject: [PATCH 8/9] [AIGTWY-4759] Remove throwaway managed diagnostics (bugs confirmed, filed separately) --- .../test_zz_managed_diagnostics.py | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 tests/integration/test_zz_managed_diagnostics.py diff --git a/tests/integration/test_zz_managed_diagnostics.py b/tests/integration/test_zz_managed_diagnostics.py deleted file mode 100644 index dea00fa0..00000000 --- a/tests/integration/test_zz_managed_diagnostics.py +++ /dev/null @@ -1,93 +0,0 @@ -"""THROWAWAY diagnostics: confirm or refute the suspected managed-config bugs against real ug. - -Each probe asserts the CORRECT behavior; a failure in CI confirms the bug (with the real value in -the message). This file is not shipped, it exists only to make the bug list definite. It asserts -config artifacts on purpose (that is how we witness the apply-layer behavior for confirmation). -""" - -import json - -import pytest -from utils.managed import ( - build_claude_agent_config, - build_coding_agent_config, - set_managed_config_stub, -) - -OPUS = "system.ai.claude-opus-4-8" -SONNET = "system.ai.claude-sonnet-4-6" - - -def _configure(session, workspace, ok=True): - return session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240, ok=ok) - - -def _state_tools(session): - state = json.loads((session.home / ".ucode" / "state.json").read_text()) - return state["workspaces"][state["current_workspace"]].get("available_tools") - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_probe_family_default_is_honored(live_session, workspace, tmp_path): - """CORRECT: an admin default_sonnet_model in the allow-list becomes the sonnet default.""" - session = live_session - config = build_coding_agent_config( - "CODING_AGENT_CLAUDE_CODE", - { - "agent": "CODING_AGENT_CLAUDE_CODE", - "config": { - "models": {"model_services": [OPUS, SONNET]}, - "default_models": {"default_model": OPUS, "default_sonnet_model": SONNET}, - }, - }, - ) - set_managed_config_stub(session, tmp_path, config) - result = _configure(session, workspace) - assert "managed config is published" in result.stdout, result.stdout - settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) - # Consumption proof: the injected two-model list drove availableModels (live has three). - assert settings.get("availableModels") == [OPUS, SONNET], settings - env = settings.get("env") or {} - written = env.get("ANTHROPIC_DEFAULT_SONNET_MODEL", "") - base = written[: -len("[1m]")] if written.endswith("[1m]") else written - assert base == SONNET, f"sonnet default was {written!r}, expected {SONNET}" - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_probe_invalid_default_agent_is_rejected(live_session, workspace, tmp_path): - """CORRECT: configure rejects a config whose default_agent is not in enabled_agents.""" - session = live_session - config = build_coding_agent_config("CODING_AGENT_CODEX", build_claude_agent_config([OPUS])) - set_managed_config_stub(session, tmp_path, config) - result = _configure(session, workspace, ok=False) - assert result.returncode != 0, f"configure accepted an invalid config:\n{result.stdout}" - - -@pytest.mark.managed_fixture -@pytest.mark.claude -def test_probe_deenabling_an_agent_removes_it(live_session, workspace, tmp_path): - """CORRECT: re-configuring with a claude-only config removes a previously-configured codex.""" - session = live_session - both = build_coding_agent_config( - "CODING_AGENT_CLAUDE_CODE", - build_claude_agent_config([OPUS]), - { - "agent": "CODING_AGENT_CODEX", - "config": { - "models": {"model_services": ["system.ai.gpt-5-6-sol"]}, - "default_models": {"default_model": "system.ai.gpt-5-6-sol"}, - }, - }, - ) - set_managed_config_stub(session, tmp_path, both) - _configure(session, workspace) - assert set(_state_tools(session)) == {"claude", "codex"}, _state_tools(session) - - claude_only = build_coding_agent_config( - "CODING_AGENT_CLAUDE_CODE", build_claude_agent_config([OPUS]) - ) - set_managed_config_stub(session, tmp_path, claude_only) - _configure(session, workspace) - assert _state_tools(session) == ["claude"], _state_tools(session) From b55ad811634cb1d95f0716322501e592077a1b26 Mon Sep 17 00:00:00 2001 From: Lilly Luo Date: Thu, 17 Sep 2026 14:05:13 +0000 Subject: [PATCH 9/9] Update managed integration output assertions --- tests/integration/test_ug_configure_managed.py | 1 - tests/integration/test_ug_configure_managed_mcp.py | 2 +- tests/integration/test_ug_configure_managed_models.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_ug_configure_managed.py b/tests/integration/test_ug_configure_managed.py index 34b49f68..3faea234 100644 --- a/tests/integration/test_ug_configure_managed.py +++ b/tests/integration/test_ug_configure_managed.py @@ -87,7 +87,6 @@ def test_ug_configure_managed_is_idempotent(live_session, workspace): for _ in range(2): result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) assert "Select coding agents to configure:" not in result.stdout, result.stdout - assert "managed config is published" in result.stdout, result.stdout settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) catalog = json.loads((session.home / ".ucode" / "codex-model-catalog.json").read_text()) picker = [o.get("model") for o in (settings.get("modelPicker") or {}).get("options", [])] diff --git a/tests/integration/test_ug_configure_managed_mcp.py b/tests/integration/test_ug_configure_managed_mcp.py index 81874bed..4f586a9a 100644 --- a/tests/integration/test_ug_configure_managed_mcp.py +++ b/tests/integration/test_ug_configure_managed_mcp.py @@ -35,7 +35,7 @@ def test_managed_fixture_claude_mcp_lists_configured_server(live_session, worksp ) set_managed_config_stub(session, tmp_path, config) result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) - assert "managed config is published" in result.stdout, result.stdout + assert "Select coding agents to configure:" not in result.stdout, result.stdout with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-mcp") as tui: tui.boot() diff --git a/tests/integration/test_ug_configure_managed_models.py b/tests/integration/test_ug_configure_managed_models.py index 48f5cabd..2bec0186 100644 --- a/tests/integration/test_ug_configure_managed_models.py +++ b/tests/integration/test_ug_configure_managed_models.py @@ -35,7 +35,7 @@ def test_managed_fixture_claude_model_picker_reflects_the_config(live_session, w ) set_managed_config_stub(session, tmp_path, config) result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) - assert "managed config is published" in result.stdout, result.stdout + assert "Select coding agents to configure:" not in result.stdout, result.stdout with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-model") as tui: tui.boot()