diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d9f87f6f..2facc9da 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -192,9 +192,9 @@ jobs: name: Managed config ยท ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 - # Temporarily non-blocking: the managed e2e workspace is not yet reachable from CI runners - # (token mint returns 403). Keep the job running for signal, but do not fail the workflow or - # gate merges on it until that workspace access is sorted. + # Kept non-blocking for now: the managed workspace (E2E_ADMIN_WORKSPACE) is repointed to + # runner-reachable ca-central, but leave these lanes for signal until the managed-config apply + # path is proven stable. Then drop this and add `managed` to the required set. continue-on-error: true timeout-minutes: 30 strategy: @@ -226,17 +226,16 @@ jobs: DATABRICKS_CLIENT_ID: ${{ secrets.E2E_ADMIN_SP_CLIENT_ID }} DATABRICKS_CLIENT_SECRET: ${{ secrets.E2E_ADMIN_SP_CLIENT_SECRET }} run: | - case "$AGENT" in - claude) args=(--claude-version "$CLAUDE_VERSION") ;; - codex) args=(--codex-version "$CODEX_VERSION") ;; - esac + # A managed config enables both agents and `ug configure` applies it to every enabled + # agent, so both CLIs must be installed even though this lane asserts only one agent. + args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION") if [[ -n "$DEPENDENCY" ]]; then args+=(--dependency "$DEPENDENCY") fi uv run --no-project --python 3.12 python scripts/run_integration.py \ --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ - "${args[@]}" -- -m "managed and $AGENT" + "${args[@]}" -- -m "(managed or managed_fixture) and $AGENT" - name: Upload managed test evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/tests/AGENTS.md b/tests/AGENTS.md index dd9d9ab4..6cc382d3 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -26,7 +26,13 @@ tests. Keep work scoped to the behavior requested by the user. 4. **No production changes just to make tests pass.** No test-only environment switches, special server branches, disabled validation, privileged-path overrides, or hardcoded success. Real bugs require normal production fixes - and regression coverage. Report failures instead of concealing them. + and regression coverage. Report failures instead of concealing them. The sole + exception is the `managed_fixture` marker: it uses the built-in + `UCODE_MANAGED_CONFIG_STUB` hook to inject the admin CodingAgentConfig INPUT so the + real `ug configure` path can be exercised across config shapes the live workspace does + not publish. The gateway, agent binaries, ug internals, and ug state stay real; the + config fetch/wire contract stays covered by the un-stubbed `managed` tests; and the + hook must never be used to disable validation or conceal a failure. 5. **Real responses and binaries.** Pin requested ug and agent versions. Never substitute a missing binary/service. Reuse explicit e2e workspace/auth settings; never pick a developer's Databricks profile automatically. diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index de5260a4..5714c263 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -10,3 +10,10 @@ calls, fake binaries/services, fabricated ug state, or test-only production behavior. Do not hide failures with skips, xfails, retries or weaker assertions. Update the matrix when coverage changes and distinguish automated coverage from gaps and checks that were not executed. + +The one carve-out is the `managed_fixture` marker: those tests inject the admin +CodingAgentConfig through the built-in `UCODE_MANAGED_CONFIG_STUB` hook so the real +`ug configure` path can be exercised across managed-config shapes the live workspace does +not publish. That controls only the admin-authored INPUT; the gateway, agent binaries, ug +internals, and ug state stay real, and the config fetch/wire contract itself stays covered +by the un-stubbed `managed` tests. Faking any of those remains banned. diff --git a/tests/integration/README.md b/tests/integration/README.md index ecce08dd..43f5b4da 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -151,9 +151,12 @@ No service is created or modified. A missing service, permission, or OAuth token fails the selected CUJ, rather than skipping it. There are **40 live cases** (including 4 TUI journeys) and **5 installation -checks** with both agents. A separate **2 managed-workspace cases** (one per agent, -marker `managed`) run against a workspace that publishes a CodingAgentConfig; see -"Managed-workspace journeys" below. See the named coverage and gaps matrix in +checks** with both agents. A separate **3 managed-workspace cases** (one per agent +plus an idempotent re-configure, marker `managed`) run against a workspace that publishes a CodingAgentConfig; see +"Managed-workspace journeys" below. A further **4 `managed_fixture` cases** inject the admin config +locally (via `UCODE_MANAGED_CONFIG_STUB`) to cover shapes the live workspace does not publish; each +differs from the published config in what it asserts so it proves the injected config drove configure. +See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -251,8 +254,9 @@ compensate for capacity failures. Both matrices use `fail-fast: false` and uploa uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, and both full lanes to pass. The **Managed config** lanes run for signal but are temporarily -non-blocking (`continue-on-error`), because the managed e2e workspace is not yet reachable from -CI runners; they neither fail the workflow nor gate merges until that access is sorted. The +non-blocking (`continue-on-error`): the managed workspace is now runner-reachable, but the lanes +stay non-blocking until the managed-config apply path is proven stable. They neither fail the +workflow nor gate merges until then. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -266,6 +270,11 @@ each agent's generated config exposes exactly the admin's static `model_services (Claude's `availableModels`/`modelPicker`, Codex's model catalog). The expected model ids live in the test and mirror the published config; update them there if the admin list changes. +Treat that published CodingAgentConfig as shared CI fixture state. The managed lanes assert its +exact model ids and its both-agent enablement, so editing the managed workspace's config (models, +enabled agents, or defaults) breaks these lanes until the constants in `test_ug_configure_managed.py` +are updated to match. Do not change it casually. + That workspace authenticates as a service principal, so CI mints a short-lived token per run from these same-repository secrets rather than storing a long-lived bearer: diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 4b10e140..7ba34491 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -5,6 +5,7 @@ markers = installation: installed-package checks that require no workspace live: requires the real workspace used by the existing e2e suite managed: requires the managed e2e workspace that publishes a CodingAgentConfig + managed_fixture: real ug/TUI against a real workspace, but the managed CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB smoke: Databricks Hosted configure/TUI and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen claude: only runs when Claude Code is explicitly selected diff --git a/tests/integration/test_ug_configure_managed.py b/tests/integration/test_ug_configure_managed.py index d2fdf803..3faea234 100644 --- a/tests/integration/test_ug_configure_managed.py +++ b/tests/integration/test_ug_configure_managed.py @@ -72,3 +72,26 @@ def test_ug_configure_managed_codex(live_session, workspace): with AgentTerminal(session, "codex", [str(session.binary), "codex"], "managed-codex") as tui: tui.boot() tui.check_input_and_exit() + + +@pytest.mark.managed +@pytest.mark.claude +def test_ug_configure_managed_is_idempotent(live_session, workspace): + """Scenario: run the managed `ug configure` twice in the same session. + + Expected: each run, with no personal agent selector, applies the admin config to both enabled + agents identically, so a repeat configure neither duplicates, drops, nor rewrites any entry. + """ + session = live_session + runs = [] + for _ in range(2): + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "Select coding agents to configure:" not in result.stdout, result.stdout + settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) + catalog = json.loads((session.home / ".ucode" / "codex-model-catalog.json").read_text()) + picker = [o.get("model") for o in (settings.get("modelPicker") or {}).get("options", [])] + listed = [m.get("slug") for m in catalog.get("models", []) if m.get("visibility") == "list"] + runs.append((settings.get("availableModels"), picker, listed)) + + expected = (MANAGED_CLAUDE_MODELS, MANAGED_CLAUDE_MODELS, [MANAGED_CODEX_MODEL]) + assert runs == [expected, expected], runs diff --git a/tests/integration/test_ug_configure_managed_mcp.py b/tests/integration/test_ug_configure_managed_mcp.py new file mode 100644 index 00000000..4f586a9a --- /dev/null +++ b/tests/integration/test_ug_configure_managed_mcp.py @@ -0,0 +1,48 @@ +"""Managed-config CUJ: the launched agent's /mcp view lists the admin's MCP servers. + +The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so the real /mcp TUI can be +driven against an MCP list the live workspace does not publish; only the config INPUT is stubbed +(auth, the config writers, and the agent binary stay real). These assert what the agent presents, +not the generated config files (that is unit tests' job). See tests/AGENTS.md rule 4. +""" + +import pytest +from utils.managed import ( + build_claude_agent_config, + build_coding_agent_config, + set_managed_config_stub, +) +from utils.terminal import AgentTerminal + +CLAUDE_OPUS = "system.ai.claude-opus-4-8" +# A real ca-central MCP service; the live published config lists no MCP servers, so its appearance +# in the /mcp view can only come from the injected config. +MCP_SERVICE = "system.ai.github" + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_claude_mcp_lists_configured_server(live_session, workspace, tmp_path): + """Scenario: launch Claude under an injected config with a managed MCP server and open /mcp. + + Expected: the injected MCP server appears in the agent's /mcp view. + """ + session = live_session + config = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", + build_claude_agent_config([CLAUDE_OPUS]), + mcp_names=[MCP_SERVICE], + ) + set_managed_config_stub(session, tmp_path, config) + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "Select coding agents to configure:" not in result.stdout, result.stdout + + with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-mcp") as tui: + tui.boot() + tui.send("/mcp", "type the /mcp command") + tui.send("\r", "open the MCP list") + tui.wait_for( + lambda s: "github" in s.lower(), + "the /mcp view to list the managed MCP server", + timeout=60, + ) diff --git a/tests/integration/test_ug_configure_managed_models.py b/tests/integration/test_ug_configure_managed_models.py new file mode 100644 index 00000000..2bec0186 --- /dev/null +++ b/tests/integration/test_ug_configure_managed_models.py @@ -0,0 +1,47 @@ +"""Managed-config CUJ: the launched agent's /model picker reflects the admin's model list. + +The admin CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB so the real /model TUI can be +driven against a model list the live workspace does not publish; only the config INPUT is stubbed +(auth, the config writers, and the agent binary stay real). These assert what the agent presents, +not the generated config files (that is unit tests' job). See tests/AGENTS.md rule 4. +""" + +import pytest +from utils.managed import ( + build_claude_agent_config, + build_coding_agent_config, + set_managed_config_stub, +) +from utils.terminal import AgentTerminal + +CLAUDE_OPUS = "system.ai.claude-opus-4-8" +# A real ca-central model absent from the live published config: its presence in the picker can +# only come from the injected config, which the live workspace's model list cannot produce. +CLAUDE_OFF_MENU = "system.ai.claude-sonnet-5" +LIVE_ONLY = "haiku-4-5" # published live, but not in the injected list below + + +@pytest.mark.managed_fixture +@pytest.mark.claude +def test_managed_fixture_claude_model_picker_reflects_the_config(live_session, workspace, tmp_path): + """Scenario: launch Claude under an injected managed config and open the /model picker. + + Expected: the picker offers the injected models (including one the live workspace does not + publish) and omits a model the live workspace does publish. + """ + session = live_session + config = build_coding_agent_config( + "CODING_AGENT_CLAUDE_CODE", build_claude_agent_config([CLAUDE_OPUS, CLAUDE_OFF_MENU]) + ) + set_managed_config_stub(session, tmp_path, config) + result = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=240) + assert "Select coding agents to configure:" not in result.stdout, result.stdout + + with AgentTerminal(session, "claude", [str(session.binary), "claude"], "managed-model") as tui: + tui.boot() + tui.send("/model", "type the /model command") + tui.send("\r", "open the model picker") + tui.wait_for( + lambda s: "sonnet-5" in s, "the /model picker to list the injected model", timeout=60 + ) + assert LIVE_ONLY not in tui.visible, tui.visible diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py new file mode 100644 index 00000000..af5e7885 --- /dev/null +++ b/tests/integration/utils/managed.py @@ -0,0 +1,46 @@ +"""Shared builders and stub injection for the managed-config integration suites. + +Pure CodingAgentConfig construction plus the ``UCODE_MANAGED_CONFIG_STUB`` filesystem/env mechanics. +The configure invocation, launch, and assertions stay visible in each test (tests/AGENTS.md), and +this module imports nothing from the ``ucode`` application package (tests/test_integration_contract.py +enforces that boundary). +""" + +import json +from pathlib import Path + + +def set_managed_config_stub(session, tmp_path, config: dict) -> None: + """Write ``config`` to a file and point ``UCODE_MANAGED_CONFIG_STUB`` at it for this session.""" + stub = Path(tmp_path) / "managed-config.json" + stub.write_text(json.dumps(config)) + session.env["UCODE_MANAGED_CONFIG_STUB"] = str(stub) + + +def build_coding_agent_config( + default_agent: str, *agents: dict, mcp_names: list[str] | None = None +) -> dict: + config = {"spec_version": 1, "default_agent": default_agent, "enabled_agents": list(agents)} + if mcp_names is not None: + config["mcp_servers"] = {"names": mcp_names} + return config + + +def build_claude_agent_config(models: list[str]) -> dict: + return { + "agent": "CODING_AGENT_CLAUDE_CODE", + "config": { + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, + }, + } + + +def build_codex_agent_config(models: list[str]) -> dict: + return { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": {"model_services": models}, + "default_models": {"default_model": models[0]}, + }, + }