diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index b03e3847b98..e0d8712aed4 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -38,6 +38,46 @@ replace github.com/azure/azure-dev/cli/azd => ../../ That `replace` points this extension at your local `cli/azd` checkout instead of the version in `go.mod`. Do not merge the extension with that `replace` still present. +## Interactive CLI test scenarios + +This extension ships a suite of goal-based scenarios for the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server under `tests/cli-interactive-tester-scenarios/`. They drive real +`azd ai agent` flows end-to-end (init, provision, deploy, invoke, run, sessions, +files, monitor, endpoint, doctor, down) and are organized by tier: + +- **Tier 0** — offline, no Azure auth, no cost (help, version, validation, picker UX) +- **Tier 1** — local-only with Azure auth (init flows) +- **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) + +Each scenario carries a set of tags based on what is being tested and how. +See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, +profile setup, and orchestration rules. + +### Guidance for coding agents + +These scenarios are **never run automatically** — they require the +cli-interactive-tester MCP server, a populated `profile.local.yaml`, and +(for Tier 2) real Azure resources. Do not invoke them on your own. Instead: + +1. **Surface them to the user** when you make a change that touches a + user-facing command path covered by an existing scenario (anything under + `internal/cmd/` that maps to a `cmd:*` tag, or shared helpers used by those + commands). In your summary, point the user at the relevant scenario(s) + and suggest they run the tester against the matching tag set to validate + the change. + +2. **Add or update a scenario** when your change introduces a new command, + flag, prompt, or user-visible flow — or meaningfully alters an existing + one. Place the new YAML alongside the others, follow the tagging taxonomy + documented in the scenarios README, and mention the new/changed scenario + in the PR description so reviewers know to exercise it. + +3. **Do not modify scenarios to match buggy behavior.** Scenarios are + user-facing specifications of how the command should behave; if a scenario + fails because of your change, prefer fixing the code unless the behavior + change is intentional and documented. + ## Error handling This extension uses `internal/exterrors` so the azd host can show a useful message, attach an optional suggestion, and emit stable telemetry. diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 2d5c10d89d6..f6a93b191c0 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -41,6 +41,7 @@ words: - CLIENTSECRET - curr - dataagent + - defaultyourvalue - envkey - exterrors - goyaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore new file mode 100644 index 00000000000..a3ca34ee539 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore @@ -0,0 +1,6 @@ +# cli-interactive-tester run artifacts (screenshots, HTML reports, scrollback) +.reports/ + +# Per-developer / per-CI scenario profile (identifying values). Bootstrap +# from profile.local.yaml.example. See README "Profile / overrides". +profile.local.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml new file mode 100644 index 00000000000..dfd555970bb --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -0,0 +1,19 @@ +# Tier 0 (offline) — `doctor` in an empty directory degrades gracefully. +name: "doctor-empty-dir" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-doctor-empty-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] + +# Guarantee an empty working dir so the "no azd project" path is exercised. +# start_session recreates the dir, so removing it is enough. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor in a directory that has no azd project. Wait for the check report to render." + - "Confirm the command reports checks as skipped (or failed) with readable, non-crashing output — it should NOT panic or print a Go stack trace." + - "Note the exit code behavior described in help: 2 when all checks are skipped (preconditions unmet), 1 on failure, 0 when at least one passes." + - "Take a screenshot of the doctor report." + - "Report a finding if doctor crashes, prints a stack trace, or gives an unhelpful/confusing message for a missing project." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml new file mode 100644 index 00000000000..dfba27f8b89 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -0,0 +1,18 @@ +# Tier 0 (offline) — `doctor --local-only` skips remote checks. +name: "doctor-local-only" +command: "azd ai agent doctor --local-only" +cwd: "~/working/azd-agents-doctor-local-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] + +# Guarantee an empty working dir for a deterministic local-only run. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-local-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor with --local-only. Wait for the report to render." + - "Confirm only local checks are attempted and remote/network-dependent checks are skipped (the report should indicate skipped remote checks)." + - "Confirm the run completes without attempting any network calls and without crashing." + - "Take a screenshot of the report." + - "Report a finding if remote checks still run, or if the output is confusing about what was skipped." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml new file mode 100644 index 00000000000..eca212c3f81 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify root help lists every expected subcommand. +name: "help-root" +command: "azd ai agent --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:help", "parallel-safe"] + +goals: + - "Wait for the help output to render (it includes an ASCII-art banner and a 'Usage:' section)." + - "Confirm the 'Available Commands' list includes: doctor, endpoint, eval, files, init, invoke, monitor, optimize, run, sample, sessions, show, version." + - "Confirm the global flags are listed: --cwd/-C, --debug, --environment/-e, --no-prompt, --output/-o." + - "Take a screenshot of the help output." + - "Report a finding if any listed command is missing, if the usage text is malformed, or if the command exits non-zero." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml new file mode 100644 index 00000000000..6e11833c869 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -0,0 +1,37 @@ +# Tier 0 (offline) — exercise the `init` interactive picker UX, then abort. +# +# This scenario probes the early interactive prompts only (init mode, language, +# template list). It intentionally ABORTS with Ctrl-C before reaching any Azure / +# Foundry project prompts, so it needs no Azure auth and creates no resources. +name: "init-picker-navigation" +command: "azd ai agent init" +cwd: "~/working/azd-agents-picker-{instance}" +tags: ["tier:0", "cmd:init", "picker", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Clean dir, then seed a file so the dir is non-empty. The init-method picker +# ("Use the code in the current directory" / "Start new from a template") only +# appears when the working directory is NOT empty; on an empty dir the command +# auto-selects the template flow and skips the picker. +pre: + - run: "rm -rf ~/working/azd-agents-picker-{instance}" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-picker-{instance} && touch ~/working/azd-agents-picker-{instance}/placeholder.txt" + cwd: "~/working" + name: "seed a placeholder file so the init-method picker appears" + +goals: + - "Wait for the first prompt asking how to initialize (it should appear because the directory is non-empty): 'Use the code in the current directory' / 'Start new from a template'." + - "Select 'Start new from a template', then wait for the language prompt." + - "Select a language (Python), then wait for the template list prompt." + - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." + - "Type a string that matches nothing; confirm the list shows an empty/no-match state and does not crash." + - "Clear the filter (backspace) and confirm the full list returns." + - "Press Down many times past the end of the list; confirm selection stays bounded and does not error." + - "If a prompt mentions a '?' hint, press '?' and confirm a helpful hint is shown." + - "Take a screenshot at each interesting state." + - "Press Ctrl-C (or Escape at the top-level prompt) to abort. Confirm it exits cleanly without a stack trace and without leaving a half-written azure.yaml." + - "Report a finding for any picker glitch: filter not recovering, crash on no-match, unbounded scrolling, unhelpful hints, or a messy/abrupt abort." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml new file mode 100644 index 00000000000..fafecf07551 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -0,0 +1,25 @@ +# Tier 0 (offline) — `init` rejects a positional argument combined with --manifest. +# +# There is no `--from-code` flag. A real, offline-detectable conflict is passing +# BOTH a positional manifest argument AND -m/--manifest, which the command +# rejects up front (CodeConflictingArguments) before any wizard or network call. +name: "init-validate-mutually-exclusive" +command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" +cwd: "~/working/azd-agents-validate-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so leftover files can't affect the validation result. +pre: + - run: "rm -rf ~/working/azd-agents-validate-{instance}" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the command to fail fast (it should not start the interactive wizard)." + - "Confirm the error message clearly states that you cannot pass both a positional argument and --manifest." + - "Confirm the process exits non-zero and does NOT create or modify any project files in the directory." + - "Take a screenshot of the error output." + - "Report a finding if the conflicting arguments are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml new file mode 100644 index 00000000000..dd8e1b3c313 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -0,0 +1,21 @@ +# Tier 0 (offline) — `init --no-prompt` with no resolvable inputs fails helpfully. +name: "init-validate-no-prompt-missing" +command: "azd ai agent init --no-prompt" +cwd: "~/working/azd-agents-validate-noprompt-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Empty dir is a precondition: no existing code/manifest to resolve from. +pre: + - run: "rm -rf ~/working/azd-agents-validate-noprompt-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run init in --no-prompt mode in an empty directory with no flags and no existing code/manifest." + - "Confirm the command does NOT hang waiting for input (no-prompt must never block on a prompt)." + - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --src, --manifest, or --project-id)." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, prompts interactively despite --no-prompt, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml new file mode 100644 index 00000000000..55711345ff0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -0,0 +1,13 @@ +# Tier 0 (offline) — `sample list` JSON output and filter flags. +name: "sample-list-json-filters" +command: "bash" +cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] + +goals: + - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." + - "Run: azd ai agent sample list --language python --output json. Confirm results are filtered to python samples only." + - "Run: azd ai agent sample list --type agent --output json. Confirm results only include agent-type templates." + - "Run: azd ai agent sample list --featured-only --output json. Confirm only featured samples are returned (a subset of the full list)." + - "Take a screenshot after each command." + - "Report a finding if any command produces invalid JSON, ignores its filter, or errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml new file mode 100644 index 00000000000..76323265ec5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -0,0 +1,11 @@ +# Tier 0 (offline) — `sample list` renders the human-readable catalog. +name: "sample-list-text" +command: "azd ai agent sample list" +cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] + +goals: + - "Wait for the curated sample catalog to render as human-readable text." + - "Confirm at least one sample entry is shown with a name/title and a manifest or repo reference usable with 'azd ai agent init -m' or 'azd init -t'." + - "Take a screenshot of the catalog output." + - "Report a finding if the list is empty, truncated, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml new file mode 100644 index 00000000000..b25e9ae8392 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml @@ -0,0 +1,11 @@ +# Tier 0 (offline) — verify `azd ai agent version` prints a version string. +name: "version" +command: "azd ai agent version" +cwd: "/tmp" +tags: ["tier:0", "cmd:version", "parallel-safe"] + +goals: + - "Wait for the command to print a version string (e.g. a value like '0.1.x-preview' or a commit-based 'vdev' build identifier)." + - "Confirm the process exits cleanly without an error or stack trace." + - "Take a screenshot of the final output." + - "Report a finding if no version is printed, if the output is empty, or if the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml new file mode 100644 index 00000000000..cd46adb445c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -0,0 +1,40 @@ +# Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# Targets the --deploy-mode code path which prompts for entry-point and runtime +# (instead of building a container image). There is no --from-code flag; the +# from-code flow is selected interactively at the init-method prompt. +name: "init-deploy-mode-code" +command: "azd ai agent init --deploy-mode code" +cwd: "~/working/azd-agents-t1-code-deploy-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a committed Python fixture so code-deploy has real source to package. +# Override the fixture location with AZD_AGENTS_FIXTURES if needed. +pre: + - run: "rm -rf ~/working/azd-agents-t1-code-deploy-{instance}" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." + - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." + - "If an existing agent manifest is detected, confirm reuse." + - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." + - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the entry-point or runtime prompts are missing, confusing, or not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml new file mode 100644 index 00000000000..81a29eb4e44 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -0,0 +1,39 @@ +# Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# Also requires GitHub login: `gh auth login` (manifest download can fall back to +# the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails +# fast if gh is not authenticated. +# Verifies that the override flags are honored in the generated files. +name: "init-flags-agent-name-model" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" +cwd: "~/working/azd-agents-t1-flags-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" + - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" + cwd: "~/working" + name: "reset working dir" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "Wait for the manifest to download and parse." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the manifest defaults." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml new file mode 100644 index 00000000000..84fbcbb5b47 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -0,0 +1,37 @@ +# Tier 1 (auth, scaffold only) — init from existing code in the current directory. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# Precondition: the cwd should already contain agent source code (and ideally an +# agent manifest). The pre hooks seed a committed Python fixture so this is +# guaranteed and the run is idempotent. Override the fixture location with +# AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. +name: "init-from-code" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-from-code-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-t1-from-code-{instance}" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." + - "Wait for the tool to inspect the current directory and treat its code as the agent source." + - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' in the output." + - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if code-detection misbehaves or if the manifest-reuse prompt is confusing." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml new file mode 100644 index 00000000000..e326894f897 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -0,0 +1,40 @@ +# Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# Also requires GitHub login: `gh auth login`. Downloading the manifest (and its +# sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub +# API is rate-limited, which would otherwise drop into an interactive gh login +# mid-run. The pre hook fails fast if gh is not authenticated. +name: "init-from-manifest-url" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" +cwd: "~/working/azd-agents-t1-manifest-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" + - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" + cwd: "~/working" + name: "reset working dir" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "Wait for the tool to fetch and parse the manifest from the provided URL." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}' (or accept the manifest's model if one is pinned)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the manifest fails to download/parse, or if any field is dropped during scaffold." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml new file mode 100644 index 00000000000..74b4bf1e555 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -0,0 +1,34 @@ +# Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. All `{name}` placeholders below come from the merged profile. +name: "init-template-dotnet" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-dotnet-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-dotnet-{instance}" + cwd: "~/working" + name: "reset working dir" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When asked how to initialize, select 'Start new from a template'." + - "Select C# / .NET as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the .NET language path behaves differently from Python in any confusing way." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml new file mode 100644 index 00000000000..da845744fc7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -0,0 +1,37 @@ +# Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. +# +# Requires Azure login (see README "Authentication"). Reads subscriptions/Foundry +# projects but does NOT run `azd provision`, so no resources are created and no +# cost is incurred. All `{name}` placeholders below come from the merged +# profile (`profile.yaml` + `profile.local.yaml`); the orchestrator passes them +# as `session_vars` to every MCP call. +name: "init-template-python" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-python-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-python-{instance}" + cwd: "~/working" + name: "reset working dir" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding for any confusing prompt, wrong default, or scaffold problem." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml new file mode 100644 index 00000000000..94234d7c8b1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -0,0 +1,54 @@ +# Tier 2 (cloud E2E) — SETUP: deploy the shared agent used by all 21-..2A scenarios. +# +# ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared +# and scaffolds the project into a subdirectory named after the agent, so the +# deployed project lives in ~/working/azd-agents-shared/{shared_agent_name} +# (where {shared_agent_name} = {prefix}-{shared_agent_suffix}, e.g. +# "alice-basic-responses") and is reused by the targeted scenarios. The agent +# name MUST be exactly that value so the subdirectory path is deterministic +# and the reuse scenarios can find it. Run 2Z-teardown-down.yaml LAST to +# clean up. All `{name}` placeholders come from the merged profile +# (`profile.yaml` + `profile.local.yaml`). +name: "setup-deploy-shared-agent" +command: "azd ai agent init" +cwd: "~/working/azd-agents-shared" +tags: ["tier:2", "cmd:init", "cmd:provision", "cmd:deploy", "serial-only"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Idempotent setup: if a previous run left a project here, tear down its Azure +# resources FIRST (so we never orphan them), then clear the dir for a fresh +# deploy. The down step gets a long timeout and continues on error (there may be +# nothing to tear down). Re-using a clean path also avoids the resource-name hash +# collision in issue #8360. +pre: + - run: "if [ -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml ]; then (cd ~/working/azd-agents-shared/{shared_agent_name} && azd down --force --purge); fi" + cwd: "~/working/azd-agents-shared" + name: "tear down any leftover deployed agent" + continue_on_error: true + timeout: 900 + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' (e.g. '{shared_agent_name}'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Select the 'Basic Responses' template from the list." + - "When prompted for the AGENT NAME, set it to EXACTLY '{shared_agent_name}' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named '{shared_agent_name}'." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Change directory into the new '{shared_agent_name}' subdirectory (run 'cd {shared_agent_name}') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." + - "Run 'azd provision' (from inside the '{shared_agent_name}' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." + - "After provision succeeds, run 'azd deploy' (from the same subdirectory) and wait for it to succeed. This deploys the agent to the provisioned infrastructure — it is a required, separate step from provision before the agent can be shown or invoked." + - "After deploy, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." + - "Take a screenshot of the successful provision, deploy, and 'show' output." + - "Report a finding if init, provision, or deploy fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml new file mode 100644 index 00000000000..8b565759c65 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `show --output json` returns well-formed JSON. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show-json" +command: "azd ai agent show --output json" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the command to print JSON." + - "Confirm the output is valid JSON and includes the agent name, version, status, and endpoint fields." + - "Confirm the JSON values match what the table form of 'show' reports." + - "Take a screenshot of the JSON output." + - "Report a finding if the JSON is malformed, missing expected fields, or inconsistent with the table output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml new file mode 100644 index 00000000000..7766378d8e7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `show` reports the deployed agent (table output). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show" +command: "azd ai agent show" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the status table to render." + - "Confirm the agent name, version, and a status field (e.g. 'active') are shown, auto-resolved from azure.yaml and the current azd environment." + - "Confirm the endpoint URL for the deployed agent is displayed." + - "Take a screenshot of the status table." + - "Report a finding if the agent cannot be resolved, the status looks wrong, or fields are missing/blank." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml new file mode 100644 index 00000000000..cb4a19233fb --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `invoke -f ` sends a file as the request body. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-input-file" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Create a small request file, e.g.: printf '%s' 'Summarize the benefits of unit testing in one sentence.' > request.txt" + - "Run: azd ai agent invoke -f request.txt and wait for the response." + - "Confirm the agent responds to the contents of the file (not to an empty or literal-path message)." + - "Take a screenshot of the response." + - "Report a finding if the file body is ignored, mis-read, or if -f errors on a valid file." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml new file mode 100644 index 00000000000..8d38ef84524 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -0,0 +1,27 @@ +# Tier 2 (cloud E2E) — `invoke` SESSION vs CONVERSATION memory semantics. +# +# A SESSION and a CONVERSATION are distinct concepts. For the responses-protocol +# agent (Basic Responses), multi-turn memory is bound to the CONVERSATION, not the +# session: `--new-session` starts a fresh session but reuses the saved conversation +# (memory persists), while `--new-conversation` is what actually resets memory. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-session-vs-conversation" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent invoke \"My name is Quinn. Remember it.\" and wait for a response." + - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session and conversation were reused, so multi-turn memory works)." + - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent STILL recalls 'Quinn'. This is EXPECTED, not a bug: a SESSION and a CONVERSATION are distinct concepts. For this responses-protocol agent, multi-turn memory is bound to the CONVERSATION, not the session. --new-session starts a fresh session but reuses the saved conversation, so memory persists." + - "Run: azd ai agent invoke --new-conversation \"What is my name?\" and confirm the agent does NOT recall 'Quinn'. This is the true memory reset: --new-conversation discards the conversation that holds multi-turn memory." + - "Take a screenshot after each invoke." + - "Report a finding if: the first two invokes fail to establish/recall memory; OR --new-conversation still recalls 'Quinn' (memory was not reset). Do NOT report a finding merely because --new-session still recalled 'Quinn' — that is the expected session-vs-conversation distinction." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml new file mode 100644 index 00000000000..5ed85dd4c08 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `invoke` a deployed agent remotely on Foundry. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-remote" +command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the remote invocation to complete (the agent is auto-detected from azure.yaml)." + - "Confirm a non-empty model response is returned (not an error)." + - "Confirm the command does NOT 404 and the response references no auth/endpoint errors." + - "Take a screenshot of the response." + - "Report a finding if invoke returns a 404, an auth error, an empty response, or hangs past a reasonable timeout." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml new file mode 100644 index 00000000000..4c8068b1ff4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -0,0 +1,25 @@ +# Tier 2 (cloud E2E) — `sessions` lifecycle: create, list, show, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `sessions` command group end-to-end in one run. +name: "sessions-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:sessions", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent sessions create. Confirm a new session is created and note its session ID." + - "Run: azd ai agent sessions list --output table. Confirm the newly created session appears in the list." + - "Run: azd ai agent sessions show using the ID from create. Confirm session details (status, version) are shown." + - "Run: azd ai agent sessions delete . Confirm the session is deleted (synchronously)." + - "Run: azd ai agent sessions list again. The service SOFT-DELETES sessions, so it is EXPECTED and acceptable for the entry to still appear with status 'deleted' (it should NOT appear as 'active'). Either the entry being gone OR present with status 'deleted' is a PASS." + - "Run: azd ai agent sessions show for the deleted session and confirm it reports the session as not found / deleted (this verifies the delete took effect even though list may still show it)." + - "Take a screenshot after each step." + - "Report a finding only if any subcommand errors, if the created session is missing from list before delete, or if after delete the session still shows as 'active' (a soft-deleted 'deleted' status is NOT a finding)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml new file mode 100644 index 00000000000..bed637575e2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -0,0 +1,29 @@ +# Tier 2 (cloud E2E) — `files` lifecycle: upload, list, stat, mkdir, download, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `files` command group end-to-end against the last invoke session. +# Note: file operations target a session — run an invoke first if no session exists. +name: "files-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:files", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure a session exists: run 'azd ai agent invoke \"hi\"' first so files commands have a session to target (or pass --session-id consistently)." + - "Create a local test file: printf 'col1,col2\\n1,2\\n' > sample.csv" + - "Run: azd ai agent files upload ./sample.csv --target-path /data/sample.csv. Confirm success." + - "Run: azd ai agent files list /data --output table. Confirm sample.csv is listed." + - "Run: azd ai agent files stat /data/sample.csv. Confirm metadata (size/type) is returned." + - "Run: azd ai agent files mkdir /data/output. Confirm the directory is created." + - "Run: azd ai agent files download /data/sample.csv --target-path ./downloaded.csv. Confirm the file downloads and its contents match the original." + - "Run: azd ai agent files delete /data/output --recursive and azd ai agent files delete /data/sample.csv. Confirm both are removed." + - "Run: azd ai agent files list /data and confirm the deleted entries are gone." + - "Take a screenshot after each step." + - "Report a finding if any subcommand errors, if upload/download corrupts the file, or if a session cannot be resolved." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml new file mode 100644 index 00000000000..4c7aabf36d1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — `monitor` console logs for the last session. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists to stream logs from. +name: "monitor-console" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure recent activity: run 'azd ai agent invoke \"generate some output\"' so the session has console logs." + - "Run: azd ai agent monitor --tail 50. Confirm recent console (stdout/stderr) log lines are fetched and printed, then the command exits (no --follow)." + - "Confirm timestamps are shown in local time by default and the output is readable." + - "Take a screenshot of the log output." + - "Report a finding if no session can be auto-resolved, if logs are empty despite recent activity, or if the output is garbled." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml new file mode 100644 index 00000000000..0aa08b4dd72 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — `monitor --type system` container/system events. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists. +name: "monitor-system" +command: "bash" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent monitor --type system --tail 50. Confirm system/container event logs are fetched and printed." + - "Confirm the output is distinct from console logs (system events such as container lifecycle, not stdout/stderr)." + - "Optionally run: azd ai agent monitor --type system --follow for a few seconds to confirm streaming works, then stop it with Ctrl-C and confirm it exits cleanly." + - "Take a screenshot of the event output." + - "Report a finding if system logs are empty, indistinguishable from console logs, or if --follow does not stream / does not stop cleanly." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml new file mode 100644 index 00000000000..0ac9922bb4b --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -0,0 +1,36 @@ +# Tier 2 (cloud E2E) — `endpoint update` patches endpoint/card without a new version. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "endpoint-update" +command: "azd ai agent endpoint update" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:endpoint", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors + # ("nothing to update") if neither is defined. The Basic Responses template + # defines neither, so inject a minimal agent_card (idempotently) before the run + # so there is something to patch. + - run: | + f="$(find ~/working/azd-agents-shared/{shared_agent_name} -name agent.yaml | head -1)" + if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then + printf '\nagent_card:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" + echo "Injected agent_card into $f" + else + echo "agent_card already present or agent.yaml not found" + fi + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "inject agent_card so there is something to patch" + continue_on_error: true + +goals: + - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agent_card was injected into agent.yaml during setup so the patch has content." + - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." + - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." + - "Take a screenshot of the update result and the post-update 'show' output." + - "Report a finding if endpoint update creates a new version, errors against an already-deployed agent, or gives a confusing result." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml new file mode 100644 index 00000000000..325fbe0c67d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -0,0 +1,37 @@ +# Tier 2 (cloud E2E) — run the agent locally and invoke it via --local. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run (project scaffolded). +# `azd ai agent run` blocks the terminal, so this needs TWO sessions: one runs the +# agent, a second invokes it with --local. +name: "run-local-and-invoke-local" +command: "azd ai agent run --port {agent} --no-inspector" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:run", "cmd:invoke", "serial-only"] + +# Reserve a free port per scenario run so parallel local runs don't collide on +# the default 8088, and so the run + invoke sessions find each other (a pool is +# shared across every start_session that passes the same scenario_path). +allocate_ports: [agent] + +notes: | + Use two tester sessions that share this scenario's port pool — start BOTH with + the same scenario_path (and the same instance_id if running in parallel): + - session_id "run-{instance}": runs this 'command' (azd ai agent run --port {agent} --no-inspector). + - session_id "invoke-{instance}": a bash session for 'azd ai agent invoke --local --port {agent}'. + The agent listens on the allocated port {agent}. Stop the run session with Ctrl-C at the end. + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "In the run session, wait until the local agent reports it is listening (look for a port {agent} / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." + - "Start a SECOND session (session_id 'invoke-{instance}', same scenario_path, same instance_id) running bash in the same cwd so it shares this scenario's allocated port {agent}." + - "In the invoke session, run: azd ai agent invoke --local --port {agent} \"Hello from local!\" and wait for a response from the locally running agent." + - "Confirm the response comes from the local process (not Foundry) and is non-empty." + - "Take a screenshot of both the run session (showing it listening on {agent}) and the invoke response." + - "Stop the run session with Ctrl-C and confirm the local agent shuts down cleanly." + - "Report a finding if the local server fails to start, if --local invoke cannot reach it on port {agent}, or if shutdown is messy." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml new file mode 100644 index 00000000000..2b8d289ba5f --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — `doctor` against a fully provisioned project (checks pass). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "doctor-provisioned-all-pass" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:doctor", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run doctor against the provisioned project. Wait for the full check suite (local + remote) to render." + - "Confirm the local checks pass (project, azure.yaml, agent service config)." + - "Confirm the remote checks pass (deployed agent reachable / active)." + - "Confirm doctor suggests a sensible next command (e.g. invoke or run) and exits 0 (at least one check passed, none failed)." + - "KNOWN-ACCEPTABLE WARNING: a WARNING about agent identity / role assignments is EXPECTED in this test subscription (its ABAC conditional role-assignment policy blocks some role reads) and must NOT be treated as a failure. Only an actual FAILED check (not a warning) is a finding." + - "Take a screenshot of the doctor report." + - "Report a finding if a check FAILS (red/error), or is skipped unexpectedly, for a healthy provisioned project, or if the suggested next step is wrong. Do NOT report the known identity/role-assignment warning described above." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml new file mode 100644 index 00000000000..c2c845f1ec5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — TEARDOWN: destroy all resources created by the setup scenario. +# +# ⚠️ Run this LAST, after all 21-..2A targeted scenarios are done. Cleans up the +# shared agent and its Azure resources to stop incurring cost. +name: "teardown-down" +command: "azd down --force --purge" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:down", "serial-only"] + +goals: + - "Run 'azd down --force --purge' in the shared project directory." + - "Wait for the full teardown to complete — all resources (Foundry account/project, model deployment, container resources) should be deleted and purged." + - "Confirm the command reports success and no resources are left behind." + - "Take a screenshot of the completed teardown." + - "Report a finding if teardown fails, leaves orphaned resources, or hangs." + +# After the in-session teardown, clear the shared working dir so the next full +# Tier 2 pass starts clean. continue_on_error so a missing dir is not fatal. +post: + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md new file mode 100644 index 00000000000..b0ed50c84ef --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -0,0 +1,572 @@ + +# `azd ai agent` — cli-interactive-tester scenarios + +Goal-based scenarios for driving the `azure.ai.agents` extension through the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server. Each file targets **one** command or flow at a time and uses the +strict `goals:` list format so the run is repeatable and reviewable. + +## How to run + +Register the cli-interactive-tester MCP server (see its README), then +**bootstrap your profile** (one-time, per checkout — see [Profile / overrides](#profile--overrides)): + +```sh +cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` and `subscription` at minimum +``` + +Then ask Copilot CLI to load a scenario and accomplish its goals. The +orchestrator must **load both profile files, merge them (local overrides +shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass +the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, +`start_session`, and `run_post_hooks` call** — the scenario YAMLs reference +those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, +`{tenant}` (optional), and `{shared_agent_name}` placeholders. + +Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting +the working dir or seeding a fixture), and a few declare **`post:` hooks** +(cleanup). The agent must invoke them via the tester's `run_pre_hooks` / +`run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. +See [Pre/post hooks](#prepost-hooks) below. + +Here's also a sample prompt to run all of the scenarios, utilizing fleet mode: + +``` +Within the agents extension, there is a tests/cli-interactive-tester-scenarios directory, containing +a set of test scenarios for the cli-interactive-tester. I want you to use the cli-interactive-tester to; + load the scenarios, + start the session and accomplish the goals, + if the scenario declares pre or post hooks, run them before/after the session, + and take screenshots at each step. + +First, read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge +them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}". +Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call — the scenarios reference {prefix}, {subscription}, {region}, {model}, +{tenant} (optional), and {shared_agent_name} placeholders. + +I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios +in tiers 0 and 1 are completely independent of each other and can be run in parallel. The scenarios +in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make +sure to take that into account when distributing the work. I want to run all of the tests regardless +of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. + +After all of these scenarios are run, create a final result report. + +Create a plan to accomplish this +``` + +For more selective fan-outs (e.g. "just the `init` scenarios" or "everything +in Tier 0") the tester's `list_scenarios` MCP tool filters by `tags:`. See +[Tags](#tags) below for the taxonomy and an example tag-filtered prompt. + +## Paths run inside WSL (on Windows) + +The cli-interactive-tester drives CLIs through **tmux**, which on Windows runs +inside **WSL**. The scenario YAML files live on the Windows filesystem (in this +repo), but every `cwd` value is resolved against the **WSL filesystem** where the +command actually executes: + +- `~/working/azd-agents-shared` → `/home//working/azd-agents-shared` +- `/tmp` → WSL's `/tmp` + +Implications: + +- `azd` and the `azure.ai.agents` extension must be installed **inside WSL**, + since that is where the scenario commands run. +- `cwd` directories do not need to pre-exist — the tester creates them if missing. +- The `cwd` convention is three-way by design: ephemeral `/tmp` for read-only + scenarios that touch no project (`version`, `--help`, `sample list`); a unique + `~/working/azd-agents-*-{instance}` dir per `init`/`doctor` scenario for + isolation (the `{instance}` suffix keeps concurrent runs of the same scenario + apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a + single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they + operate on the same deployed agent. `20-setup` runs `init` in that shared dir, + which scaffolds the project into a subdirectory named after the agent, so the + deployed project actually lives in `~/working/azd-agents-shared/{shared_agent_name}` + (where `{shared_agent_name} = {prefix}-{shared_agent_suffix}` from your + [profile](#profile--overrides), e.g. `alice-basic-responses`); the reuse and + teardown scenarios run with that subdirectory as their `cwd`. + +On macOS/Linux these are simply native paths (no WSL involved). + +### This applies to MCP tool arguments too + +The same path-resolution rule applies to **every path-shaped argument an +orchestrator passes to the tester's MCP tools** — most importantly the `path:` +argument on `load_scenario`, `run_pre_hooks`, and `run_post_hooks`, and the +`scenario_path:` argument on `start_session`. The server resolves them on the +WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: + +| Orchestrator OS | Pass to MCP tools | Don't pass | +| --- | --- | --- | +| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml` | `C:\Repos\azure-dev\...\00-version.yaml` | +| macOS / Linux | native absolute path | — | + +**Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the +path style is almost certainly the cause — translate `C:\…` to `/mnt/c/…` and +retry one call before fanning out. + +## Authentication + +Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in +manually before** starting a run. The scenarios do **not** perform login +themselves, and the test-driving agent **cannot** complete it either: `az login` +opens a **separate browser window** for account selection that requires +human interaction outside the terminal the agent controls. Treat auth as a +one-time manual prerequisite, not a scenario step. + +Inside WSL, a human runs (substituting `{tenant}` and `{subscription}` with +the values from their [profile](#profile--overrides) — omit `--tenant` +entirely if `tenant` isn't set in `profile.local.yaml`): + +``` +az login --tenant {tenant} # or just `az login` if {tenant} is unset +``` + +This opens the interactive sign-in flow and then: + +1. **Browser account selection** — a separate browser window opens; the human + picks the account in the `{tenant}` tenant (or any tenant, if `{tenant}` + isn't set). The agent cannot do this. +2. **Subscription selection** — back in the terminal, select the + `{subscription}` subscription. + +Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL +session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of +them reuse that session credential. + +### GitHub login (manifest scenarios) + +The manifest scenarios (`10-init-from-manifest-url`, +`10-init-flags-agent-name-model`) download an agent manifest — and its sibling +files — from a public GitHub repo. The CLI first tries the anonymous GitHub API, +but when that's rate-limited (60 req/hr) it falls back to the `gh` CLI, which +would otherwise drop into an **interactive GitHub login** mid-run. Like +`az login`, this is a one-time manual prerequisite the agent can't complete, so a +human must run it once per WSL session **before** driving those scenarios: + +``` +gh auth login +``` + +Those scenarios include a `pre` hook that runs `gh auth status` and **fails fast** +if GitHub CLI isn't authenticated, so a missing login surfaces as a clear setup +error instead of a hung interactive prompt. + +## Parallel-readiness & port allocation + +The tester can run **N concurrent instances of the same scenario** and can +**allocate free TCP ports** per run. Scenarios here are authored to take +advantage of both where it's safe. + +- **`{instance}` substitution.** `start_session(..., instance_id="1")` exposes + `{instance}` for substitution into `command`, `cwd`, `env`, hook fields, and + `goals`. It **defaults to `"main"`** when `instance_id` is omitted, so a single + run is unchanged (dirs/names just end in `-main`). +- **Which scenarios are parallel-ready:** + - **Tier 0 work-dir scenarios** (`doctor`, picker, validate) and **all Tier 1 + `init` scenarios** suffix their `cwd` (and hook paths) with `-{instance}`, so + concurrent instances get isolated working directories. + - **Tier 1 resource names** are suffixed with `-{instance}` too (via the + RESOURCE NAMING goal and the `--agent-name` flag), so parallel instances + don't collide on Azure resource names. + - **`27-run-local-and-invoke-local`** declares `allocate_ports: [agent]` and + binds `azd ai agent run`/`invoke --local` to `--port {agent}`. A port pool is + shared across every `start_session` with the same `scenario_path`, so the + `run` and `invoke` sessions find each other; parallel local runs each get a + distinct port instead of colliding on the default `8088`. +- **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), + plus `20-setup` and `2Z-teardown`, all share the one deployed agent under + `~/working/azd-agents-shared` (the project itself lives in the + `{shared_agent_name}` subdirectory created by `20-setup`). They are + **not** parameterized with `{instance}` (doing so would break the shared-agent + assumption) and should be run serially. + +To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the +same `instance_id` for paired `run`/`invoke` sessions of one scenario). + +## Orchestrating a fleet run + +When a driving agent wants to run **many scenarios concurrently** (e.g. via +parallel background sub-agents, one scenario per sub-agent), pick the right +fan-out primitive for the shape of the run: + +- **Different scenarios in parallel** (the common case for a full Tier 0/1 + sweep): give each sub-agent a distinct, descriptive `session_id` — e.g. + `fleet-00-version`, `fleet-10-init-from-code` — and call `start_session` with + the scenario's own `cwd`. **No `instance_id` is needed**: each scenario's `cwd` + already isolates itself via the `{instance}` substitution, which defaults to + `"main"` when `instance_id` is omitted. +- **Same scenario N times in parallel:** use `instance_id="1"`, `"2"`, … per + call. See [Parallel-readiness](#parallel-readiness--port-allocation) for which + scenarios are authored to support this. +- **Tier 2 ordering is fixed**, not parallel-friendly. Run `20-setup` first, + then the targeted `21-…2A-` scenarios **serially** (they share one deployed + agent and mutate shared state — sessions, files, endpoint configuration — + so parallel runs interfere), then `2Z-teardown` last. See the + [Tier 2](#tier-2--cloud-end-to-end-prefix-2x---%EF%B8%8F-incurs-azure-cost) + section. + +### Operational guardrails for the orchestrator + +A few hard-won lessons that apply regardless of fleet size: + +- **Validate the recipe with one sub-agent before fanning out.** Spend 30 + seconds confirming that `load_scenario`, `start_session`, and one + `send_action` round-trip work end-to-end for *one* scenario before launching + a wave. This is the cheapest way to catch wiring issues (wrong path style, + wrong tool surface, auth not set up) before they multiply across many agents. +- **Background sub-agents are typically not cancellable mid-run.** Once launched, + they will run to completion (or until the runtime times them out). For Tier 1 + and especially Tier 2 scenarios with Azure side effects, launch + conservatively — a stop request can't recall an in-flight `azd provision`. +- **Keep waves small.** The wall-clock bottleneck on a fleet run is per-agent + LLM time and per-account model concurrency, not the MCP server (which is + per-`session_id`-parallel by design). Launching 4–6 sub-agents at a time and + rolling forward typically finishes a sweep faster than launching everything + at once. + +## Driving conventions + +These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent +should follow them so the runs actually *test* the CLI instead of papering over +its bugs: + +- **Don't verify/retry after a `select`.** These runs exist to catch picker + bugs; reading back the echo and "correcting" a pick hides the very defect the + test is for. Send the action and let downstream prompts surface any failure. +- **Treat a select miss as a hard failure.** The tester's `select_by_text` is + fail-loud: a missing target raises `LookupError`, surfaced as + `ERROR during 'select': …`. **Report a finding and stop** — do not retry with a + different `choice_text`/`choice_index` to work around it. +- **Prefer `choice_text` over `choice_index`** when the label is stable (indices + shift between releases). +- **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent + name) come pre-populated with an editable default; typing without clearing + *appends* to it. Select-all then delete (or backspace) first so your value + replaces the default instead of producing `defaultyourvalue`. +- **Pause before the first cloud-creating action.** Provisioning is expensive and + irreversible-ish; confirm with the user before entering an `init`/`provision` + flow that creates real resources (especially when running in parallel). +- **Pass `run_name=` to every `start_session` call.** The + scenario stem is the YAML filename without `.yaml` (e.g. `00-version`, + `21-show-json`, `27-run-local-and-invoke-local`). Without `run_name` the + tester auto-names the run folder `agent_YYYYMMDD_HHMMSS`, which makes + archived runs in `.reports//tester-reports/` hard to cross-reference + with the scenario list. For scenarios that start two sessions + (e.g. `27-run-local-and-invoke-local`), suffix the run_name with a role tag + (`27-run-local-and-invoke-local-run`, `27-run-local-and-invoke-local-invoke`) + so each session gets its own clearly named folder. +- **Pass `output_dir` to every `start_session` call** so the tester writes + screenshots and HTML reports directly into this repo's archive layout + instead of its own working directory. Use the WSL path of the + `.reports//tester-reports/` folder under this scenarios + directory, with `` of the form `YYYYMMDD-HHMMSS`. Pick **one** + `` per suite run and reuse it across every session — this + groups all scenarios from one run under a single folder. The driving agent + should also write the final cross-scenario summary to + `.reports//FINAL-REPORT.md` at the end. Example + `output_dir` (the WSL view of this scenarios directory in this repo): + `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.reports/20260603-171132/tester-reports`. + If your clone lives elsewhere, substitute the WSL path of *your* + `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. +- **Record a time-to-complete per scenario.** Capture wall-clock duration for + every scenario (from `start_session` to `finish_session`, including pre/post + hooks) and include it as a `Duration` column in the per-scenario tables of + `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). + This makes regression slowdowns easy to spot across runs — Tier 2 in + particular has scenarios that legitimately take many minutes (provision, + deploy) and others that should complete in seconds. + + +## Tiers + +Scenarios are organized into three tiers by cost and prerequisites. Each +scenario also carries a `tags:` list that exposes the same axes plus the +command(s) under test — see [Tags](#tags) for the full taxonomy and how to +filter via `list_scenarios`. + +### Tier 0 — Offline (prefix `00-`) +No Azure auth, no network resource creation. Fast and deterministic. Safe to run +in any order, any time. + +| File | Targets | +|------|---------| +| `00-version.yaml` | `version` | +| `00-help-root.yaml` | root help / command discovery | +| `00-sample-list-text.yaml` | `sample list` (text) | +| `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | +| `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | +| `00-doctor-local-only.yaml` | `doctor --local-only` | +| `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | +| `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | +| `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | + +### Tier 1 — Auth, scaffold only (prefix `10-`) +Requires Azure login (reads subscriptions/Foundry projects) but **does not +provision** any resources and incurs no cost. Each completes a project scaffold +and verifies the generated files, then stops before `azd provision`. + +| File | Targets | +|------|---------| +| `10-init-template-python.yaml` | `init` new-from-template, Python | +| `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | +| `10-init-from-manifest-url.yaml` | `init -m ` (needs `gh auth login`) | +| `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | +| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | +| `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | + +### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost +Provisions real resources. **Run order matters:** + +1. `20-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. +2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). +3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. + +All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` +so they operate on the same deployed agent. `20-setup` runs `init` there, which +scaffolds the project into the `{shared_agent_name}` subdirectory; the +reuse and teardown scenarios run with `~/working/azd-agents-shared/{shared_agent_name}` +as their `cwd`. + +| File | Targets | +|------|---------| +| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` + `azd deploy` (SETUP) | +| `21-show.yaml` | `show` (table) | +| `21-show-json.yaml` | `show --output json` | +| `22-invoke-remote.yaml` | `invoke` (remote) | +| `22-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | +| `22-invoke-input-file.yaml` | `invoke -f ` | +| `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | +| `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | +| `25-monitor-console.yaml` | `monitor` (console) | +| `25-monitor-system.yaml` | `monitor --type system` | +| `26-endpoint-update.yaml` | `endpoint update` | +| `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | +| `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | +| `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | + +## Tags + +Every scenario carries a top-level `tags:` list so an orchestrator can pick +subsets via the tester's `list_scenarios` MCP tool. The tool's filter is **OR +across the requested tags, case-sensitive, exact match**: a scenario matches +when its `tags` contains at least one of the requested values. + +Three namespaces are used (all lowercase, kebab-case, colon-separated for +grouping — colons are treated as ordinary characters by the filter): + +| Namespace | Values | Meaning | +|---|---|---| +| `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | + +**Examples** (the tool's `tags:` parameter is OR across the list): + +| Goal | `list_scenarios(tags=…)` | +|---|---| +| All `init` scenarios across every tier | `["cmd:init"]` | +| Everything offline (no Azure auth, no cost) | `["tier:0"]` | +| All Tier 2 cloud scenarios | `["tier:2"]` | +| Invoke + sessions reuse scenarios | `["cmd:invoke", "cmd:sessions"]` | +| CLI arg-validation scenarios only | `["negative-path"]` | +| Everything safe to run in parallel | `["parallel-safe"]` | + +Sample prompt that uses tag filtering: + +``` +Use the cli-interactive-tester to run every `init` scenario across all tiers. + +First, call list_scenarios with root="tests/cli-interactive-tester-scenarios" +and tags=["cmd:init"] to enumerate the matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and +profile.local.yaml and merge them (local overrides shared); also derive +shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call. + +For each scenario returned by list_scenarios: load it, run any pre hooks, +start the session and accomplish the goals (take screenshots at each step), +finish the session, run any post hooks. The Tier 0/1 `init` scenarios are +parallel-safe (also tagged `parallel-safe`); fan them out via fleet mode. +The Tier 2 `init` scenario (`20-setup-deploy-shared-agent`) is `serial-only` +— run it on its own and only if I confirm I want to spend on Azure resources. +``` + +You can also get copilot to generate the tags list instead of manually specifying +it. For example, if you want to run all of the scenarios to test the changes +in a PR, modify the above prompt to start with something like: + +``` +Here's a PR: https://github.com/Azure/azure-dev/pull/8532. In the +tests\cli-interactive-tester-scenarios directory, there are a set of test scenarios, +with tags to categorize what they're testing. I want you to come up with a set of +tags which, when used to select these test scenarios, would properly test the +changes made by the PR provided. + +Next, call list_scenarios with those tags, to enumerate matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and .... + +``` + +And, if you're running these scenarios as a part of creating or reviewing a PR, +you can ask copilot to generate a summary report and add it as a comment directly +on the pull request. + +When adding a new scenario, give it a `tags:` list that follows this +taxonomy: at minimum a `tier:N`, at least one `cmd:*`, and either +`parallel-safe` or `serial-only`. `list_scenarios` prints `tags: []` for any +file missing a `tags:` field, so an empty list in its output signals a +regression to fix. + +> `list_scenarios` walks every `*.yaml` under the directory, including +> `profile.yaml` / `profile.local.yaml` (which surface as `(unnamed)` with +> `tags: (none)`). Filter by any `tier:*` / `cmd:*` / trait tag to exclude +> them — they intentionally carry no tags because they are configuration, +> not scenarios. + +## Profile / overrides + +Developer- and environment-specific values (subscription, region, model, +resource-name prefix, optional tenant) are **not** hardcoded in the scenario +YAMLs. Instead, the scenarios reference them via `{name}` placeholders, and +the orchestrator supplies the values as `session_vars` on every tester call. + +Two files in this directory drive the values: + +| File | Tracked? | Contents | Notes | +|---|---|---|---| +| `profile.yaml` | ✅ checked in | repo-shared defaults | `region`, `model`, `shared_agent_suffix` | +| `profile.local.yaml` | ❌ gitignored | per-developer / per-CI overrides | required: `prefix`, `subscription`. optional: `tenant` (no default) | +| `profile.local.yaml.example` | ✅ checked in | starter template | copy to `profile.local.yaml` and edit | + +Variables exposed to scenarios via `session_vars`: + +| Placeholder | Source | Default | Notes | +|---|---|---|---| +| `{prefix}` | `profile.local.yaml` | **required** | resource-name prefix; should be lowercase + hyphen-friendly so `sanitizeAgentName` doesn't mutate it | +| `{subscription}` | `profile.local.yaml` | **required** | subscription display name | +| `{tenant}` | `profile.local.yaml` | optional, no default | only consumed by the `az login` guidance above; when unset, drop `--tenant` and rely on the user's default tenant | +| `{region}` | `profile.yaml` | `East US 2` | | +| `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | +| `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | +| `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}` | Tier 2 subdirectory name — orchestrator must compute and pass alongside the others | + +**Bootstrap (one-time per checkout):** + +```sh +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` (lowercase, hyphen-friendly) and `subscription` +``` + +The orchestrator must load both files, merge (local overrides shared), derive +`shared_agent_name`, and pass the merged map as `session_vars=` on every +`load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. +Failing to thread `session_vars` leaves `{prefix}` etc. unresolved in goals and +the run will execute against literal placeholder strings. + +## Conventions + +- **Tunable values** (subscription, region, model, prefix, tenant) come from + the profile pair above — see [Profile / overrides](#profile--overrides). +- **Resource naming**: every newly created Azure resource (Foundry + project/account, azd environment, agent, model deployment, resource group) is + named with the `{prefix}-` value from your profile (and, in parallel-ready + Tier 1 scenarios, a `-{instance}` suffix) so test resources are easy to + identify, keep distinct across concurrent runs, and clean up. Note that some + fields lowercase the value and replace invalid characters with hyphens — that + normalization is expected (see `sanitizeAgentName` in the extension). +- `command:` invokes the installed extension as `azd ai agent …`. +- Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent + auto-detection prompts. +- Every scenario asks the driver to screenshot key steps and file a finding + (`report_finding`) for any confusing UX, error, or doc mismatch. + +## Pre/post hooks + +Scenarios use the tester's **`pre:`** and **`post:`** hook lists for host-side +setup and cleanup. Hooks run on the host (inside WSL on Windows), outside the +tmux session, **sequentially and fail-fast** unless a hook sets +`continue_on_error: true`. Each entry is a string or a mapping with `run` +(required), `cwd` (defaults to the scenario `cwd`, created if missing), `env`, +`continue_on_error` (default `false`), `timeout` (default **120s**), and `name`. + +How they're used here: + +- **`pre` reset** — stateful Tier 0/1 scenarios `rm -rf` their own working dir so + re-runs start clean. (`start_session` recreates the dir, so removing it is + enough; the doctor/init scenarios just need an empty dir.) +- **`pre` fixture seed** — the existing-code scenarios + (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python + fixture into the dir so the source exists before the wizard's "Use the code in + the current directory" flow inspects it (see [Fixtures](#fixtures)). +- **`pre` gh-auth guard** — the manifest scenarios (`10-init-from-manifest-url`, + `10-init-flags-agent-name-model`) run `gh auth status` and fail fast if GitHub + CLI isn't authenticated, because downloading the manifest can fall back to the + `gh` CLI (and an interactive login) when the anonymous GitHub API is + rate-limited. Run `gh auth login` first (see [Authentication](#authentication)). +- **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs + `azd down --force --purge` if a leftover project exists in the shared dir (so it + never orphans live Azure resources), then clears the dir. The down hook uses + `timeout: 900` and `continue_on_error: true`. +- **`pre` precondition guard (Tier 2 reuse)** — `21-…2A` print a clear "run + 20-setup first" warning if the shared agent isn't deployed (non-fatal). +- **`post` cleanup** — `2Z-teardown-down` clears the shared working dir after the + in-session `azd down` completes. + +## Fixtures + +`fixtures/from-code/` holds a minimal Python agent source tree (`app.py` + +`requirements.txt`) that satisfies the extension's existing-code detection +(it looks for `requirements.txt` or any `.py`, and defaults the entry point to +`app.py`). The existing-code scenarios copy it into the working dir via a `pre` +hook, then select "Use the code in the current directory" at the init prompt. + +The hook references the fixture by absolute path with an overridable env var: + +```sh +cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/." "$cwd" +``` + +If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view +of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this +`fixtures/` directory before running the existing-code scenarios. + +## Re-running scenarios (idempotency) + +Idempotency is handled **per scenario** via `pre`/`post` hooks rather than a +separate reset step — every scenario that holds state resets itself, so they can +be run back to back in any order within a tier: + +- Tier 0/1 stateful scenarios **pre-wipe** their own `cwd`. Cleanup is pre-wipe + **only** (no `post` delete), so the generated scaffold stays on disk for + inspection after a run while the next run still starts clean. +- The shared Tier 2 dir is reset by `20-setup`'s `pre` hook, which **downs any + leftover deployed project first** to avoid orphaning live Azure resources (this + also sidesteps the resource-name hash collision behind + [#8360](https://github.com/Azure/azure-dev/issues/8360)). `2Z-teardown-down` + additionally clears the dir in a `post` hook. +- Read-only scenarios (`version`, `--help`, `sample list`) run in `/tmp`, hold no + state, and declare no hooks. + +> If a Tier 2 run is interrupted before `2Z-teardown`, just re-run +> `20-setup-deploy-shared-agent` — its `pre` hook downs any live project in the +> shared dir before redeploying, so resources won't be orphaned. + +## Notes + +- `files` and `sessions` are exercised as one lifecycle scenario per command + group (rather than one file per subcommand) to avoid cross-scenario ordering + dependencies — still one command at a time. +- `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` + uses two sessions (one to run, one to invoke `--local`) that share an + allocated `{agent}` port (see + [Parallel-readiness](#parallel-readiness--port-allocation)). +- Run artifacts (screenshots, HTML reports) land in `.reports/`, which is + git-ignored. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py new file mode 100644 index 00000000000..d0d8a0c1c0b --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py @@ -0,0 +1,17 @@ +"""Minimal agent source fixture for the `azd ai agent init` existing-code scenarios. + +This file exists only so the extension's from-code detection treats the working +directory as a Python agent project (it looks for requirements.txt or any .py +file, and uses app.py as the default entry point). The init flow scaffolds an +agent.yaml around this code; the body does not need to be a fully functional +agent for the scaffold-only Tier 1 scenarios. +""" + + +def handler(request: str) -> str: + """Echo the incoming request back to the caller.""" + return f"echo: {request}" + + +if __name__ == "__main__": + print(handler("hello")) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt new file mode 100644 index 00000000000..b6ed8d63f87 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt @@ -0,0 +1,4 @@ +# Minimal dependency set so `azd ai agent init` (existing-code flow) detects a Python +# project. Keep this lightweight — the Tier 1 existing-code scenarios only scaffold +# and do not install or run the agent. +azure-ai-projects diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example new file mode 100644 index 00000000000..e20187f0c14 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example @@ -0,0 +1,27 @@ +# Per-developer / per-CI scenario profile — identifying values. +# +# Bootstrap: +# cp profile.local.yaml.example profile.local.yaml +# # then edit profile.local.yaml +# +# profile.local.yaml is gitignored. CI populates it from pipeline secrets. +# Any key set here overrides the same key in profile.yaml. + +# REQUIRED. Prefix applied to every newly created Azure resource (Foundry +# project/account, azd environment, agent, model deployment, resource group) +# so test resources are easy to identify and clean up. Use lowercase and +# hyphens — some fields normalize the value (sanitizeAgentName lowercases +# and replaces invalid characters), and a clean prefix avoids surprises. +# Concurrent runs of the same Tier 1 scenario automatically suffix this +# with -{instance} to keep parallel resource names distinct. +prefix: "your-alias" + +# REQUIRED. Display name of the Azure subscription to select when scenarios +# prompt for one. Tier 1 and Tier 2 scenarios both read this. +subscription: "azd ai agent development" + +# OPTIONAL. Tenant for `az login --tenant `. +# When unset, the driving agent omits the --tenant flag entirely and relies +# on your default tenant. Set this only if you actually need to scope to a +# specific tenant. +# tenant: "azdaiagent.onmicrosoft.com" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml new file mode 100644 index 00000000000..30ef5c004d4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml @@ -0,0 +1,25 @@ +# Repo-shared scenario profile — defaults for non-identifying values. +# +# This file is checked in. Identifying values (prefix, subscription, optional +# tenant) live in profile.local.yaml, which is gitignored and bootstrapped +# from profile.local.yaml.example. +# +# The driving agent loads both files (local overrides this one), validates +# that the required keys are present, derives `shared_agent_name` = +# "-", and then passes the merged map as +# `session_vars` to EVERY MCP tool call (load_scenario, run_pre_hooks, +# start_session, run_post_hooks) — see the "Profile / overrides" section +# of README.md. + +# Default Azure region for new resources created by Tier 1/2 scenarios. +region: "East US 2" + +# Default model deployment chosen during init / referenced by --model flags. +# gpt-4.1-mini is cheap and fast — appropriate for tests. +model: "gpt-4.1-mini" + +# Suffix appended to {prefix} to form the Tier 2 shared agent's name (and +# therefore the subdir name that `azd ai agent init` scaffolds into under +# ~/working/azd-agents-shared/). The shared name is exposed to scenarios +# as {shared_agent_name}. +shared_agent_suffix: "basic-responses"