From 16db9dd88e552258392eee4c3e1204de8edcc81c Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 2 Jun 2026 11:15:29 -0700 Subject: [PATCH 01/42] Add test scenarios for the cli-interactive-tester tool Signed-off-by: trangevi --- .../00-doctor-empty-dir.yaml | 18 ++ .../00-doctor-local-only.yaml | 17 ++ .../00-help-root.yaml | 11 + .../00-init-picker-navigation.yaml | 30 +++ .../00-init-validate-mutually-exclusive.yaml | 20 ++ .../00-init-validate-no-prompt-missing.yaml | 20 ++ .../00-sample-list-json-filters.yaml | 12 ++ .../00-sample-list-text.yaml | 10 + .../00-version.yaml | 10 + .../10-init-deploy-mode-code.yaml | 35 ++++ .../10-init-flags-agent-name-model.yaml | 29 +++ .../10-init-from-code.yaml | 33 +++ .../10-init-from-manifest-url.yaml | 29 +++ .../10-init-template-dotnet.yaml | 31 +++ .../10-init-template-python.yaml | 32 +++ .../20-setup-deploy-shared-agent.yaml | 43 ++++ .../21-show-json.yaml | 20 ++ .../21-show.yaml | 20 ++ .../22-invoke-input-file.yaml | 20 ++ .../22-invoke-new-session.yaml | 20 ++ .../22-invoke-remote.yaml | 20 ++ .../23-sessions-lifecycle.yaml | 23 ++ .../24-files-lifecycle.yaml | 28 +++ .../25-monitor-console.yaml | 21 ++ .../25-monitor-system.yaml | 21 ++ .../26-endpoint-update.yaml | 20 ++ .../27-run-local-and-invoke-local.yaml | 30 +++ .../28-eval-init-run-show.yaml | 23 ++ .../28-eval-update.yaml | 20 ++ .../29-optimize-submit-status.yaml | 21 ++ .../2A-doctor-provisioned-all-pass.yaml | 21 ++ .../2Z-teardown-down.yaml | 22 ++ .../README.md | 196 ++++++++++++++++++ .../fixtures/from-code/app.py | 17 ++ .../fixtures/from-code/requirements.txt | 4 + 35 files changed, 947 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml new file mode 100644 index 00000000000..8c64eec4fe7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -0,0 +1,18 @@ +# Tier 0 (offline) — `doctor` in an empty directory degrades gracefully. +name: "doctor-empty-dir" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-doctor-empty" + +# Guarantee an empty working dir so the "no azd project" path is exercised. +# start_session recreates the dir, so removing it is enough. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-empty" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor in a directory that has no azd project. Wait for the check report to render." + - "Confirm the command reports checks as skipped (or failed) with readable, non-crashing output — it should NOT panic or print a Go stack trace." + - "Note the exit code behavior described in help: 2 when all checks are skipped (preconditions unmet), 1 on failure, 0 when at least one passes." + - "Take a screenshot of the doctor report." + - "Report a finding if doctor crashes, prints a stack trace, or gives an unhelpful/confusing message for a missing project." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml new file mode 100644 index 00000000000..2f8ebd1a374 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -0,0 +1,17 @@ +# Tier 0 (offline) — `doctor --local-only` skips remote checks. +name: "doctor-local-only" +command: "azd ai agent doctor --local-only" +cwd: "~/working/azd-agents-doctor-empty" + +# Guarantee an empty working dir for a deterministic local-only run. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-empty" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor with --local-only. Wait for the report to render." + - "Confirm only local checks are attempted and remote/network-dependent checks are skipped (the report should indicate skipped remote checks)." + - "Confirm the run completes without attempting any network calls and without crashing." + - "Take a screenshot of the report." + - "Report a finding if remote checks still run, or if the output is confusing about what was skipped." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml new file mode 100644 index 00000000000..bd9db3b63bd --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml @@ -0,0 +1,11 @@ +# Tier 0 (offline) — verify root help lists every expected subcommand. +name: "help-root" +command: "azd ai agent --help" +cwd: "/tmp" + +goals: + - "Wait for the help output to render (it includes an ASCII-art banner and a 'Usage:' section)." + - "Confirm the 'Available Commands' list includes: doctor, endpoint, eval, files, init, invoke, monitor, optimize, run, sample, sessions, show, version." + - "Confirm the global flags are listed: --cwd/-C, --debug, --environment/-e, --no-prompt, --output/-o." + - "Take a screenshot of the help output." + - "Report a finding if any listed command is missing, if the usage text is malformed, or if the command exits non-zero." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml new file mode 100644 index 00000000000..1d4958cb13d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -0,0 +1,30 @@ +# Tier 0 (offline) — exercise the `init` interactive picker UX, then abort. +# +# This scenario probes the early interactive prompts only (init mode, language, +# template list). It intentionally ABORTS with Ctrl-C before reaching any Azure / +# Foundry project prompts, so it needs no Azure auth and creates no resources. +name: "init-picker-navigation" +command: "azd ai agent init" +cwd: "~/working/azd-agents-picker" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Clean dir so the abort-before-Azure check can't be confused by prior state. +pre: + - run: "rm -rf ~/working/azd-agents-picker" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the first prompt asking how to initialize (e.g. start from a template / use existing code / use a manifest)." + - "Select 'Start new from a template', then wait for the language prompt." + - "Select a language (Python), then wait for the template list prompt." + - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." + - "Type a string that matches nothing; confirm the list shows an empty/no-match state and does not crash." + - "Clear the filter (backspace) and confirm the full list returns." + - "Press Down many times past the end of the list; confirm selection stays bounded and does not error." + - "If a prompt mentions a '?' hint, press '?' and confirm a helpful hint is shown." + - "Take a screenshot at each interesting state." + - "Press Ctrl-C (or Escape at the top-level prompt) to abort. Confirm it exits cleanly without a stack trace and without leaving a half-written azure.yaml." + - "Report a finding for any picker glitch: filter not recovering, crash on no-match, unbounded scrolling, unhelpful hints, or a messy/abrupt abort." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml new file mode 100644 index 00000000000..1ce21ce14d8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -0,0 +1,20 @@ +# Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. +name: "init-validate-mutually-exclusive" +command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" +cwd: "~/working/azd-agents-validate" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so leftover files can't affect the validation result. +pre: + - run: "rm -rf ~/working/azd-agents-validate" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the command to fail fast (it should not start the interactive wizard)." + - "Confirm the error message clearly states that --from-code and --manifest/-m are mutually exclusive." + - "Confirm the process exits non-zero and does NOT create or modify any project files in the directory." + - "Take a screenshot of the error output." + - "Report a finding if the flags are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml new file mode 100644 index 00000000000..7843e58c31d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -0,0 +1,20 @@ +# Tier 0 (offline) — `init --no-prompt` with no resolvable inputs fails helpfully. +name: "init-validate-no-prompt-missing" +command: "azd ai agent init --no-prompt" +cwd: "~/working/azd-agents-validate-noprompt" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Empty dir is a precondition: no existing code/manifest to resolve from. +pre: + - run: "rm -rf ~/working/azd-agents-validate-noprompt" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run init in --no-prompt mode in an empty directory with no flags and no existing code/manifest." + - "Confirm the command does NOT hang waiting for input (no-prompt must never block on a prompt)." + - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --from-code, --manifest, or --project-id)." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, prompts interactively despite --no-prompt, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml new file mode 100644 index 00000000000..a3679f6ec8b --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — `sample list` JSON output and filter flags. +name: "sample-list-json-filters" +command: "bash" +cwd: "/tmp" + +goals: + - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." + - "Run: azd ai agent sample list --language python --output json. Confirm results are filtered to python samples only." + - "Run: azd ai agent sample list --type agent --output json. Confirm results only include agent-type templates." + - "Run: azd ai agent sample list --featured-only --output json. Confirm only featured samples are returned (a subset of the full list)." + - "Take a screenshot after each command." + - "Report a finding if any command produces invalid JSON, ignores its filter, or errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml new file mode 100644 index 00000000000..51b05c29be6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -0,0 +1,10 @@ +# Tier 0 (offline) — `sample list` renders the human-readable catalog. +name: "sample-list-text" +command: "azd ai agent sample list" +cwd: "/tmp" + +goals: + - "Wait for the curated sample catalog to render as human-readable text." + - "Confirm at least one sample entry is shown with a name/title and a manifest or repo reference usable with 'azd ai agent init -m' or 'azd init -t'." + - "Take a screenshot of the catalog output." + - "Report a finding if the list is empty, truncated, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml new file mode 100644 index 00000000000..a58a88b0076 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml @@ -0,0 +1,10 @@ +# Tier 0 (offline) — verify `azd ai agent version` prints a version string. +name: "version" +command: "azd ai agent version" +cwd: "/tmp" + +goals: + - "Wait for the command to print a version string (e.g. a value like '0.1.x-preview' or a commit-based 'vdev' build identifier)." + - "Confirm the process exits cleanly without an error or stack trace." + - "Take a screenshot of the final output." + - "Report a finding if no version is printed, if the output is empty, or if the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml new file mode 100644 index 00000000000..647a7b34675 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -0,0 +1,35 @@ +# Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Targets the --deploy-mode code path which prompts for entry-point and runtime +# (instead of building a container image). +name: "init-deploy-mode-code" +command: "azd ai agent init --from-code --deploy-mode code" +cwd: "~/working/azd-agents-t1-code-deploy" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a committed Python fixture so code-deploy has real source to package. +# Override the fixture location with AZD_AGENTS_FIXTURES if needed. +pre: + - run: "rm -rf ~/working/azd-agents-t1-code-deploy" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." + - "If an existing agent manifest is detected, confirm reuse." + - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." + - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the entry-point or runtime prompts are missing, confusing, or not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml new file mode 100644 index 00000000000..36f15bdaf00 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -0,0 +1,29 @@ +# Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Verifies that the override flags are honored in the generated files. +name: "init-flags-agent-name-model" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name qa-named-agent --model gpt-4.1-mini" +cwd: "~/working/azd-agents-t1-flags" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-flags" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the manifest to download and parse." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "Accept any remaining model defaults (version, SKU, capacity, deployment name)." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name 'qa-named-agent' and the model 'gpt-4.1-mini' (the values passed via flags, not the manifest defaults)." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml new file mode 100644 index 00000000000..cebe8f71fc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -0,0 +1,33 @@ +# Tier 1 (auth, scaffold only) — init from existing code in the current directory. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Precondition: the cwd should already contain agent source code (and ideally an +# agent manifest). The pre hooks seed a committed Python fixture so this is +# guaranteed and the run is idempotent. Override the fixture location with +# AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. +name: "init-from-code" +command: "azd ai agent init --from-code" +cwd: "~/working/azd-agents-t1-from-code" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-t1-from-code" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-from-code && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "Wait for the tool to inspect the current directory and treat its code as the agent source." + - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' in the output." + - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if code-detection misbehaves or if the manifest-reuse prompt is confusing." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml new file mode 100644 index 00000000000..7f790eb49a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -0,0 +1,29 @@ +# Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +name: "init-from-manifest-url" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" +cwd: "~/working/azd-agents-t1-manifest" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-manifest" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the tool to fetch and parse the manifest from the provided URL." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity, deployment name)." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the manifest fails to download/parse, or if any field is dropped during scaffold." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml new file mode 100644 index 00000000000..a1f2c0487b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -0,0 +1,31 @@ +# Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +name: "init-template-dotnet" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-dotnet" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-dotnet" + cwd: "~/working" + name: "reset working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select C# / .NET as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the .NET language path behaves differently from Python in any confusing way." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml new file mode 100644 index 00000000000..d3614385d6f --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -0,0 +1,32 @@ +# Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. +# +# Requires `azd auth login`. Reads subscriptions/Foundry projects but does NOT +# run `azd provision`, so no resources are created and no cost is incurred. +name: "init-template-python" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-python" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-python" + cwd: "~/working" + name: "reset working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding for any confusing prompt, wrong default, or scaffold problem." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml new file mode 100644 index 00000000000..4ceb20913d4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -0,0 +1,43 @@ +# Tier 2 (cloud E2E) — SETUP: deploy the shared agent used by all 21-..2A scenarios. +# +# ⚠️ Incurs Azure cost. Run this FIRST. The deployed agent lives in the shared +# working directory ~/working/azd-agents-shared and is reused by the targeted +# scenarios. Run 2Z-teardown-down.yaml LAST to clean up. +name: "setup-deploy-shared-agent" +command: "azd ai agent init" +cwd: "~/working/azd-agents-shared" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Idempotent setup: if a previous run left a project here, tear down its Azure +# resources FIRST (so we never orphan them), then clear the dir for a fresh +# deploy. The down step gets a long timeout and continues on error (there may be +# nothing to tear down). Re-using a clean path also avoids the resource-name hash +# collision in issue #8360. +pre: + - run: "if [ -f ~/working/azd-agents-shared/azure.yaml ]; then (cd ~/working/azd-agents-shared && azd down --force --purge); fi" + cwd: "~/working/azd-agents-shared" + name: "tear down any leftover deployed agent" + continue_on_error: true + timeout: 900 + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Select the 'Basic Responses' template from the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." + - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." + - "Take a screenshot of the successful provision and 'show' output." + - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml new file mode 100644 index 00000000000..0f9dc491c3a --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `show --output json` returns well-formed JSON. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show-json" +command: "azd ai agent show --output json" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the command to print JSON." + - "Confirm the output is valid JSON and includes the agent name, version, status, and endpoint fields." + - "Confirm the JSON values match what the table form of 'show' reports." + - "Take a screenshot of the JSON output." + - "Report a finding if the JSON is malformed, missing expected fields, or inconsistent with the table output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml new file mode 100644 index 00000000000..fb06d2ff454 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `show` reports the deployed agent (table output). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show" +command: "azd ai agent show" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the status table to render." + - "Confirm the agent name, version, and a status field (e.g. 'active') are shown, auto-resolved from azure.yaml and the current azd environment." + - "Confirm the endpoint URL for the deployed agent is displayed." + - "Take a screenshot of the status table." + - "Report a finding if the agent cannot be resolved, the status looks wrong, or fields are missing/blank." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml new file mode 100644 index 00000000000..79ab211dbcb --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke -f ` sends a file as the request body. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-input-file" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Create a small request file, e.g.: printf '%s' 'Summarize the benefits of unit testing in one sentence.' > request.txt" + - "Run: azd ai agent invoke -f request.txt and wait for the response." + - "Confirm the agent responds to the contents of the file (not to an empty or literal-path message)." + - "Take a screenshot of the response." + - "Report a finding if the file body is ignored, mis-read, or if -f errors on a valid file." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml new file mode 100644 index 00000000000..ce93cf77620 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke --new-session` resets conversation state. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-new-session" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent invoke \"My name is Quinn. Remember it.\" and wait for a response." + - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session reused the same conversation)." + - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent does NOT recall 'Quinn' — the new session discarded prior history." + - "Take a screenshot after each invoke." + - "Report a finding if --new-session still carries over prior conversation context, or if session persistence between the first two invokes does not work." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml new file mode 100644 index 00000000000..9fedf175a0c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke` a deployed agent remotely on Foundry. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-remote" +command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the remote invocation to complete (the agent is auto-detected from azure.yaml)." + - "Confirm a non-empty model response is returned (not an error)." + - "Confirm the command does NOT 404 and the response references no auth/endpoint errors." + - "Take a screenshot of the response." + - "Report a finding if invoke returns a 404, an auth error, an empty response, or hangs past a reasonable timeout." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml new file mode 100644 index 00000000000..f559df8cc08 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — `sessions` lifecycle: create, list, show, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `sessions` command group end-to-end in one run. +name: "sessions-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent sessions create. Confirm a new session is created and note its session ID." + - "Run: azd ai agent sessions list --output table. Confirm the newly created session appears in the list." + - "Run: azd ai agent sessions show using the ID from create. Confirm session details (status, version) are shown." + - "Run: azd ai agent sessions delete . Confirm the session is deleted (synchronously)." + - "Run: azd ai agent sessions list again and confirm the deleted session no longer appears." + - "Take a screenshot after each step." + - "Report a finding if any subcommand errors, if the created session is missing from list, or if delete does not remove it." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml new file mode 100644 index 00000000000..4356ab7a54d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -0,0 +1,28 @@ +# Tier 2 (cloud E2E) — `files` lifecycle: upload, list, stat, mkdir, download, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `files` command group end-to-end against the last invoke session. +# Note: file operations target a session — run an invoke first if no session exists. +name: "files-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure a session exists: run 'azd ai agent invoke \"hi\"' first so files commands have a session to target (or pass --session-id consistently)." + - "Create a local test file: printf 'col1,col2\\n1,2\\n' > sample.csv" + - "Run: azd ai agent files upload ./sample.csv --target-path /data/sample.csv. Confirm success." + - "Run: azd ai agent files list /data --output table. Confirm sample.csv is listed." + - "Run: azd ai agent files stat /data/sample.csv. Confirm metadata (size/type) is returned." + - "Run: azd ai agent files mkdir /data/output. Confirm the directory is created." + - "Run: azd ai agent files download /data/sample.csv --target-path ./downloaded.csv. Confirm the file downloads and its contents match the original." + - "Run: azd ai agent files delete /data/output --recursive and azd ai agent files delete /data/sample.csv. Confirm both are removed." + - "Run: azd ai agent files list /data and confirm the deleted entries are gone." + - "Take a screenshot after each step." + - "Report a finding if any subcommand errors, if upload/download corrupts the file, or if a session cannot be resolved." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml new file mode 100644 index 00000000000..d3288f76781 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `monitor` console logs for the last session. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists to stream logs from. +name: "monitor-console" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure recent activity: run 'azd ai agent invoke \"generate some output\"' so the session has console logs." + - "Run: azd ai agent monitor --tail 50. Confirm recent console (stdout/stderr) log lines are fetched and printed, then the command exits (no --follow)." + - "Confirm timestamps are shown in local time by default and the output is readable." + - "Take a screenshot of the log output." + - "Report a finding if no session can be auto-resolved, if logs are empty despite recent activity, or if the output is garbled." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml new file mode 100644 index 00000000000..08c182ab1d3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `monitor --type system` container/system events. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists. +name: "monitor-system" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent monitor --type system --tail 50. Confirm system/container event logs are fetched and printed." + - "Confirm the output is distinct from console logs (system events such as container lifecycle, not stdout/stderr)." + - "Optionally run: azd ai agent monitor --type system --follow for a few seconds to confirm streaming works, then stop it with Ctrl-C and confirm it exits cleanly." + - "Take a screenshot of the event output." + - "Report a finding if system logs are empty, indistinguishable from console logs, or if --follow does not stream / does not stop cleanly." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml new file mode 100644 index 00000000000..8cc712f582c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `endpoint update` patches endpoint/card without a new version. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "endpoint-update" +command: "azd ai agent endpoint update" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run endpoint update for the default (auto-detected) agent service." + - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." + - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." + - "Take a screenshot of the update result and the post-update 'show' output." + - "Report a finding if endpoint update creates a new version, errors against an already-deployed agent, or gives a confusing result." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml new file mode 100644 index 00000000000..f27e7b4267c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -0,0 +1,30 @@ +# Tier 2 (cloud E2E) — run the agent locally and invoke it via --local. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run (project scaffolded). +# `azd ai agent run` blocks the terminal, so this needs TWO sessions: one runs the +# agent, a second invokes it with --local. +name: "run-local-and-invoke-local" +command: "azd ai agent run --no-inspector" +cwd: "~/working/azd-agents-shared" + +notes: | + Use two tester sessions: + - session_id "run": runs 'azd ai agent run --no-inspector' (this 'command'). + - session_id "invoke": a bash session for 'azd ai agent invoke --local'. + The agent listens on port 8088 by default. Stop the run session with Ctrl-C at the end. + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "In the run session, wait until the local agent reports it is listening (look for a port 8088 / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." + - "Start a SECOND session (session_id 'invoke') running bash in the same cwd." + - "In the invoke session, run: azd ai agent invoke --local \"Hello from local!\" and wait for a response from the locally running agent." + - "Confirm the response comes from the local process (not Foundry) and is non-empty." + - "Take a screenshot of both the run session (showing it listening) and the invoke response." + - "Stop the run session with Ctrl-C and confirm the local agent shuts down cleanly." + - "Report a finding if the local server fails to start, if --local invoke cannot reach it, or if shutdown is messy." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml new file mode 100644 index 00000000000..df9dfcd33ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — `eval` flow: init a suite, run it, then list/show results. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully +# (a deployed agent is required to generate and run evals). +name: "eval-init-run-show" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --eval-model gpt-4.1-mini --max-samples 15 and wait for it to generate an eval config and dataset." + - "Confirm an eval.yaml (and dataset) is written to the project root." + - "Run: azd ai agent eval run and wait for the evaluation run to complete." + - "Run: azd ai agent eval list and confirm the run appears." + - "Run: azd ai agent eval show and confirm run details / scores are displayed." + - "Take a screenshot after init, run, and show." + - "Report a finding if generation fails, eval.yaml is malformed, the run errors, or results are missing/unreadable." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml new file mode 100644 index 00000000000..7101c392f82 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `eval update` refreshes dataset and evaluators. +# +# Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. +name: "eval-update" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Confirm an eval.yaml from a prior 'eval init' exists in the project root (run 28-eval-init-run-show.yaml first if not)." + - "Run: azd ai agent eval update --dataset-only and confirm only the dataset is updated (evaluators left unchanged)." + - "Run: azd ai agent eval update --evaluator-only and confirm only the evaluators are updated (dataset left unchanged)." + - "Take a screenshot after each update." + - "Report a finding if --dataset-only or --evaluator-only updates the wrong thing, errors, or prompts despite the scoping flag." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml new file mode 100644 index 00000000000..1c53ded1649 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `optimize` submit (no-wait) then check status/list. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "optimize-submit-status" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent optimize --eval-model gpt-4.1-mini --max-iterations 1 --no-wait (agent auto-detected from the azd project). Provide any other values it prompts for using sensible defaults." + - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." + - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." + - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." + - "Take a screenshot after submit, status, and list." + - "Report a finding if submission fails, the job ID is not surfaced, or status/list cannot find the job. (Optional: cancel with 'azd ai agent optimize cancel ' to avoid leaving it running.)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml new file mode 100644 index 00000000000..cffe0288a63 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `doctor` against a fully provisioned project (checks pass). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "doctor-provisioned-all-pass" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run doctor against the provisioned project. Wait for the full check suite (local + remote) to render." + - "Confirm the local checks pass (project, azure.yaml, agent service config)." + - "Confirm the remote checks pass (deployed agent reachable / active)." + - "Confirm doctor suggests a sensible next command (e.g. invoke or run) and exits 0 (at least one check passed, none failed)." + - "Take a screenshot of the doctor report." + - "Report a finding if any check fails or is skipped unexpectedly for a healthy provisioned project, or if the suggested next step is wrong." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml new file mode 100644 index 00000000000..bcad6af5fc7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — TEARDOWN: destroy all resources created by the setup scenario. +# +# ⚠️ Run this LAST, after all 21-..2A targeted scenarios are done. Cleans up the +# shared agent and its Azure resources to stop incurring cost. +name: "teardown-down" +command: "azd down --force --purge" +cwd: "~/working/azd-agents-shared" + +goals: + - "Run 'azd down --force --purge' in the shared project directory." + - "Wait for the full teardown to complete — all resources (Foundry account/project, model deployment, container resources) should be deleted and purged." + - "Confirm the command reports success and no resources are left behind." + - "Take a screenshot of the completed teardown." + - "Report a finding if teardown fails, leaves orphaned resources, or hangs." + +# After the in-session teardown, clear the shared working dir so the next full +# Tier 2 pass starts clean. continue_on_error so a missing dir is not fatal. +post: + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md new file mode 100644 index 00000000000..895a892d7db --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -0,0 +1,196 @@ + +# `azd ai agent` — cli-interactive-tester scenarios + +Goal-based scenarios for driving the `azure.ai.agents` extension through the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server. Each file targets **one** command or flow at a time and uses the +strict `goals:` list format so the run is repeatable and reviewable. + +## How to run + +Register the cli-interactive-tester MCP server (see its README), then ask +Copilot CLI to load a scenario and accomplish its goals, e.g.: + +``` +Use the cli-interactive-tester to load the scenario at +tests/cli-interactive-tester-scenarios/00-version.yaml. If it declares pre hooks, +run them first; then start the session, accomplish the goals, take screenshots at +each step, and run any post hooks after finishing. +``` + +Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting +the working dir or seeding a fixture), and a few declare **`post:` hooks** +(cleanup). The agent must invoke them via the tester's `run_pre_hooks` / +`run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. +See [Pre/post hooks](#prepost-hooks) below. + +## Paths run inside WSL (on Windows) + +The cli-interactive-tester drives CLIs through **tmux**, which on Windows runs +inside **WSL**. The scenario YAML files live on the Windows filesystem (in this +repo), but every `cwd` value is resolved against the **WSL filesystem** where the +command actually executes: + +- `~/working/azd-agents-shared` → `/home//working/azd-agents-shared` +- `/tmp` → WSL's `/tmp` + +Implications: + +- `azd` and the `azure.ai.agents` extension must be installed **inside WSL**, + since that is where the scenario commands run. +- `cwd` directories do not need to pre-exist — the tester creates them if missing. +- The `cwd` convention is three-way by design: ephemeral `/tmp` for read-only + scenarios that touch no project (`version`, `--help`, `sample list`); a unique + `~/working/azd-agents-*` dir per `init`/`doctor` scenario for isolation; and a + single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they + operate on the same deployed agent. + +On macOS/Linux these are simply native paths (no WSL involved). + +## Tiers + +Scenarios are organized into three tiers by cost and prerequisites. + +### Tier 0 — Offline (prefix `00-`) +No Azure auth, no network resource creation. Fast and deterministic. Safe to run +in any order, any time. + +| File | Targets | +|------|---------| +| `00-version.yaml` | `version` | +| `00-help-root.yaml` | root help / command discovery | +| `00-sample-list-text.yaml` | `sample list` (text) | +| `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | +| `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | +| `00-doctor-local-only.yaml` | `doctor --local-only` | +| `00-init-validate-mutually-exclusive.yaml` | `init` flag validation (`--from-code` + `-m`) | +| `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | +| `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | + +### Tier 1 — Auth, scaffold only (prefix `10-`) +Requires `azd auth login` (reads subscriptions/Foundry projects) but **does not +provision** any resources and incurs no cost. Each completes a project scaffold +and verifies the generated files, then stops before `azd provision`. + +| File | Targets | +|------|---------| +| `10-init-template-python.yaml` | `init` new-from-template, Python | +| `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | +| `10-init-from-manifest-url.yaml` | `init -m ` | +| `10-init-from-code.yaml` | `init --from-code` | +| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` | +| `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | + +### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost +Provisions real resources. **Run order matters:** + +1. `20-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. +2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). +3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. + +All Tier 2 scenarios share one working directory (`~/working/azd-agents-shared`) +so they operate on the same deployed agent. + +| File | Targets | +|------|---------| +| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` (SETUP) | +| `21-show.yaml` | `show` (table) | +| `21-show-json.yaml` | `show --output json` | +| `22-invoke-remote.yaml` | `invoke` (remote) | +| `22-invoke-new-session.yaml` | `invoke --new-session` | +| `22-invoke-input-file.yaml` | `invoke -f ` | +| `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | +| `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | +| `25-monitor-console.yaml` | `monitor` (console) | +| `25-monitor-system.yaml` | `monitor --type system` | +| `26-endpoint-update.yaml` | `endpoint update` | +| `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | +| `28-eval-init-run-show.yaml` | `eval init/run/list/show` | +| `28-eval-update.yaml` | `eval update` | +| `29-optimize-submit-status.yaml` | `optimize` + `optimize status/list` | +| `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | +| `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | + +## Conventions + +- **Subscription**: `azd ai agent development` +- **Region**: `East US 2` +- **Model**: `gpt-4.1-mini` (cheap/fast for testing) +- `command:` invokes the installed extension as `azd ai agent …`. +- Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent + auto-detection prompts. +- Every scenario asks the driver to screenshot key steps and file a finding + (`report_finding`) for any confusing UX, error, or doc mismatch. + +## Pre/post hooks + +Scenarios use the tester's **`pre:`** and **`post:`** hook lists for host-side +setup and cleanup. Hooks run on the host (inside WSL on Windows), outside the +tmux session, **sequentially and fail-fast** unless a hook sets +`continue_on_error: true`. Each entry is a string or a mapping with `run` +(required), `cwd` (defaults to the scenario `cwd`, created if missing), `env`, +`continue_on_error` (default `false`), `timeout` (default **120s**), and `name`. + +How they're used here: + +- **`pre` reset** — stateful Tier 0/1 scenarios `rm -rf` their own working dir so + re-runs start clean. (`start_session` recreates the dir, so removing it is + enough; the doctor/init scenarios just need an empty dir.) +- **`pre` fixture seed** — the `--from-code` scenarios + (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python + fixture into the dir so the source exists before `init --from-code` inspects it + (see [Fixtures](#fixtures)). +- **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs + `azd down --force --purge` if a leftover project exists in the shared dir (so it + never orphans live Azure resources), then clears the dir. The down hook uses + `timeout: 900` and `continue_on_error: true`. +- **`pre` precondition guard (Tier 2 reuse)** — `21-…2A` print a clear "run + 20-setup first" warning if the shared agent isn't deployed (non-fatal). +- **`post` cleanup** — `2Z-teardown-down` clears the shared working dir after the + in-session `azd down` completes. + +## Fixtures + +`fixtures/from-code/` holds a minimal Python agent source tree (`app.py` + +`requirements.txt`) that satisfies the extension's `--from-code` detection +(it looks for `requirements.txt` or any `.py`, and defaults the entry point to +`app.py`). The from-code scenarios copy it into the working dir via a `pre` hook. + +The hook references the fixture by absolute path with an overridable env var: + +```sh +cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/." "$cwd" +``` + +If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view +of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this +`fixtures/` directory before running the from-code scenarios. + +## Re-running scenarios (idempotency) + +Idempotency is handled **per scenario** via `pre`/`post` hooks rather than a +separate reset step — every scenario that holds state resets itself, so they can +be run back to back in any order within a tier: + +- Tier 0/1 stateful scenarios **pre-wipe** their own `cwd`. Cleanup is pre-wipe + **only** (no `post` delete), so the generated scaffold stays on disk for + inspection after a run while the next run still starts clean. +- The shared Tier 2 dir is reset by `20-setup`'s `pre` hook, which **downs any + leftover deployed project first** to avoid orphaning live Azure resources (this + also sidesteps the resource-name hash collision behind + [#8360](https://github.com/Azure/azure-dev/issues/8360)). `2Z-teardown-down` + additionally clears the dir in a `post` hook. +- Read-only scenarios (`version`, `--help`, `sample list`) run in `/tmp`, hold no + state, and declare no hooks. + +> If a Tier 2 run is interrupted before `2Z-teardown`, just re-run +> `20-setup-deploy-shared-agent` — its `pre` hook downs any live project in the +> shared dir before redeploying, so resources won't be orphaned. + +## Notes + +- `files` and `sessions` are exercised as one lifecycle scenario per command + group (rather than one file per subcommand) to avoid cross-scenario ordering + dependencies — still one command at a time. +- `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` + uses two sessions (one to run, one to invoke `--local`). diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py new file mode 100644 index 00000000000..56b577b6277 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py @@ -0,0 +1,17 @@ +"""Minimal agent source fixture for `azd ai agent init --from-code` scenarios. + +This file exists only so the extension's from-code detection treats the working +directory as a Python agent project (it looks for requirements.txt or any .py +file, and uses app.py as the default entry point). The init flow scaffolds an +agent.yaml around this code; the body does not need to be a fully functional +agent for the scaffold-only Tier 1 scenarios. +""" + + +def handler(request: str) -> str: + """Echo the incoming request back to the caller.""" + return f"echo: {request}" + + +if __name__ == "__main__": + print(handler("hello")) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt new file mode 100644 index 00000000000..d94c3d032d6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt @@ -0,0 +1,4 @@ +# Minimal dependency set so `azd ai agent init --from-code` detects a Python +# project. Keep this lightweight — the Tier 1 from-code scenarios only scaffold +# and do not install or run the agent. +azure-ai-projects From 4fbaa67e4bc714125e792205230aa76424d1251a Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 2 Jun 2026 14:14:40 -0700 Subject: [PATCH 02/42] Some scenario edits Signed-off-by: trangevi --- .../10-init-deploy-mode-code.yaml | 5 ++- .../10-init-flags-agent-name-model.yaml | 11 +++--- .../10-init-from-code.yaml | 5 ++- .../10-init-from-manifest-url.yaml | 7 ++-- .../10-init-template-dotnet.yaml | 7 ++-- .../10-init-template-python.yaml | 7 ++-- .../20-setup-deploy-shared-agent.yaml | 5 ++- .../README.md | 38 ++++++++++++++++++- 8 files changed, 63 insertions(+), 22 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 647a7b34675..9b7d1d0fbf4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime # (instead of building a container image). name: "init-deploy-mode-code" @@ -21,11 +21,12 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 36f15bdaf00..df3792cd44a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,9 +1,9 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name qa-named-agent --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent --model gpt-4.1-mini" cwd: "~/working/azd-agents-t1-flags" env: @@ -16,14 +16,15 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - - "Accept any remaining model defaults (version, SKU, capacity, deployment name)." + - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name 'qa-named-agent' and the model 'gpt-4.1-mini' (the values passed via flags, not the manifest defaults)." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index cebe8f71fc3..4f8f4fd9db7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from existing code in the current directory. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Precondition: the cwd should already contain agent source code (and ideally an # agent manifest). The pre hooks seed a committed Python fixture so this is # guaranteed and the run is idempotent. Override the fixture location with @@ -21,9 +21,10 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 7f790eb49a7..95c9a84d0d2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest" @@ -15,13 +15,14 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." - - "Accept the defaults for any remaining model prompts (version, SKU, capacity, deployment name)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index a1f2c0487b3..901f0c53670 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet" @@ -15,15 +15,16 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index d3614385d6f..c3a035828c5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. # -# Requires `azd auth login`. Reads subscriptions/Foundry projects but does NOT +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Reads subscriptions/Foundry projects but does NOT # run `azd provision`, so no resources are created and no cost is incurred. name: "init-template-python" command: "azd ai agent init" @@ -16,15 +16,16 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 4ceb20913d4..f18400d226e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -26,15 +26,16 @@ pre: name: "clear the shared working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 895a892d7db..a609d8d4b23 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -1,4 +1,4 @@ - + # `azd ai agent` — cli-interactive-tester scenarios Goal-based scenarios for driving the `azure.ai.agents` extension through the @@ -47,6 +47,34 @@ Implications: On macOS/Linux these are simply native paths (no WSL involved). +## Authentication + +Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in +manually before** starting a run. The scenarios do **not** perform login +themselves, and the test-driving agent **cannot** complete it either: `az login` +opens a **separate browser window** for account selection that requires +human interaction outside the terminal the agent controls. Treat auth as a +one-time manual prerequisite, not a scenario step. + +Inside WSL, a human runs: + +``` +az login --tenant azdaiagent.onmicrosoft.com +``` + +This opens the interactive sign-in flow and then: + +1. **Browser account selection** — a separate browser window opens; the human + picks the account in the `azdaiagent.onmicrosoft.com` tenant. (The agent + cannot do this.) +2. **Subscription selection** — back in the terminal, select the + `azd ai agent development` subscription. + +Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL +session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of +them reuse that session credential. + + ## Tiers Scenarios are organized into three tiers by cost and prerequisites. @@ -68,7 +96,7 @@ in any order, any time. | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | ### Tier 1 — Auth, scaffold only (prefix `10-`) -Requires `azd auth login` (reads subscriptions/Foundry projects) but **does not +Requires Azure login (reads subscriptions/Foundry projects) but **does not provision** any resources and incurs no cost. Each completes a project scaffold and verifies the generated files, then stops before `azd provision`. @@ -116,6 +144,12 @@ so they operate on the same deployed agent. - **Subscription**: `azd ai agent development` - **Region**: `East US 2` - **Model**: `gpt-4.1-mini` (cheap/fast for testing) +- **Resource name prefix**: every newly created Azure resource (Foundry + project/account, azd environment, agent, model deployment, resource group) is + named with a `trangevi-` prefix so test resources are easy to identify and + clean up. Note that some fields lowercase the value and replace invalid + characters with hyphens — that normalization is expected (see + `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. From 61fc535a1bd21a3cd7e4aa27377e20c64400622e Mon Sep 17 00:00:00 2001 From: trangevi Date: Wed, 3 Jun 2026 09:41:48 -0700 Subject: [PATCH 03/42] Picking up recent tester tool updates Signed-off-by: trangevi --- .../.gitignore | 2 + .../00-doctor-empty-dir.yaml | 4 +- .../00-doctor-local-only.yaml | 4 +- .../00-init-picker-navigation.yaml | 4 +- .../00-init-validate-mutually-exclusive.yaml | 4 +- .../00-init-validate-no-prompt-missing.yaml | 4 +- .../10-init-deploy-mode-code.yaml | 8 +-- .../10-init-flags-agent-name-model.yaml | 10 +-- .../10-init-from-code.yaml | 8 +-- .../10-init-from-manifest-url.yaml | 6 +- .../10-init-template-dotnet.yaml | 6 +- .../10-init-template-python.yaml | 6 +- .../27-run-local-and-invoke-local.yaml | 26 ++++--- .../README.md | 68 +++++++++++++++++-- 14 files changed, 112 insertions(+), 48 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore new file mode 100644 index 00000000000..7fde892bf70 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore @@ -0,0 +1,2 @@ +# cli-interactive-tester run artifacts (screenshots, HTML reports, scrollback) +.reports/ diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml index 8c64eec4fe7..53aba1a8dbc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -1,12 +1,12 @@ # Tier 0 (offline) — `doctor` in an empty directory degrades gracefully. name: "doctor-empty-dir" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-doctor-empty" +cwd: "~/working/azd-agents-doctor-empty-{instance}" # Guarantee an empty working dir so the "no azd project" path is exercised. # start_session recreates the dir, so removing it is enough. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty" + - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 2f8ebd1a374..7c8662e6e3c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -1,11 +1,11 @@ # Tier 0 (offline) — `doctor --local-only` skips remote checks. name: "doctor-local-only" command: "azd ai agent doctor --local-only" -cwd: "~/working/azd-agents-doctor-empty" +cwd: "~/working/azd-agents-doctor-empty-{instance}" # Guarantee an empty working dir for a deterministic local-only run. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty" + - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index 1d4958cb13d..bbf2c0c7fd5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -5,14 +5,14 @@ # Foundry project prompts, so it needs no Azure auth and creates no resources. name: "init-picker-navigation" command: "azd ai agent init" -cwd: "~/working/azd-agents-picker" +cwd: "~/working/azd-agents-picker-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Clean dir so the abort-before-Azure check can't be confused by prior state. pre: - - run: "rm -rf ~/working/azd-agents-picker" + - run: "rm -rf ~/working/azd-agents-picker-{instance}" cwd: "~/working" name: "reset working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 1ce21ce14d8..6ee69d98636 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -1,14 +1,14 @@ # Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. name: "init-validate-mutually-exclusive" command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" -cwd: "~/working/azd-agents-validate" +cwd: "~/working/azd-agents-validate-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so leftover files can't affect the validation result. pre: - - run: "rm -rf ~/working/azd-agents-validate" + - run: "rm -rf ~/working/azd-agents-validate-{instance}" cwd: "~/working" name: "reset working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index 7843e58c31d..ac31b1f2127 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -1,14 +1,14 @@ # Tier 0 (offline) — `init --no-prompt` with no resolvable inputs fails helpfully. name: "init-validate-no-prompt-missing" command: "azd ai agent init --no-prompt" -cwd: "~/working/azd-agents-validate-noprompt" +cwd: "~/working/azd-agents-validate-noprompt-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Empty dir is a precondition: no existing code/manifest to resolve from. pre: - - run: "rm -rf ~/working/azd-agents-validate-noprompt" + - run: "rm -rf ~/working/azd-agents-validate-noprompt-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 9b7d1d0fbf4..89260d57ba8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -5,7 +5,7 @@ # (instead of building a container image). name: "init-deploy-mode-code" command: "azd ai agent init --from-code --deploy-mode code" -cwd: "~/working/azd-agents-t1-code-deploy" +cwd: "~/working/azd-agents-t1-code-deploy-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" @@ -13,15 +13,15 @@ env: # Seed a committed Python fixture so code-deploy has real source to package. # Override the fixture location with AZD_AGENTS_FIXTURES if needed. pre: - - run: "rm -rf ~/working/azd-agents-t1-code-deploy" + - run: "rm -rf ~/working/azd-agents-t1-code-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-code-deploy && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy/" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index df3792cd44a..5c500ef9fdc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -3,20 +3,20 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent --model gpt-4.1-mini" -cwd: "~/working/azd-agents-t1-flags" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +cwd: "~/working/azd-agents-t1-flags-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-flags" + - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." @@ -25,6 +25,6 @@ goals: - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 4f8f4fd9db7..01026e394da 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -7,21 +7,21 @@ # AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. name: "init-from-code" command: "azd ai agent init --from-code" -cwd: "~/working/azd-agents-t1-from-code" +cwd: "~/working/azd-agents-t1-from-code-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" pre: - - run: "rm -rf ~/working/azd-agents-t1-from-code" + - run: "rm -rf ~/working/azd-agents-t1-from-code-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-from-code && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code/" + - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 95c9a84d0d2..6d5b8d38252 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -3,19 +3,19 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" -cwd: "~/working/azd-agents-t1-manifest" +cwd: "~/working/azd-agents-t1-manifest-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-manifest" + - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 901f0c53670..9c915254688 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -3,19 +3,19 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-template-dotnet" command: "azd ai agent init" -cwd: "~/working/azd-agents-t1-dotnet" +cwd: "~/working/azd-agents-t1-dotnet-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-dotnet" + - run: "rm -rf ~/working/azd-agents-t1-dotnet-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index c3a035828c5..d9b41930c1d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -4,19 +4,19 @@ # run `azd provision`, so no resources are created and no cost is incurred. name: "init-template-python" command: "azd ai agent init" -cwd: "~/working/azd-agents-t1-python" +cwd: "~/working/azd-agents-t1-python-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-python" + - run: "rm -rf ~/working/azd-agents-t1-python-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index f27e7b4267c..45a35130019 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -4,14 +4,20 @@ # `azd ai agent run` blocks the terminal, so this needs TWO sessions: one runs the # agent, a second invokes it with --local. name: "run-local-and-invoke-local" -command: "azd ai agent run --no-inspector" +command: "azd ai agent run --port {agent} --no-inspector" cwd: "~/working/azd-agents-shared" +# Reserve a free port per scenario run so parallel local runs don't collide on +# the default 8088, and so the run + invoke sessions find each other (a pool is +# shared across every start_session that passes the same scenario_path). +allocate_ports: [agent] + notes: | - Use two tester sessions: - - session_id "run": runs 'azd ai agent run --no-inspector' (this 'command'). - - session_id "invoke": a bash session for 'azd ai agent invoke --local'. - The agent listens on port 8088 by default. Stop the run session with Ctrl-C at the end. + Use two tester sessions that share this scenario's port pool — start BOTH with + the same scenario_path (and the same instance_id if running in parallel): + - session_id "run-{instance}": runs this 'command' (azd ai agent run --port {agent} --no-inspector). + - session_id "invoke-{instance}": a bash session for 'azd ai agent invoke --local --port {agent}'. + The agent listens on the allocated port {agent}. Stop the run session with Ctrl-C at the end. # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: @@ -21,10 +27,10 @@ pre: continue_on_error: true goals: - - "In the run session, wait until the local agent reports it is listening (look for a port 8088 / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." - - "Start a SECOND session (session_id 'invoke') running bash in the same cwd." - - "In the invoke session, run: azd ai agent invoke --local \"Hello from local!\" and wait for a response from the locally running agent." + - "In the run session, wait until the local agent reports it is listening (look for a port {agent} / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." + - "Start a SECOND session (session_id 'invoke-{instance}', same scenario_path, same instance_id) running bash in the same cwd so it shares this scenario's allocated port {agent}." + - "In the invoke session, run: azd ai agent invoke --local --port {agent} \"Hello from local!\" and wait for a response from the locally running agent." - "Confirm the response comes from the local process (not Foundry) and is non-empty." - - "Take a screenshot of both the run session (showing it listening) and the invoke response." + - "Take a screenshot of both the run session (showing it listening on {agent}) and the invoke response." - "Stop the run session with Ctrl-C and confirm the local agent shuts down cleanly." - - "Report a finding if the local server fails to start, if --local invoke cannot reach it, or if shutdown is messy." + - "Report a finding if the local server fails to start, if --local invoke cannot reach it on port {agent}, or if shutdown is messy." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index a609d8d4b23..60a6cb3cf6b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -41,7 +41,9 @@ Implications: - `cwd` directories do not need to pre-exist — the tester creates them if missing. - The `cwd` convention is three-way by design: ephemeral `/tmp` for read-only scenarios that touch no project (`version`, `--help`, `sample list`); a unique - `~/working/azd-agents-*` dir per `init`/`doctor` scenario for isolation; and a + `~/working/azd-agents-*-{instance}` dir per `init`/`doctor` scenario for + isolation (the `{instance}` suffix keeps concurrent runs of the same scenario + apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they operate on the same deployed agent. @@ -74,6 +76,55 @@ Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of them reuse that session credential. +## Parallel-readiness & port allocation + +The tester can run **N concurrent instances of the same scenario** and can +**allocate free TCP ports** per run. Scenarios here are authored to take +advantage of both where it's safe. + +- **`{instance}` substitution.** `start_session(..., instance_id="1")` exposes + `{instance}` for substitution into `command`, `cwd`, `env`, hook fields, and + `goals`. It **defaults to `"main"`** when `instance_id` is omitted, so a single + run is unchanged (dirs/names just end in `-main`). +- **Which scenarios are parallel-ready:** + - **Tier 0 work-dir scenarios** (`doctor`, picker, validate) and **all Tier 1 + `init` scenarios** suffix their `cwd` (and hook paths) with `-{instance}`, so + concurrent instances get isolated working directories. + - **Tier 1 resource names** are suffixed with `-{instance}` too (via the + RESOURCE NAMING goal and the `--agent-name` flag), so parallel instances + don't collide on Azure resource names. + - **`27-run-local-and-invoke-local`** declares `allocate_ports: [agent]` and + binds `azd ai agent run`/`invoke --local` to `--port {agent}`. A port pool is + shared across every `start_session` with the same `scenario_path`, so the + `run` and `invoke` sessions find each other; parallel local runs each get a + distinct port instead of colliding on the default `8088`. +- **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), + plus `20-setup` and `2Z-teardown`, all share the one deployed agent in + `~/working/azd-agents-shared`. They are **not** parameterized with `{instance}` + (doing so would break the shared-agent assumption) and should be run serially. + +To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the +same `instance_id` for paired `run`/`invoke` sessions of one scenario). + +## Driving conventions + +These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent +should follow them so the runs actually *test* the CLI instead of papering over +its bugs: + +- **Don't verify/retry after a `select`.** These runs exist to catch picker + bugs; reading back the echo and "correcting" a pick hides the very defect the + test is for. Send the action and let downstream prompts surface any failure. +- **Treat a select miss as a hard failure.** The tester's `select_by_text` is + fail-loud: a missing target raises `LookupError`, surfaced as + `ERROR during 'select': …`. **Report a finding and stop** — do not retry with a + different `choice_text`/`choice_index` to work around it. +- **Prefer `choice_text` over `choice_index`** when the label is stable (indices + shift between releases). +- **Pause before the first cloud-creating action.** Provisioning is expensive and + irreversible-ish; confirm with the user before entering an `init`/`provision` + flow that creates real resources (especially when running in parallel). + ## Tiers @@ -146,10 +197,11 @@ so they operate on the same deployed agent. - **Model**: `gpt-4.1-mini` (cheap/fast for testing) - **Resource name prefix**: every newly created Azure resource (Foundry project/account, azd environment, agent, model deployment, resource group) is - named with a `trangevi-` prefix so test resources are easy to identify and - clean up. Note that some fields lowercase the value and replace invalid - characters with hyphens — that normalization is expected (see - `sanitizeAgentName` in the extension). + named with a `trangevi-` prefix (and, in parallel-ready Tier 1 scenarios, a + `-{instance}` suffix) so test resources are easy to identify, keep distinct + across concurrent runs, and clean up. Note that some fields lowercase the value + and replace invalid characters with hyphens — that normalization is expected + (see `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. @@ -227,4 +279,8 @@ be run back to back in any order within a tier: group (rather than one file per subcommand) to avoid cross-scenario ordering dependencies — still one command at a time. - `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` - uses two sessions (one to run, one to invoke `--local`). + uses two sessions (one to run, one to invoke `--local`) that share an + allocated `{agent}` port (see + [Parallel-readiness](#parallel-readiness--port-allocation)). +- Run artifacts (screenshots, HTML reports) land in `.reports/`, which is + git-ignored. From 596826b623e2341e24aea3b04706774f88c3695e Mon Sep 17 00:00:00 2001 From: trangevi Date: Wed, 3 Jun 2026 15:43:25 -0700 Subject: [PATCH 04/42] Some scenario updates Signed-off-by: trangevi --- .../00-init-picker-navigation.yaml | 10 +++- .../00-init-validate-mutually-exclusive.yaml | 12 +++-- .../00-init-validate-no-prompt-missing.yaml | 2 +- .../10-init-deploy-mode-code.yaml | 8 ++-- .../10-init-flags-agent-name-model.yaml | 11 ++++- .../10-init-from-code.yaml | 5 +- .../10-init-from-manifest-url.yaml | 12 ++++- .../10-init-template-dotnet.yaml | 2 +- .../10-init-template-python.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 2 +- .../README.md | 48 +++++++++++++++---- .../fixtures/from-code/app.py | 2 +- .../fixtures/from-code/requirements.txt | 4 +- 13 files changed, 88 insertions(+), 32 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index bbf2c0c7fd5..d9907dde3c8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -10,14 +10,20 @@ cwd: "~/working/azd-agents-picker-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Clean dir so the abort-before-Azure check can't be confused by prior state. +# Clean dir, then seed a file so the dir is non-empty. The init-method picker +# ("Use the code in the current directory" / "Start new from a template") only +# appears when the working directory is NOT empty; on an empty dir the command +# auto-selects the template flow and skips the picker. pre: - run: "rm -rf ~/working/azd-agents-picker-{instance}" cwd: "~/working" name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-picker-{instance} && touch ~/working/azd-agents-picker-{instance}/placeholder.txt" + cwd: "~/working" + name: "seed a placeholder file so the init-method picker appears" goals: - - "Wait for the first prompt asking how to initialize (e.g. start from a template / use existing code / use a manifest)." + - "Wait for the first prompt asking how to initialize (it should appear because the directory is non-empty): 'Use the code in the current directory' / 'Start new from a template'." - "Select 'Start new from a template', then wait for the language prompt." - "Select a language (Python), then wait for the template list prompt." - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 6ee69d98636..5d5df023e7a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -1,6 +1,10 @@ -# Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. +# Tier 0 (offline) — `init` rejects a positional argument combined with --manifest. +# +# There is no `--from-code` flag. A real, offline-detectable conflict is passing +# BOTH a positional manifest argument AND -m/--manifest, which the command +# rejects up front (CodeConflictingArguments) before any wizard or network call. name: "init-validate-mutually-exclusive" -command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" +command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" cwd: "~/working/azd-agents-validate-{instance}" env: @@ -14,7 +18,7 @@ pre: goals: - "Wait for the command to fail fast (it should not start the interactive wizard)." - - "Confirm the error message clearly states that --from-code and --manifest/-m are mutually exclusive." + - "Confirm the error message clearly states that you cannot pass both a positional argument and --manifest." - "Confirm the process exits non-zero and does NOT create or modify any project files in the directory." - "Take a screenshot of the error output." - - "Report a finding if the flags are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." + - "Report a finding if the conflicting arguments are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index ac31b1f2127..d5a46e55cb1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -15,6 +15,6 @@ pre: goals: - "Run init in --no-prompt mode in an empty directory with no flags and no existing code/manifest." - "Confirm the command does NOT hang waiting for input (no-prompt must never block on a prompt)." - - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --from-code, --manifest, or --project-id)." + - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --src, --manifest, or --project-id)." - "Take a screenshot of the error output." - "Report a finding if it hangs, prompts interactively despite --no-prompt, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 89260d57ba8..4663a6dbc89 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -2,9 +2,10 @@ # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime -# (instead of building a container image). +# (instead of building a container image). There is no --from-code flag; the +# from-code flow is selected interactively at the init-method prompt. name: "init-deploy-mode-code" -command: "azd ai agent init --from-code --deploy-mode code" +command: "azd ai agent init --deploy-mode code" cwd: "~/working/azd-agents-t1-code-deploy-{instance}" env: @@ -21,7 +22,8 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 5c500ef9fdc..08797dbd7a2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,6 +1,9 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Also requires GitHub login: `gh auth login` (manifest download can fall back to +# the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails +# fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" @@ -9,14 +12,18 @@ cwd: "~/working/azd-agents-t1-flags-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 01026e394da..bc8529c8853 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -6,7 +6,7 @@ # guaranteed and the run is idempotent. Override the fixture location with # AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. name: "init-from-code" -command: "azd ai agent init --from-code" +command: "azd ai agent init" cwd: "~/working/azd-agents-t1-from-code-{instance}" env: @@ -21,7 +21,8 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 6d5b8d38252..d534205437f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,10 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Also requires GitHub login: `gh auth login`. Downloading the manifest (and its +# sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub +# API is rate-limited, which would otherwise drop into an interactive gh login +# mid-run. The pre hook fails fast if gh is not authenticated. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" @@ -8,14 +12,18 @@ cwd: "~/working/azd-agents-t1-manifest-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 9c915254688..a9a2dbd69b8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -15,7 +15,7 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index d9b41930c1d..a8516b86b4f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -16,7 +16,7 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index f18400d226e..36e619802c7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -26,7 +26,7 @@ pre: name: "clear the shared working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 60a6cb3cf6b..1daf046970f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -76,6 +76,24 @@ Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of them reuse that session credential. +### GitHub login (manifest scenarios) + +The manifest scenarios (`10-init-from-manifest-url`, +`10-init-flags-agent-name-model`) download an agent manifest — and its sibling +files — from a public GitHub repo. The CLI first tries the anonymous GitHub API, +but when that's rate-limited (60 req/hr) it falls back to the `gh` CLI, which +would otherwise drop into an **interactive GitHub login** mid-run. Like +`az login`, this is a one-time manual prerequisite the agent can't complete, so a +human must run it once per WSL session **before** driving those scenarios: + +``` +gh auth login +``` + +Those scenarios include a `pre` hook that runs `gh auth status` and **fails fast** +if GitHub CLI isn't authenticated, so a missing login surfaces as a clear setup +error instead of a hung interactive prompt. + ## Parallel-readiness & port allocation The tester can run **N concurrent instances of the same scenario** and can @@ -121,6 +139,10 @@ its bugs: different `choice_text`/`choice_index` to work around it. - **Prefer `choice_text` over `choice_index`** when the label is stable (indices shift between releases). +- **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent + name) come pre-populated with an editable default; typing without clearing + *appends* to it. Select-all then delete (or backspace) first so your value + replaces the default instead of producing `defaultyourvalue`. - **Pause before the first cloud-creating action.** Provisioning is expensive and irreversible-ish; confirm with the user before entering an `init`/`provision` flow that creates real resources (especially when running in parallel). @@ -142,7 +164,7 @@ in any order, any time. | `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | | `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | | `00-doctor-local-only.yaml` | `doctor --local-only` | -| `00-init-validate-mutually-exclusive.yaml` | `init` flag validation (`--from-code` + `-m`) | +| `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | | `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | @@ -155,9 +177,9 @@ and verifies the generated files, then stops before `azd provision`. |------|---------| | `10-init-template-python.yaml` | `init` new-from-template, Python | | `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | -| `10-init-from-manifest-url.yaml` | `init -m ` | -| `10-init-from-code.yaml` | `init --from-code` | -| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` | +| `10-init-from-manifest-url.yaml` | `init -m ` (needs `gh auth login`) | +| `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | +| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | | `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | ### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost @@ -222,10 +244,15 @@ How they're used here: - **`pre` reset** — stateful Tier 0/1 scenarios `rm -rf` their own working dir so re-runs start clean. (`start_session` recreates the dir, so removing it is enough; the doctor/init scenarios just need an empty dir.) -- **`pre` fixture seed** — the `--from-code` scenarios +- **`pre` fixture seed** — the existing-code scenarios (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python - fixture into the dir so the source exists before `init --from-code` inspects it - (see [Fixtures](#fixtures)). + fixture into the dir so the source exists before the wizard's "Use the code in + the current directory" flow inspects it (see [Fixtures](#fixtures)). +- **`pre` gh-auth guard** — the manifest scenarios (`10-init-from-manifest-url`, + `10-init-flags-agent-name-model`) run `gh auth status` and fail fast if GitHub + CLI isn't authenticated, because downloading the manifest can fall back to the + `gh` CLI (and an interactive login) when the anonymous GitHub API is + rate-limited. Run `gh auth login` first (see [Authentication](#authentication)). - **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs `azd down --force --purge` if a leftover project exists in the shared dir (so it never orphans live Azure resources), then clears the dir. The down hook uses @@ -238,9 +265,10 @@ How they're used here: ## Fixtures `fixtures/from-code/` holds a minimal Python agent source tree (`app.py` + -`requirements.txt`) that satisfies the extension's `--from-code` detection +`requirements.txt`) that satisfies the extension's existing-code detection (it looks for `requirements.txt` or any `.py`, and defaults the entry point to -`app.py`). The from-code scenarios copy it into the working dir via a `pre` hook. +`app.py`). The existing-code scenarios copy it into the working dir via a `pre` +hook, then select "Use the code in the current directory" at the init prompt. The hook references the fixture by absolute path with an overridable env var: @@ -250,7 +278,7 @@ cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this -`fixtures/` directory before running the from-code scenarios. +`fixtures/` directory before running the existing-code scenarios. ## Re-running scenarios (idempotency) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py index 56b577b6277..d0d8a0c1c0b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py @@ -1,4 +1,4 @@ -"""Minimal agent source fixture for `azd ai agent init --from-code` scenarios. +"""Minimal agent source fixture for the `azd ai agent init` existing-code scenarios. This file exists only so the extension's from-code detection treats the working directory as a Python agent project (it looks for requirements.txt or any .py diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt index d94c3d032d6..b6ed8d63f87 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt @@ -1,4 +1,4 @@ -# Minimal dependency set so `azd ai agent init --from-code` detects a Python -# project. Keep this lightweight — the Tier 1 from-code scenarios only scaffold +# Minimal dependency set so `azd ai agent init` (existing-code flow) detects a Python +# project. Keep this lightweight — the Tier 1 existing-code scenarios only scaffold # and do not install or run the agent. azure-ai-projects From 71767dbfe6570364f464f1515190544845e82b55 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 10:54:35 -0700 Subject: [PATCH 05/42] Some more fixes to the scenarios Signed-off-by: trangevi --- .../10-init-flags-agent-name-model.yaml | 2 +- .../10-init-from-manifest-url.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 15 ++++++++---- .../21-show-json.yaml | 6 ++--- .../21-show.yaml | 6 ++--- .../22-invoke-input-file.yaml | 6 ++--- .../22-invoke-new-session.yaml | 6 ++--- .../22-invoke-remote.yaml | 6 ++--- .../23-sessions-lifecycle.yaml | 6 ++--- .../24-files-lifecycle.yaml | 6 ++--- .../25-monitor-console.yaml | 6 ++--- .../25-monitor-system.yaml | 6 ++--- .../26-endpoint-update.yaml | 23 +++++++++++++++---- .../27-run-local-and-invoke-local.yaml | 6 ++--- .../28-eval-init-run-show.yaml | 9 ++++---- .../28-eval-update.yaml | 6 ++--- .../29-optimize-submit-status.yaml | 14 +++++++---- .../2A-doctor-provisioned-all-pass.yaml | 9 ++++---- .../2Z-teardown-down.yaml | 2 +- .../README.md | 20 +++++++++++----- .../fixtures/optimize-tasks.jsonl | 3 +++ 21 files changed, 102 insertions(+), 63 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 08797dbd7a2..93e79474fb8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -6,7 +6,7 @@ # fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" cwd: "~/working/azd-agents-t1-flags-{instance}" env: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index d534205437f..fe28a2566fa 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -6,7 +6,7 @@ # API is rate-limited, which would otherwise drop into an interactive gh login # mid-run. The pre hook fails fast if gh is not authenticated. name: "init-from-manifest-url" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" env: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 36e619802c7..807b6721f00 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -1,8 +1,11 @@ # Tier 2 (cloud E2E) — SETUP: deploy the shared agent used by all 21-..2A scenarios. # -# ⚠️ Incurs Azure cost. Run this FIRST. The deployed agent lives in the shared -# working directory ~/working/azd-agents-shared and is reused by the targeted -# scenarios. Run 2Z-teardown-down.yaml LAST to clean up. +# ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared +# and scaffolds the project into a subdirectory named after the agent, so the +# deployed project lives in ~/working/azd-agents-shared/trangevi-basic-responses +# and is reused by the targeted scenarios. The agent name MUST be exactly +# 'trangevi-basic-responses' so that subdirectory path is deterministic. Run +# 2Z-teardown-down.yaml LAST to clean up. name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" @@ -16,7 +19,7 @@ env: # nothing to tear down). Re-using a clean path also avoids the resource-name hash # collision in issue #8360. pre: - - run: "if [ -f ~/working/azd-agents-shared/azure.yaml ]; then (cd ~/working/azd-agents-shared && azd down --force --purge); fi" + - run: "if [ -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml ]; then (cd ~/working/azd-agents-shared/trangevi-basic-responses && azd down --force --purge); fi" cwd: "~/working/azd-agents-shared" name: "tear down any leftover deployed agent" continue_on_error: true @@ -30,6 +33,7 @@ goals: - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." + - "When prompted for the AGENT NAME, set it to EXACTLY 'trangevi-basic-responses' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named 'trangevi-basic-responses'." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." @@ -38,7 +42,8 @@ goals: - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." + - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." + - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This creates the real Azure resources and deploys the agent." - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - "Take a screenshot of the successful provision and 'show' output." - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index 0f9dc491c3a..dc1ae9b47ad 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show-json" command: "azd ai agent show --output json" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index fb06d2ff454..4268464f912 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show" command: "azd ai agent show" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 79ab211dbcb..79766bfe9f4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-input-file" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index ce93cf77620..4aec58782e6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-new-session" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 9fedf175a0c..77c0f9d798e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index f559df8cc08..fb5a5a781d7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -4,12 +4,12 @@ # Targets the `sessions` command group end-to-end in one run. name: "sessions-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index 4356ab7a54d..dbb81bfa47a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -5,12 +5,12 @@ # Note: file operations target a session — run an invoke first if no session exists. name: "files-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index d3288f76781..deff52e7d42 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists to stream logs from. name: "monitor-console" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 08c182ab1d3..85b64e0fbfe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists. name: "monitor-system" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 8cc712f582c..9b970cd50ce 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -3,17 +3,32 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "endpoint-update" command: "azd ai agent endpoint update" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true + # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors + # ("nothing to update") if neither is defined. The Basic Responses template + # defines neither, so inject a minimal agent_card (idempotently) before the run + # so there is something to patch. + - run: | + f="$(find ~/working/azd-agents-shared/trangevi-basic-responses -name agent.yaml | head -1)" + if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then + printf '\nagent_card:\n description: "trangevi endpoint-update test card"\n skills:\n - id: "trangevi-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" + echo "Injected agent_card into $f" + else + echo "agent_card already present or agent.yaml not found" + fi + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + name: "inject agent_card so there is something to patch" + continue_on_error: true goals: - - "Run endpoint update for the default (auto-detected) agent service." + - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agent_card was injected into agent.yaml during setup so the patch has content." - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." - "Take a screenshot of the update result and the post-update 'show' output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index 45a35130019..e0b10da49d9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -5,7 +5,7 @@ # agent, a second invokes it with --local. name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is @@ -21,8 +21,8 @@ notes: | # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml index df9dfcd33ff..aa93e385566 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml @@ -4,17 +4,18 @@ # (a deployed agent is required to generate and run evals). name: "eval-init-run-show" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true goals: - - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --eval-model gpt-4.1-mini --max-samples 15 and wait for it to generate an eval config and dataset." + - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --max-samples 15 and wait for it to generate an eval config and dataset. (Do NOT pass --eval-model — let the command prompt so a real, deployed model is selected.)" + - "At the eval-model selection prompt, accept the pre-filled default deployment. If no default is offered, choose 'Select another deployment' and pick the model deployment that was created during setup (the 'trangevi-'-prefixed gpt-4.1-mini deployment). Do NOT type a bare model name like 'gpt-4.1-mini' — the service resolves this value as a deployment name, and a non-existent deployment returns a 400 error." - "Confirm an eval.yaml (and dataset) is written to the project root." - "Run: azd ai agent eval run and wait for the evaluation run to complete." - "Run: azd ai agent eval list and confirm the run appears." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml index 7101c392f82..19abb9242c1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml @@ -3,12 +3,12 @@ # Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. name: "eval-update" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml index 1c53ded1649..07c46c3e685 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -3,17 +3,23 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "optimize-submit-status" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +# Also copy a small dataset fixture into the project so `optimize` has tasks to +# submit (it requires --dataset, an eval.yaml, or interactive selection). pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true + - run: "cp \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/optimize-tasks.jsonl\" ~/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + name: "stage optimize dataset fixture" + continue_on_error: true goals: - - "Run: azd ai agent optimize --eval-model gpt-4.1-mini --max-iterations 1 --no-wait (agent auto-detected from the azd project). Provide any other values it prompts for using sensible defaults." + - "Run: azd ai agent optimize --dataset optimize-tasks.jsonl --max-iterations 1 --no-wait (agent auto-detected from the azd project; a dataset fixture was staged into the project root during setup). Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index cffe0288a63..166313fae5b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true @@ -17,5 +17,6 @@ goals: - "Confirm the local checks pass (project, azure.yaml, agent service config)." - "Confirm the remote checks pass (deployed agent reachable / active)." - "Confirm doctor suggests a sensible next command (e.g. invoke or run) and exits 0 (at least one check passed, none failed)." + - "KNOWN-ACCEPTABLE WARNING: a WARNING about agent identity / role assignments is EXPECTED in this test subscription (its ABAC conditional role-assignment policy blocks some role reads) and must NOT be treated as a failure. Only an actual FAILED check (not a warning) is a finding." - "Take a screenshot of the doctor report." - - "Report a finding if any check fails or is skipped unexpectedly for a healthy provisioned project, or if the suggested next step is wrong." + - "Report a finding if a check FAILS (red/error), or is skipped unexpectedly, for a healthy provisioned project, or if the suggested next step is wrong. Do NOT report the known identity/role-assignment warning described above." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index bcad6af5fc7..1432baa5212 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -4,7 +4,7 @@ # shared agent and its Azure resources to stop incurring cost. name: "teardown-down" command: "azd down --force --purge" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 1daf046970f..14f5d8a7177 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -45,7 +45,10 @@ Implications: isolation (the `{instance}` suffix keeps concurrent runs of the same scenario apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they - operate on the same deployed agent. + operate on the same deployed agent. `20-setup` runs `init` in that shared dir, + which scaffolds the project into a subdirectory named after the agent, so the + deployed project actually lives in `~/working/azd-agents-shared/trangevi-basic-responses`; + the reuse and teardown scenarios run with that subdirectory as their `cwd`. On macOS/Linux these are simply native paths (no WSL involved). @@ -117,9 +120,11 @@ advantage of both where it's safe. `run` and `invoke` sessions find each other; parallel local runs each get a distinct port instead of colliding on the default `8088`. - **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), - plus `20-setup` and `2Z-teardown`, all share the one deployed agent in - `~/working/azd-agents-shared`. They are **not** parameterized with `{instance}` - (doing so would break the shared-agent assumption) and should be run serially. + plus `20-setup` and `2Z-teardown`, all share the one deployed agent under + `~/working/azd-agents-shared` (the project itself lives in the + `trangevi-basic-responses` subdirectory created by `20-setup`). They are + **not** parameterized with `{instance}` (doing so would break the shared-agent + assumption) and should be run serially. To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the same `instance_id` for paired `run`/`invoke` sessions of one scenario). @@ -189,8 +194,11 @@ Provisions real resources. **Run order matters:** 2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). 3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. -All Tier 2 scenarios share one working directory (`~/working/azd-agents-shared`) -so they operate on the same deployed agent. +All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` +so they operate on the same deployed agent. `20-setup` runs `init` there, which +scaffolds the project into the `trangevi-basic-responses` subdirectory; the +reuse and teardown scenarios run with `~/working/azd-agents-shared/trangevi-basic-responses` +as their `cwd`. | File | Targets | |------|---------| diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl new file mode 100644 index 00000000000..ec1f77655e3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl @@ -0,0 +1,3 @@ +{"prompt":"What is 2+2?","groundTruth":"4"} +{"prompt":"What is the capital of France?","groundTruth":"Paris"} +{"prompt":"Name a primary color.","groundTruth":"Red"} From df67a66389d8c58ac752536d4f7b34c901ac09ec Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 15:03:05 -0700 Subject: [PATCH 06/42] Some more improvements Signed-off-by: trangevi --- .../20-setup-deploy-shared-agent.yaml | 9 +- .../22-invoke-new-session.yaml | 16 +++- .../23-sessions-lifecycle.yaml | 5 +- .../29-optimize-submit-status.yaml | 2 +- .../README.md | 91 ++++++++++++++++++- 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 807b6721f00..8b8bada67dd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -43,7 +43,8 @@ goals: - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." - - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This creates the real Azure resources and deploys the agent." - - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - - "Take a screenshot of the successful provision and 'show' output." - - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." + - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." + - "After provision succeeds, run 'azd deploy' (from the same subdirectory) and wait for it to succeed. This deploys the agent to the provisioned infrastructure — it is a required, separate step from provision before the agent can be shown or invoked." + - "After deploy, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." + - "Take a screenshot of the successful provision, deploy, and 'show' output." + - "Report a finding if init, provision, or deploy fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index 4aec58782e6..d56211ce4e6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -1,7 +1,12 @@ -# Tier 2 (cloud E2E) — `invoke --new-session` resets conversation state. +# Tier 2 (cloud E2E) — `invoke` SESSION vs CONVERSATION memory semantics. +# +# A SESSION and a CONVERSATION are distinct concepts. For the responses-protocol +# agent (Basic Responses), multi-turn memory is bound to the CONVERSATION, not the +# session: `--new-session` starts a fresh session but reuses the saved conversation +# (memory persists), while `--new-conversation` is what actually resets memory. # # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -name: "invoke-new-session" +name: "invoke-session-vs-conversation" command: "bash" cwd: "~/working/azd-agents-shared/trangevi-basic-responses" @@ -14,7 +19,8 @@ pre: goals: - "Run: azd ai agent invoke \"My name is Quinn. Remember it.\" and wait for a response." - - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session reused the same conversation)." - - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent does NOT recall 'Quinn' — the new session discarded prior history." + - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session and conversation were reused, so multi-turn memory works)." + - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent STILL recalls 'Quinn'. This is EXPECTED, not a bug: a SESSION and a CONVERSATION are distinct concepts. For this responses-protocol agent, multi-turn memory is bound to the CONVERSATION, not the session. --new-session starts a fresh session but reuses the saved conversation, so memory persists." + - "Run: azd ai agent invoke --new-conversation \"What is my name?\" and confirm the agent does NOT recall 'Quinn'. This is the true memory reset: --new-conversation discards the conversation that holds multi-turn memory." - "Take a screenshot after each invoke." - - "Report a finding if --new-session still carries over prior conversation context, or if session persistence between the first two invokes does not work." + - "Report a finding if: the first two invokes fail to establish/recall memory; OR --new-conversation still recalls 'Quinn' (memory was not reset). Do NOT report a finding merely because --new-session still recalled 'Quinn' — that is the expected session-vs-conversation distinction." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index fb5a5a781d7..1bf3ef57e6c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -18,6 +18,7 @@ goals: - "Run: azd ai agent sessions list --output table. Confirm the newly created session appears in the list." - "Run: azd ai agent sessions show using the ID from create. Confirm session details (status, version) are shown." - "Run: azd ai agent sessions delete . Confirm the session is deleted (synchronously)." - - "Run: azd ai agent sessions list again and confirm the deleted session no longer appears." + - "Run: azd ai agent sessions list again. The service SOFT-DELETES sessions, so it is EXPECTED and acceptable for the entry to still appear with status 'deleted' (it should NOT appear as 'active'). Either the entry being gone OR present with status 'deleted' is a PASS." + - "Run: azd ai agent sessions show for the deleted session and confirm it reports the session as not found / deleted (this verifies the delete took effect even though list may still show it)." - "Take a screenshot after each step." - - "Report a finding if any subcommand errors, if the created session is missing from list, or if delete does not remove it." + - "Report a finding only if any subcommand errors, if the created session is missing from list before delete, or if after delete the session still shows as 'active' (a soft-deleted 'deleted' status is NOT a finding)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml index 07c46c3e685..ebbfa72f8fe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -19,7 +19,7 @@ pre: continue_on_error: true goals: - - "Run: azd ai agent optimize --dataset optimize-tasks.jsonl --max-iterations 1 --no-wait (agent auto-detected from the azd project; a dataset fixture was staged into the project root during setup). Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." + - "Run: azd ai agent optimize --dataset \"$HOME/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl\" --max-iterations 1 --no-wait (agent auto-detected from the azd project; the dataset fixture was staged into the project root during setup). Use an ABSOLUTE path for --dataset as shown: a bare relative name is resolved relative to the agent SERVICE subdirectory, not the project root, so the absolute path avoids a 'dataset not accessible' error. Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 14f5d8a7177..5568cb47ea1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -52,6 +52,23 @@ Implications: On macOS/Linux these are simply native paths (no WSL involved). +### This applies to MCP tool arguments too + +The same path-resolution rule applies to **every path-shaped argument an +orchestrator passes to the tester's MCP tools** — most importantly the `path:` +argument on `load_scenario`, `run_pre_hooks`, and `run_post_hooks`, and the +`scenario_path:` argument on `start_session`. The server resolves them on the +WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: + +| Orchestrator OS | Pass to MCP tools | Don't pass | +| --- | --- | --- | +| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml` | `C:\Repos\azure-dev\...\00-version.yaml` | +| macOS / Linux | native absolute path | — | + +**Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the +path style is almost certainly the cause — translate `C:\…` to `/mnt/c/…` and +retry one call before fanning out. + ## Authentication Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in @@ -129,6 +146,47 @@ advantage of both where it's safe. To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the same `instance_id` for paired `run`/`invoke` sessions of one scenario). +## Orchestrating a fleet run + +When a driving agent wants to run **many scenarios concurrently** (e.g. via +parallel background sub-agents, one scenario per sub-agent), pick the right +fan-out primitive for the shape of the run: + +- **Different scenarios in parallel** (the common case for a full Tier 0/1 + sweep): give each sub-agent a distinct, descriptive `session_id` — e.g. + `fleet-00-version`, `fleet-10-init-from-code` — and call `start_session` with + the scenario's own `cwd`. **No `instance_id` is needed**: each scenario's `cwd` + already isolates itself via the `{instance}` substitution, which defaults to + `"main"` when `instance_id` is omitted. +- **Same scenario N times in parallel:** use `instance_id="1"`, `"2"`, … per + call. See [Parallel-readiness](#parallel-readiness--port-allocation) for which + scenarios are authored to support this. +- **Tier 2 ordering is fixed**, not parallel-friendly. Run `20-setup` first, + then the targeted `21-…2A-` scenarios **serially** (they share one deployed + agent and mutate shared state — sessions, files, endpoint configuration — + so parallel runs interfere), then `2Z-teardown` last. See the + [Tier 2](#tier-2--cloud-end-to-end-prefix-2x---%EF%B8%8F-incurs-azure-cost) + section. + +### Operational guardrails for the orchestrator + +A few hard-won lessons that apply regardless of fleet size: + +- **Validate the recipe with one sub-agent before fanning out.** Spend 30 + seconds confirming that `load_scenario`, `start_session`, and one + `send_action` round-trip work end-to-end for *one* scenario before launching + a wave. This is the cheapest way to catch wiring issues (wrong path style, + wrong tool surface, auth not set up) before they multiply across many agents. +- **Background sub-agents are typically not cancellable mid-run.** Once launched, + they will run to completion (or until the runtime times them out). For Tier 1 + and especially Tier 2 scenarios with Azure side effects, launch + conservatively — a stop request can't recall an in-flight `azd provision`. +- **Keep waves small.** The wall-clock bottleneck on a fleet run is per-agent + LLM time and per-account model concurrency, not the MCP server (which is + per-`session_id`-parallel by design). Launching 4–6 sub-agents at a time and + rolling forward typically finishes a sweep faster than launching everything + at once. + ## Driving conventions These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent @@ -151,6 +209,35 @@ its bugs: - **Pause before the first cloud-creating action.** Provisioning is expensive and irreversible-ish; confirm with the user before entering an `init`/`provision` flow that creates real resources (especially when running in parallel). +- **Pass `run_name=` to every `start_session` call.** The + scenario stem is the YAML filename without `.yaml` (e.g. `00-version`, + `21-show-json`, `27-run-local-and-invoke-local`). Without `run_name` the + tester auto-names the run folder `agent_YYYYMMDD_HHMMSS`, which makes + archived runs in `.reports//tester-reports/` hard to cross-reference + with the scenario list. For scenarios that start two sessions + (e.g. `27-run-local-and-invoke-local`), suffix the run_name with a role tag + (`27-run-local-and-invoke-local-run`, `27-run-local-and-invoke-local-invoke`) + so each session gets its own clearly named folder. +- **Pass `output_dir` to every `start_session` call** so the tester writes + screenshots and HTML reports directly into this repo's archive layout + instead of its own working directory. Use the WSL path of the + `.reports//tester-reports/` folder under this scenarios + directory, with `` of the form `YYYYMMDD-HHMMSS`. Pick **one** + `` per suite run and reuse it across every session — this + groups all scenarios from one run under a single folder. The driving agent + should also write the final cross-scenario summary to + `.reports//FINAL-REPORT.md` at the end. Example + `output_dir` (the WSL view of this scenarios directory in this repo): + `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.reports/20260603-171132/tester-reports`. + If your clone lives elsewhere, substitute the WSL path of *your* + `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. +- **Record a time-to-complete per scenario.** Capture wall-clock duration for + every scenario (from `start_session` to `finish_session`, including pre/post + hooks) and include it as a `Duration` column in the per-scenario tables of + `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). + This makes regression slowdowns easy to spot across runs — Tier 2 in + particular has scenarios that legitimately take many minutes (provision, + deploy, eval dataset generation) and others that should complete in seconds. ## Tiers @@ -202,11 +289,11 @@ as their `cwd`. | File | Targets | |------|---------| -| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` (SETUP) | +| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` + `azd deploy` (SETUP) | | `21-show.yaml` | `show` (table) | | `21-show-json.yaml` | `show --output json` | | `22-invoke-remote.yaml` | `invoke` (remote) | -| `22-invoke-new-session.yaml` | `invoke --new-session` | +| `22-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | | `22-invoke-input-file.yaml` | `invoke -f ` | | `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | | `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | From 3eac215885617c1e22172264261494bf3ea3bee0 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 17:04:20 -0700 Subject: [PATCH 07/42] Remove optimization and evals because I don't understand them yet, will add back later Signed-off-by: trangevi --- .../28-eval-init-run-show.yaml | 24 ----------------- .../28-eval-update.yaml | 20 -------------- .../29-optimize-submit-status.yaml | 27 ------------------- .../README.md | 5 +--- .../fixtures/optimize-tasks.jsonl | 3 --- 5 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml deleted file mode 100644 index aa93e385566..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Tier 2 (cloud E2E) — `eval` flow: init a suite, run it, then list/show results. -# -# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully -# (a deployed agent is required to generate and run evals). -name: "eval-init-run-show" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - -goals: - - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --max-samples 15 and wait for it to generate an eval config and dataset. (Do NOT pass --eval-model — let the command prompt so a real, deployed model is selected.)" - - "At the eval-model selection prompt, accept the pre-filled default deployment. If no default is offered, choose 'Select another deployment' and pick the model deployment that was created during setup (the 'trangevi-'-prefixed gpt-4.1-mini deployment). Do NOT type a bare model name like 'gpt-4.1-mini' — the service resolves this value as a deployment name, and a non-existent deployment returns a 400 error." - - "Confirm an eval.yaml (and dataset) is written to the project root." - - "Run: azd ai agent eval run and wait for the evaluation run to complete." - - "Run: azd ai agent eval list and confirm the run appears." - - "Run: azd ai agent eval show and confirm run details / scores are displayed." - - "Take a screenshot after init, run, and show." - - "Report a finding if generation fails, eval.yaml is malformed, the run errors, or results are missing/unreadable." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml deleted file mode 100644 index 19abb9242c1..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Tier 2 (cloud E2E) — `eval update` refreshes dataset and evaluators. -# -# Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. -name: "eval-update" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - -goals: - - "Confirm an eval.yaml from a prior 'eval init' exists in the project root (run 28-eval-init-run-show.yaml first if not)." - - "Run: azd ai agent eval update --dataset-only and confirm only the dataset is updated (evaluators left unchanged)." - - "Run: azd ai agent eval update --evaluator-only and confirm only the evaluators are updated (dataset left unchanged)." - - "Take a screenshot after each update." - - "Report a finding if --dataset-only or --evaluator-only updates the wrong thing, errors, or prompts despite the scoping flag." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml deleted file mode 100644 index ebbfa72f8fe..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Tier 2 (cloud E2E) — `optimize` submit (no-wait) then check status/list. -# -# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -name: "optimize-submit-status" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -# Also copy a small dataset fixture into the project so `optimize` has tasks to -# submit (it requires --dataset, an eval.yaml, or interactive selection). -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - - run: "cp \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/optimize-tasks.jsonl\" ~/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "stage optimize dataset fixture" - continue_on_error: true - -goals: - - "Run: azd ai agent optimize --dataset \"$HOME/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl\" --max-iterations 1 --no-wait (agent auto-detected from the azd project; the dataset fixture was staged into the project root during setup). Use an ABSOLUTE path for --dataset as shown: a bare relative name is resolved relative to the agent SERVICE subdirectory, not the project root, so the absolute path avoids a 'dataset not accessible' error. Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." - - "Take a screenshot after submit, status, and list." - - "Report a finding if submission fails, the job ID is not surfaced, or status/list cannot find the job. (Optional: cancel with 'azd ai agent optimize cancel ' to avoid leaving it running.)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 5568cb47ea1..0ffedc534fa 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -237,7 +237,7 @@ its bugs: `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). This makes regression slowdowns easy to spot across runs — Tier 2 in particular has scenarios that legitimately take many minutes (provision, - deploy, eval dataset generation) and others that should complete in seconds. + deploy) and others that should complete in seconds. ## Tiers @@ -301,9 +301,6 @@ as their `cwd`. | `25-monitor-system.yaml` | `monitor --type system` | | `26-endpoint-update.yaml` | `endpoint update` | | `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | -| `28-eval-init-run-show.yaml` | `eval init/run/list/show` | -| `28-eval-update.yaml` | `eval update` | -| `29-optimize-submit-status.yaml` | `optimize` + `optimize status/list` | | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl deleted file mode 100644 index ec1f77655e3..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"prompt":"What is 2+2?","groundTruth":"4"} -{"prompt":"What is the capital of France?","groundTruth":"Paris"} -{"prompt":"Name a primary color.","groundTruth":"Red"} From c18315cb27f7435048d315d6a9f91a67c790396c Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 17:29:49 -0700 Subject: [PATCH 08/42] Add prompt to readme Signed-off-by: trangevi --- .../README.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 0ffedc534fa..2539711b89b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -24,6 +24,28 @@ the working dir or seeding a fixture), and a few declare **`post:` hooks** `run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. See [Pre/post hooks](#prepost-hooks) below. +Here's also a sample prompt to run all of the scenarios, utilizing fleet mode: + +``` +Within the agents extension, there is a tests/cli-interactive-tester-scenarios directory, containing +a set of test scenarios for the cli-interactive-tester. I want you to use the cli-interactive-tester to; + load the scenarios, + start the session and accomplish the goals, + if the scenario declares pre or post hooks, run them before/after the session, + and take screenshots at each step. + +I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios +in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios +in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make +sure to take that into account when distributing the work. I want to run all of the tests regardless +of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. + +After all of these scenarios are run, create a final result report. + +Create a plan to accomplish this +``` + + ## Paths run inside WSL (on Windows) The cli-interactive-tester drives CLIs through **tmux**, which on Windows runs From 1d1e5f2d978bec55e324142db439a1419e62daec Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 12:34:16 -0700 Subject: [PATCH 09/42] Add parameterization support Signed-off-by: trangevi --- .../.gitignore | 4 + .../10-init-deploy-mode-code.yaml | 13 +- .../10-init-flags-agent-name-model.yaml | 17 +-- .../10-init-from-code.yaml | 13 +- .../10-init-from-manifest-url.yaml | 15 ++- .../10-init-template-dotnet.yaml | 15 ++- .../10-init-template-python.yaml | 19 +-- .../20-setup-deploy-shared-agent.yaml | 31 +++-- .../21-show-json.yaml | 6 +- .../21-show.yaml | 6 +- .../22-invoke-input-file.yaml | 6 +- .../22-invoke-new-session.yaml | 6 +- .../22-invoke-remote.yaml | 6 +- .../23-sessions-lifecycle.yaml | 6 +- .../24-files-lifecycle.yaml | 6 +- .../25-monitor-console.yaml | 6 +- .../25-monitor-system.yaml | 6 +- .../26-endpoint-update.yaml | 12 +- .../27-run-local-and-invoke-local.yaml | 6 +- .../2A-doctor-provisioned-all-pass.yaml | 6 +- .../2Z-teardown-down.yaml | 2 +- .../README.md | 124 ++++++++++++++---- .../profile.local.yaml.example | 27 ++++ .../profile.yaml | 25 ++++ 24 files changed, 261 insertions(+), 122 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore index 7fde892bf70..a3ca34ee539 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore @@ -1,2 +1,6 @@ # cli-interactive-tester run artifacts (screenshots, HTML reports, scrollback) .reports/ + +# Per-developer / per-CI scenario profile (identifying values). Bootstrap +# from profile.local.yaml.example. See README "Profile / overrides". +profile.local.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 4663a6dbc89..501f2fdfb32 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime # (instead of building a container image). There is no --from-code flag; the # from-code flow is selected interactively at the init-method prompt. @@ -22,16 +23,16 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." - "Take a screenshot of the completed init output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 93e79474fb8..cfc34c166fe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,12 +1,13 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Also requires GitHub login: `gh auth login` (manifest download can fall back to # the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails # fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" cwd: "~/working/azd-agents-t1-flags-{instance}" env: @@ -23,15 +24,15 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index bc8529c8853..a34ed68a941 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from existing code in the current directory. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Precondition: the cwd should already contain agent source code (and ideally an # agent manifest). The pre hooks seed a committed Python fixture so this is # guaranteed and the run is idempotent. Override the fixture location with @@ -21,14 +22,14 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' in the output." - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." - "Take a screenshot of the completed init output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index fe28a2566fa..155477a364d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Also requires GitHub login: `gh auth login`. Downloading the manifest (and its # sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub # API is rate-limited, which would otherwise drop into an interactive gh login @@ -23,14 +24,14 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." - - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}' (or accept the manifest's model if one is pinned)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index a9a2dbd69b8..842de2de13a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. All `{name}` placeholders below come from the merged profile. name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet-{instance}" @@ -15,16 +16,16 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index a8516b86b4f..b6258a96d07 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -1,7 +1,10 @@ # Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Reads subscriptions/Foundry projects but does NOT -# run `azd provision`, so no resources are created and no cost is incurred. +# Requires Azure login (see README "Authentication"). Reads subscriptions/Foundry +# projects but does NOT run `azd provision`, so no resources are created and no +# cost is incurred. All `{name}` placeholders below come from the merged +# profile (`profile.yaml` + `profile.local.yaml`); the orchestrator passes them +# as `session_vars` to every MCP call. name: "init-template-python" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-python-{instance}" @@ -16,16 +19,16 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 8b8bada67dd..039c7acaea7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -2,10 +2,13 @@ # # ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared # and scaffolds the project into a subdirectory named after the agent, so the -# deployed project lives in ~/working/azd-agents-shared/trangevi-basic-responses -# and is reused by the targeted scenarios. The agent name MUST be exactly -# 'trangevi-basic-responses' so that subdirectory path is deterministic. Run -# 2Z-teardown-down.yaml LAST to clean up. +# deployed project lives in ~/working/azd-agents-shared/{shared_agent_name} +# (where {shared_agent_name} = {prefix}-{shared_agent_suffix}, e.g. +# "alice-basic-responses") and is reused by the targeted scenarios. The agent +# name MUST be exactly that value so the subdirectory path is deterministic +# and the reuse scenarios can find it. Run 2Z-teardown-down.yaml LAST to +# clean up. All `{name}` placeholders come from the merged profile +# (`profile.yaml` + `profile.local.yaml`). name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" @@ -19,7 +22,7 @@ env: # nothing to tear down). Re-using a clean path also avoids the resource-name hash # collision in issue #8360. pre: - - run: "if [ -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml ]; then (cd ~/working/azd-agents-shared/trangevi-basic-responses && azd down --force --purge); fi" + - run: "if [ -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml ]; then (cd ~/working/azd-agents-shared/{shared_agent_name} && azd down --force --purge); fi" cwd: "~/working/azd-agents-shared" name: "tear down any leftover deployed agent" continue_on_error: true @@ -29,21 +32,21 @@ pre: name: "clear the shared working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' (e.g. '{shared_agent_name}'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." - - "When prompted for the AGENT NAME, set it to EXACTLY 'trangevi-basic-responses' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named 'trangevi-basic-responses'." + - "When prompted for the AGENT NAME, set it to EXACTLY '{shared_agent_name}' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named '{shared_agent_name}'." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." - - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." + - "Change directory into the new '{shared_agent_name}' subdirectory (run 'cd {shared_agent_name}') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." + - "Run 'azd provision' (from inside the '{shared_agent_name}' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." - "After provision succeeds, run 'azd deploy' (from the same subdirectory) and wait for it to succeed. This deploys the agent to the provisioned infrastructure — it is a required, separate step from provision before the agent can be shown or invoked." - "After deploy, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - "Take a screenshot of the successful provision, deploy, and 'show' output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index dc1ae9b47ad..d71400568c9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show-json" command: "azd ai agent show --output json" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index 4268464f912..78e0a131bdc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show" command: "azd ai agent show" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 79766bfe9f4..74109e0c82f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-input-file" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index d56211ce4e6..839c37dea8b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -8,12 +8,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-session-vs-conversation" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 77c0f9d798e..0f42731b15c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index 1bf3ef57e6c..2b633eef537 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -4,12 +4,12 @@ # Targets the `sessions` command group end-to-end in one run. name: "sessions-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index dbb81bfa47a..6d1ebbcfa92 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -5,12 +5,12 @@ # Note: file operations target a session — run an invoke first if no session exists. name: "files-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index deff52e7d42..479db222209 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists to stream logs from. name: "monitor-console" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 85b64e0fbfe..8b6ebb63acf 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists. name: "monitor-system" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 9b970cd50ce..705f2ccee21 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "endpoint-update" command: "azd ai agent endpoint update" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors @@ -16,14 +16,14 @@ pre: # defines neither, so inject a minimal agent_card (idempotently) before the run # so there is something to patch. - run: | - f="$(find ~/working/azd-agents-shared/trangevi-basic-responses -name agent.yaml | head -1)" + f="$(find ~/working/azd-agents-shared/{shared_agent_name} -name agent.yaml | head -1)" if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then - printf '\nagent_card:\n description: "trangevi endpoint-update test card"\n skills:\n - id: "trangevi-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" + printf '\nagent_card:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" echo "Injected agent_card into $f" else echo "agent_card already present or agent.yaml not found" fi - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "inject agent_card so there is something to patch" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index e0b10da49d9..86d9d66a112 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -5,7 +5,7 @@ # agent, a second invokes it with --local. name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is @@ -21,8 +21,8 @@ notes: | # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index 166313fae5b..af2dea47182 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index 1432baa5212..45e9eb6d4b1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -4,7 +4,7 @@ # shared agent and its Azure resources to stop incurring cost. name: "teardown-down" command: "azd down --force --purge" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 2539711b89b..e89a211e8cd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -1,4 +1,4 @@ - + # `azd ai agent` — cli-interactive-tester scenarios Goal-based scenarios for driving the `azure.ai.agents` extension through the @@ -8,14 +8,37 @@ strict `goals:` list format so the run is repeatable and reviewable. ## How to run -Register the cli-interactive-tester MCP server (see its README), then ask -Copilot CLI to load a scenario and accomplish its goals, e.g.: +Register the cli-interactive-tester MCP server (see its README), then +**bootstrap your profile** (one-time, per checkout — see [Profile / overrides](#profile--overrides)): + +```sh +cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` and `subscription` at minimum +``` + +Then ask Copilot CLI to load a scenario and accomplish its goals. The +orchestrator must **load both profile files, merge them (local overrides +shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass +the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, +`start_session`, and `run_post_hooks` call** — the scenario YAMLs reference +those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, +`{tenant}` (optional), and `{shared_agent_name}` placeholders. Example +prompt: ``` -Use the cli-interactive-tester to load the scenario at -tests/cli-interactive-tester-scenarios/00-version.yaml. If it declares pre hooks, -run them first; then start the session, accomplish the goals, take screenshots at -each step, and run any post hooks after finishing. +Use the cli-interactive-tester to drive the scenario at +tests/cli-interactive-tester-scenarios/00-version.yaml. + +First, read tests/cli-interactive-tester-scenarios/profile.yaml and +profile.local.yaml and merge them (local overrides shared); also derive +shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call. + +If the scenario declares pre hooks, run them first; then start the session, +accomplish the goals, take screenshots at each step, and run any post hooks +after finishing. ``` Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting @@ -34,6 +57,12 @@ a set of test scenarios for the cli-interactive-tester. I want you to use the cl if the scenario declares pre or post hooks, run them before/after the session, and take screenshots at each step. +First, read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge +them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}". +Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call — the scenarios reference {prefix}, {subscription}, {region}, {model}, +{tenant} (optional), and {shared_agent_name} placeholders. + I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make @@ -69,8 +98,10 @@ Implications: single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they operate on the same deployed agent. `20-setup` runs `init` in that shared dir, which scaffolds the project into a subdirectory named after the agent, so the - deployed project actually lives in `~/working/azd-agents-shared/trangevi-basic-responses`; - the reuse and teardown scenarios run with that subdirectory as their `cwd`. + deployed project actually lives in `~/working/azd-agents-shared/{shared_agent_name}` + (where `{shared_agent_name} = {prefix}-{shared_agent_suffix}` from your + [profile](#profile--overrides), e.g. `alice-basic-responses`); the reuse and + teardown scenarios run with that subdirectory as their `cwd`. On macOS/Linux these are simply native paths (no WSL involved). @@ -100,19 +131,21 @@ opens a **separate browser window** for account selection that requires human interaction outside the terminal the agent controls. Treat auth as a one-time manual prerequisite, not a scenario step. -Inside WSL, a human runs: +Inside WSL, a human runs (substituting `{tenant}` and `{subscription}` with +the values from their [profile](#profile--overrides) — omit `--tenant` +entirely if `tenant` isn't set in `profile.local.yaml`): ``` -az login --tenant azdaiagent.onmicrosoft.com +az login --tenant {tenant} # or just `az login` if {tenant} is unset ``` This opens the interactive sign-in flow and then: 1. **Browser account selection** — a separate browser window opens; the human - picks the account in the `azdaiagent.onmicrosoft.com` tenant. (The agent - cannot do this.) + picks the account in the `{tenant}` tenant (or any tenant, if `{tenant}` + isn't set). The agent cannot do this. 2. **Subscription selection** — back in the terminal, select the - `azd ai agent development` subscription. + `{subscription}` subscription. Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of @@ -161,7 +194,7 @@ advantage of both where it's safe. - **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), plus `20-setup` and `2Z-teardown`, all share the one deployed agent under `~/working/azd-agents-shared` (the project itself lives in the - `trangevi-basic-responses` subdirectory created by `20-setup`). They are + `{shared_agent_name}` subdirectory created by `20-setup`). They are **not** parameterized with `{instance}` (doing so would break the shared-agent assumption) and should be run serially. @@ -305,8 +338,8 @@ Provisions real resources. **Run order matters:** All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` so they operate on the same deployed agent. `20-setup` runs `init` there, which -scaffolds the project into the `trangevi-basic-responses` subdirectory; the -reuse and teardown scenarios run with `~/working/azd-agents-shared/trangevi-basic-responses` +scaffolds the project into the `{shared_agent_name}` subdirectory; the +reuse and teardown scenarios run with `~/working/azd-agents-shared/{shared_agent_name}` as their `cwd`. | File | Targets | @@ -326,18 +359,57 @@ as their `cwd`. | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | +## Profile / overrides + +Developer- and environment-specific values (subscription, region, model, +resource-name prefix, optional tenant) are **not** hardcoded in the scenario +YAMLs. Instead, the scenarios reference them via `{name}` placeholders, and +the orchestrator supplies the values as `session_vars` on every tester call. + +Two files in this directory drive the values: + +| File | Tracked? | Contents | Notes | +|---|---|---|---| +| `profile.yaml` | ✅ checked in | repo-shared defaults | `region`, `model`, `shared_agent_suffix` | +| `profile.local.yaml` | ❌ gitignored | per-developer / per-CI overrides | required: `prefix`, `subscription`. optional: `tenant` (no default) | +| `profile.local.yaml.example` | ✅ checked in | starter template | copy to `profile.local.yaml` and edit | + +Variables exposed to scenarios via `session_vars`: + +| Placeholder | Source | Default | Notes | +|---|---|---|---| +| `{prefix}` | `profile.local.yaml` | **required** | resource-name prefix; should be lowercase + hyphen-friendly so `sanitizeAgentName` doesn't mutate it | +| `{subscription}` | `profile.local.yaml` | **required** | subscription display name | +| `{tenant}` | `profile.local.yaml` | optional, no default | only consumed by the `az login` guidance above; when unset, drop `--tenant` and rely on the user's default tenant | +| `{region}` | `profile.yaml` | `East US 2` | | +| `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | +| `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | +| `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}` | Tier 2 subdirectory name — orchestrator must compute and pass alongside the others | + +**Bootstrap (one-time per checkout):** + +```sh +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` (lowercase, hyphen-friendly) and `subscription` +``` + +The orchestrator must load both files, merge (local overrides shared), derive +`shared_agent_name`, and pass the merged map as `session_vars=` on every +`load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. +Failing to thread `session_vars` leaves `{prefix}` etc. unresolved in goals and +the run will execute against literal placeholder strings. + ## Conventions -- **Subscription**: `azd ai agent development` -- **Region**: `East US 2` -- **Model**: `gpt-4.1-mini` (cheap/fast for testing) -- **Resource name prefix**: every newly created Azure resource (Foundry +- **Tunable values** (subscription, region, model, prefix, tenant) come from + the profile pair above — see [Profile / overrides](#profile--overrides). +- **Resource naming**: every newly created Azure resource (Foundry project/account, azd environment, agent, model deployment, resource group) is - named with a `trangevi-` prefix (and, in parallel-ready Tier 1 scenarios, a - `-{instance}` suffix) so test resources are easy to identify, keep distinct - across concurrent runs, and clean up. Note that some fields lowercase the value - and replace invalid characters with hyphens — that normalization is expected - (see `sanitizeAgentName` in the extension). + named with the `{prefix}-` value from your profile (and, in parallel-ready + Tier 1 scenarios, a `-{instance}` suffix) so test resources are easy to + identify, keep distinct across concurrent runs, and clean up. Note that some + fields lowercase the value and replace invalid characters with hyphens — that + normalization is expected (see `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example new file mode 100644 index 00000000000..234b8f217ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example @@ -0,0 +1,27 @@ +# Per-developer / per-CI scenario profile — identifying values. +# +# Bootstrap: +# cp profile.local.yaml.example profile.local.yaml +# # then edit profile.local.yaml +# +# profile.local.yaml is gitignored. CI populates it from pipeline secrets. +# Any key set here overrides the same key in profile.yaml. + +# REQUIRED. Prefix applied to every newly created Azure resource (Foundry +# project/account, azd environment, agent, model deployment, resource group) +# so test resources are easy to identify and clean up. Use lowercase and +# hyphens — some fields normalize the value (sanitizeAgentName lowercases +# and replaces invalid characters), and a clean prefix avoids surprises. +# Concurrent runs of the same Tier 1 scenario automatically suffix this +# with -{instance} to keep parallel resource names distinct. +prefix: "your-alias" + +# REQUIRED. Display name of the Azure subscription to select when scenarios +# prompt for one. Tier 1 and Tier 2 scenarios both read this. +subscription: "azd ai agent development" + +# OPTIONAL. Tenant for `az login --tenant ` (and `gh auth login`). +# When unset, the driving agent omits the --tenant flag entirely and relies +# on your default tenant. Set this only if you actually need to scope to a +# specific tenant. +# tenant: "azdaiagent.onmicrosoft.com" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml new file mode 100644 index 00000000000..30ef5c004d4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml @@ -0,0 +1,25 @@ +# Repo-shared scenario profile — defaults for non-identifying values. +# +# This file is checked in. Identifying values (prefix, subscription, optional +# tenant) live in profile.local.yaml, which is gitignored and bootstrapped +# from profile.local.yaml.example. +# +# The driving agent loads both files (local overrides this one), validates +# that the required keys are present, derives `shared_agent_name` = +# "-", and then passes the merged map as +# `session_vars` to EVERY MCP tool call (load_scenario, run_pre_hooks, +# start_session, run_post_hooks) — see the "Profile / overrides" section +# of README.md. + +# Default Azure region for new resources created by Tier 1/2 scenarios. +region: "East US 2" + +# Default model deployment chosen during init / referenced by --model flags. +# gpt-4.1-mini is cheap and fast — appropriate for tests. +model: "gpt-4.1-mini" + +# Suffix appended to {prefix} to form the Tier 2 shared agent's name (and +# therefore the subdir name that `azd ai agent init` scaffolds into under +# ~/working/azd-agents-shared/). The shared name is exposed to scenarios +# as {shared_agent_name}. +shared_agent_suffix: "basic-responses" From d3cd9b5007066aa5c6a4fd2940b719f9140a516b Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:03:46 -0700 Subject: [PATCH 10/42] Add tags Signed-off-by: trangevi --- .../00-doctor-empty-dir.yaml | 1 + .../00-doctor-local-only.yaml | 1 + .../00-help-root.yaml | 1 + .../00-init-picker-navigation.yaml | 1 + .../00-init-validate-mutually-exclusive.yaml | 1 + .../00-init-validate-no-prompt-missing.yaml | 1 + .../00-sample-list-json-filters.yaml | 1 + .../00-sample-list-text.yaml | 1 + .../00-version.yaml | 1 + .../10-init-deploy-mode-code.yaml | 1 + .../10-init-flags-agent-name-model.yaml | 1 + .../10-init-from-code.yaml | 1 + .../10-init-from-manifest-url.yaml | 1 + .../10-init-template-dotnet.yaml | 1 + .../10-init-template-python.yaml | 1 + .../20-setup-deploy-shared-agent.yaml | 1 + .../21-show-json.yaml | 1 + .../21-show.yaml | 1 + .../22-invoke-input-file.yaml | 1 + .../22-invoke-new-session.yaml | 1 + .../22-invoke-remote.yaml | 1 + .../23-sessions-lifecycle.yaml | 1 + .../24-files-lifecycle.yaml | 1 + .../25-monitor-console.yaml | 1 + .../25-monitor-system.yaml | 1 + .../26-endpoint-update.yaml | 1 + .../27-run-local-and-invoke-local.yaml | 1 + .../2A-doctor-provisioned-all-pass.yaml | 1 + .../2Z-teardown-down.yaml | 1 + .../README.md | 108 +++++++++++++++--- 30 files changed, 119 insertions(+), 18 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml index 53aba1a8dbc..dfd555970bb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -2,6 +2,7 @@ name: "doctor-empty-dir" command: "azd ai agent doctor" cwd: "~/working/azd-agents-doctor-empty-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir so the "no azd project" path is exercised. # start_session recreates the dir, so removing it is enough. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 7c8662e6e3c..05c010c5529 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -2,6 +2,7 @@ name: "doctor-local-only" command: "azd ai agent doctor --local-only" cwd: "~/working/azd-agents-doctor-empty-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir for a deterministic local-only run. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml index bd9db3b63bd..eca212c3f81 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml @@ -2,6 +2,7 @@ name: "help-root" command: "azd ai agent --help" cwd: "/tmp" +tags: ["tier:0", "cmd:help", "parallel-safe"] goals: - "Wait for the help output to render (it includes an ASCII-art banner and a 'Usage:' section)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index d9907dde3c8..6e11833c869 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -6,6 +6,7 @@ name: "init-picker-navigation" command: "azd ai agent init" cwd: "~/working/azd-agents-picker-{instance}" +tags: ["tier:0", "cmd:init", "picker", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 5d5df023e7a..fafecf07551 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -6,6 +6,7 @@ name: "init-validate-mutually-exclusive" command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" cwd: "~/working/azd-agents-validate-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index d5a46e55cb1..dd8e1b3c313 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -2,6 +2,7 @@ name: "init-validate-no-prompt-missing" command: "azd ai agent init --no-prompt" cwd: "~/working/azd-agents-validate-noprompt-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml index a3679f6ec8b..55711345ff0 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -2,6 +2,7 @@ name: "sample-list-json-filters" command: "bash" cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml index 51b05c29be6..76323265ec5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -2,6 +2,7 @@ name: "sample-list-text" command: "azd ai agent sample list" cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Wait for the curated sample catalog to render as human-readable text." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml index a58a88b0076..b25e9ae8392 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml @@ -2,6 +2,7 @@ name: "version" command: "azd ai agent version" cwd: "/tmp" +tags: ["tier:0", "cmd:version", "parallel-safe"] goals: - "Wait for the command to print a version string (e.g. a value like '0.1.x-preview' or a commit-based 'vdev' build identifier)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 501f2fdfb32..cd46adb445c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -8,6 +8,7 @@ name: "init-deploy-mode-code" command: "azd ai agent init --deploy-mode code" cwd: "~/working/azd-agents-t1-code-deploy-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index cfc34c166fe..81a29eb4e44 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -9,6 +9,7 @@ name: "init-flags-agent-name-model" command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" cwd: "~/working/azd-agents-t1-flags-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index a34ed68a941..84fbcbb5b47 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -9,6 +9,7 @@ name: "init-from-code" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-from-code-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 155477a364d..e326894f897 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -9,6 +9,7 @@ name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 842de2de13a..74b4bf1e555 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -5,6 +5,7 @@ name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index b6258a96d07..da845744fc7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -8,6 +8,7 @@ name: "init-template-python" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-python-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 039c7acaea7..94234d7c8b1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -12,6 +12,7 @@ name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" +tags: ["tier:2", "cmd:init", "cmd:provision", "cmd:deploy", "serial-only"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index d71400568c9..8b565759c65 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -4,6 +4,7 @@ name: "show-json" command: "azd ai agent show --output json" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index 78e0a131bdc..7766378d8e7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -4,6 +4,7 @@ name: "show" command: "azd ai agent show" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 74109e0c82f..cb4a19233fb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -4,6 +4,7 @@ name: "invoke-input-file" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index 839c37dea8b..8d38ef84524 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -9,6 +9,7 @@ name: "invoke-session-vs-conversation" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 0f42731b15c..5ed85dd4c08 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -4,6 +4,7 @@ name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index 2b633eef537..4c8068b1ff4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -5,6 +5,7 @@ name: "sessions-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:sessions", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index 6d1ebbcfa92..bed637575e2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -6,6 +6,7 @@ name: "files-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:files", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index 479db222209..4c7aabf36d1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -5,6 +5,7 @@ name: "monitor-console" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 8b6ebb63acf..0aa08b4dd72 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -5,6 +5,7 @@ name: "monitor-system" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 705f2ccee21..0ac9922bb4b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -4,6 +4,7 @@ name: "endpoint-update" command: "azd ai agent endpoint update" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:endpoint", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index 86d9d66a112..325fbe0c67d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -6,6 +6,7 @@ name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:run", "cmd:invoke", "serial-only"] # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index af2dea47182..2b8d289ba5f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -4,6 +4,7 @@ name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:doctor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index 45e9eb6d4b1..c2c845f1ec5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -5,6 +5,7 @@ name: "teardown-down" command: "azd down --force --purge" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:down", "serial-only"] goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index e89a211e8cd..92a50795e44 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -23,23 +23,7 @@ shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` call** — the scenario YAMLs reference those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, -`{tenant}` (optional), and `{shared_agent_name}` placeholders. Example -prompt: - -``` -Use the cli-interactive-tester to drive the scenario at -tests/cli-interactive-tester-scenarios/00-version.yaml. - -First, read tests/cli-interactive-tester-scenarios/profile.yaml and -profile.local.yaml and merge them (local overrides shared); also derive -shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as -session_vars on every load_scenario / run_pre_hooks / start_session / -run_post_hooks call. - -If the scenario declares pre hooks, run them first; then start the session, -accomplish the goals, take screenshots at each step, and run any post hooks -after finishing. -``` +`{tenant}` (optional), and `{shared_agent_name}` placeholders. Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting the working dir or seeding a fixture), and a few declare **`post:` hooks** @@ -74,6 +58,9 @@ After all of these scenarios are run, create a final result report. Create a plan to accomplish this ``` +For more selective fan-outs (e.g. "just the `init` scenarios" or "everything +in Tier 0") the tester's `list_scenarios` MCP tool filters by `tags:`. See +[Tags](#tags) below for the taxonomy and an example tag-filtered prompt. ## Paths run inside WSL (on Windows) @@ -297,7 +284,10 @@ its bugs: ## Tiers -Scenarios are organized into three tiers by cost and prerequisites. +Scenarios are organized into three tiers by cost and prerequisites. Each +scenario also carries a `tags:` list that exposes the same axes plus the +command(s) under test — see [Tags](#tags) for the full taxonomy and how to +filter via `list_scenarios`. ### Tier 0 — Offline (prefix `00-`) No Azure auth, no network resource creation. Fast and deterministic. Safe to run @@ -359,6 +349,88 @@ as their `cwd`. | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | +## Tags + +Every scenario carries a top-level `tags:` list so an orchestrator can pick +subsets via the tester's `list_scenarios` MCP tool. The tool's filter is **OR +across the requested tags, case-sensitive, exact match**: a scenario matches +when its `tags` contains at least one of the requested values. + +Three namespaces are used (all lowercase, kebab-case, colon-separated for +grouping — colons are treated as ordinary characters by the filter): + +| Namespace | Values | Meaning | +|---|---|---| +| `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | + +**Examples** (the tool's `tags:` parameter is OR across the list): + +| Goal | `list_scenarios(tags=…)` | +|---|---| +| All `init` scenarios across every tier | `["cmd:init"]` | +| Everything offline (no Azure auth, no cost) | `["tier:0"]` | +| All Tier 2 cloud scenarios | `["tier:2"]` | +| Invoke + sessions reuse scenarios | `["cmd:invoke", "cmd:sessions"]` | +| CLI arg-validation scenarios only | `["negative-path"]` | +| Everything safe to run in parallel | `["parallel-safe"]` | + +Sample prompt that uses tag filtering: + +``` +Use the cli-interactive-tester to run every `init` scenario across all tiers. + +First, call list_scenarios with root="tests/cli-interactive-tester-scenarios" +and tags=["cmd:init"] to enumerate the matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and +profile.local.yaml and merge them (local overrides shared); also derive +shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call. + +For each scenario returned by list_scenarios: load it, run any pre hooks, +start the session and accomplish the goals (take screenshots at each step), +finish the session, run any post hooks. The Tier 0/1 `init` scenarios are +parallel-safe (also tagged `parallel-safe`); fan them out via fleet mode. +The Tier 2 `init` scenario (`20-setup-deploy-shared-agent`) is `serial-only` +— run it on its own and only if I confirm I want to spend on Azure resources. +``` + +You can also get copilot to generate the tags list instead of manually specifying +it. For example, if you want to run all of the scenarios to test the changes +in a PR, modify the above prompt to start with something like: + +``` +Here's a PR: https://github.com/Azure/azure-dev/pull/8532. In the +tests\cli-interactive-tester-scenarios directory, there are a set of test scenarios, +with tags to categorize what they're testing. I want you to come up with a set of +tags which, when used to select these test scenarios, would properly test the +changes made by the PR provided. + +Next, call list_scenarios with those tags, to enumerate matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and .... + +``` + +And, if you're running these scenarios as a part of creating or reviewing a PR, +you can ask copilot to generate a summary report and add it as a comment directly +on the pull request. + +When adding a new scenario, give it a `tags:` list that follows this +taxonomy: at minimum a `tier:N`, at least one `cmd:*`, and either +`parallel-safe` or `serial-only`. `list_scenarios` prints `tags: []` for any +file missing a `tags:` field, so an empty list in its output signals a +regression to fix. + +> `list_scenarios` walks every `*.yaml` under the directory, including +> `profile.yaml` / `profile.local.yaml` (which surface as `(unnamed)` with +> `tags: (none)`). Filter by any `tier:*` / `cmd:*` / trait tag to exclude +> them — they intentionally carry no tags because they are configuration, +> not scenarios. + ## Profile / overrides Developer- and environment-specific values (subscription, region, model, From ea2bac4de33930d9a2977e74c9eb9d7a9233b05e Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:24:51 -0700 Subject: [PATCH 11/42] Agents.md update, to direct people to the testing Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index b03e3847b98..3ff5a3e6ec9 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -38,6 +38,46 @@ replace github.com/azure/azure-dev/cli/azd => ../../ That `replace` points this extension at your local `cli/azd` checkout instead of the version in `go.mod`. Do not merge the extension with that `replace` still present. +## Interactive CLI test scenarios + +This extension ships a suite of goal-based scenarios for the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server under `tests/cli-interactive-tester-scenarios/`. They drive real +`azd ai agent` flows end-to-end (init, provision, deploy, invoke, run, sessions, +files, monitor, endpoint, doctor, down) and are organized by tier: + +- **Tier 0** — offline, no Azure auth, no cost (help, version, validation, picker UX) +- **Tier 1** — local-only with Azure auth (init flows) +- **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) + +Each scenario caries a set of tags based on what is being tested and how. +See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, +profile setup, and orchestration rules. + +### Guidance for coding agents + +These scenarios are **never run automatically** — they require the +cli-interactive-tester MCP server, a populated `profile.local.yaml`, and +(for Tier 2) real Azure resources. Do not invoke them on your own. Instead: + +1. **Surface them to the user** when you make a change that touches a + user-facing command path covered by an existing scenario (anything under + `internal/cmd/` that maps to a `cmd:*` tag, or shared helpers used by those + commands). In your summary, point the user at the relevant scenario(s) + and suggest they run the tester against the matching tag set to validate + the change. + +2. **Add or update a scenario** when your change introduces a new command, + flag, prompt, or user-visible flow — or meaningfully alters an existing + one. Place the new YAML alongside the others, follow the tagging taxonomy + documented in the scenarios README, and mention the new/changed scenario + in the PR description so reviewers know to exercise it. + +3. **Do not modify scenarios to match buggy behavior.** Scenarios are + user-facing specifications of how the command should behave; if a scenario + fails because of your change, prefer fixing the code unless the behavior + change is intentional and documented. + ## Error handling This extension uses `internal/exterrors` so the azd host can show a useful message, attach an optional suggestion, and emit stable telemetry. From 8a63c1d3363563de5c38d277b551b28780a05cd5 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:32:41 -0700 Subject: [PATCH 12/42] cspell Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/cspell.yaml | 1 + .../tests/cli-interactive-tester-scenarios/README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 2d5c10d89d6..f6a93b191c0 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -41,6 +41,7 @@ words: - CLIENTSECRET - curr - dataagent + - defaultyourvalue - envkey - exterrors - goyaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 92a50795e44..b0ed50c84ef 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -48,7 +48,7 @@ run_post_hooks call — the scenarios reference {prefix}, {subscription}, {regio {tenant} (optional), and {shared_agent_name} placeholders. I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios -in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios +in tiers 0 and 1 are completely independent of each other and can be run in parallel. The scenarios in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make sure to take that into account when distributing the work. I want to run all of the tests regardless of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. From 4793e9ff5e7cb4e8cf72830aad8ff1c789cd90f7 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:50:54 -0700 Subject: [PATCH 13/42] PR comments Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 2 +- .../00-doctor-local-only.yaml | 4 ++-- .../profile.local.yaml.example | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index 3ff5a3e6ec9..e0d8712aed4 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -50,7 +50,7 @@ files, monitor, endpoint, doctor, down) and are organized by tier: - **Tier 1** — local-only with Azure auth (init flows) - **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) -Each scenario caries a set of tags based on what is being tested and how. +Each scenario carries a set of tags based on what is being tested and how. See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, profile setup, and orchestration rules. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 05c010c5529..dfba27f8b89 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -1,12 +1,12 @@ # Tier 0 (offline) — `doctor --local-only` skips remote checks. name: "doctor-local-only" command: "azd ai agent doctor --local-only" -cwd: "~/working/azd-agents-doctor-empty-{instance}" +cwd: "~/working/azd-agents-doctor-local-{instance}" tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir for a deterministic local-only run. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" + - run: "rm -rf ~/working/azd-agents-doctor-local-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example index 234b8f217ff..e20187f0c14 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example @@ -20,7 +20,7 @@ prefix: "your-alias" # prompt for one. Tier 1 and Tier 2 scenarios both read this. subscription: "azd ai agent development" -# OPTIONAL. Tenant for `az login --tenant ` (and `gh auth login`). +# OPTIONAL. Tenant for `az login --tenant `. # When unset, the driving agent omits the --tenant flag entirely and relies # on your default tenant. Set this only if you actually need to scope to a # specific tenant. From b3ce2e3c440232b0ee18431a412466cf4421bfcd Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Mon, 8 Jun 2026 11:58:12 -0400 Subject: [PATCH 14/42] Add agent-scenario-tests skill for local PR scenario regression checks Adds a workflow skill under .github/skills/agent-scenario-tests/ that resolves the current branch's PR, maps changed files to impacted cli-interactive-tester scenario tags, drives the matching scenarios through the tester MCP server, and posts a results comment on the PR. Cost-aware: Tier 2 runs only after explicit user confirmation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/agent-scenario-tests/SKILL.md | 73 ++++++++++++++++ .../references/impact-mapping.md | 72 ++++++++++++++++ .../references/prerequisites.md | 45 ++++++++++ .../references/reporting.md | 51 ++++++++++++ .../references/running-scenarios.md | 72 ++++++++++++++++ .../references/workflow.md | 83 +++++++++++++++++++ 6 files changed, 396 insertions(+) create mode 100644 .github/skills/agent-scenario-tests/SKILL.md create mode 100644 .github/skills/agent-scenario-tests/references/impact-mapping.md create mode 100644 .github/skills/agent-scenario-tests/references/prerequisites.md create mode 100644 .github/skills/agent-scenario-tests/references/reporting.md create mode 100644 .github/skills/agent-scenario-tests/references/running-scenarios.md create mode 100644 .github/skills/agent-scenario-tests/references/workflow.md diff --git a/.github/skills/agent-scenario-tests/SKILL.md b/.github/skills/agent-scenario-tests/SKILL.md new file mode 100644 index 00000000000..5f36a321f22 --- /dev/null +++ b/.github/skills/agent-scenario-tests/SKILL.md @@ -0,0 +1,73 @@ +--- +name: agent-scenario-tests +license: MIT +metadata: + version: "1.0" + # Bump major on breaking prompt/trigger changes; bump minor on new references or mapping rules. +description: >- + **WORKFLOW SKILL** — Runs the azure.ai.agents extension's cli-interactive-tester + scenarios locally as a PR regression check. Resolves the current branch's PR, + maps changed files to impacted scenario tags, drives the matching scenarios + through the cli-interactive-tester MCP server, and posts a results comment on the PR. + + INVOKES: git CLI, gh CLI, cli-interactive-tester MCP tools (list_scenarios, + load_scenario, run_pre_hooks, start_session, send_action, finish_session, + run_post_hooks), ask_user. + + USE FOR: run agent scenarios, scenario regression check, cli-interactive-tester, + test agents extension PR, run impacted scenarios, check agents extension for regressions, + agent scenario tests, validate azure.ai.agents change. + + DO NOT USE FOR: azd core preflight (use azd-preflight), changelog (use changelog-generation), + creating PRs (use pull-request), authoring brand-new scenarios from scratch without a code + change, running scenarios for any extension other than azure.ai.agents. +--- + +# agent-scenario-tests + +Runs the `azure.ai.agents` extension's interactive CLI scenarios as a **local** +PR regression gate and reports the results back on the pull request. + +## Overview + +The `azure.ai.agents` extension ships goal-based scenarios for the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server under `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. +These scenarios are **never run in CI** — they need the tester MCP server, a populated +`profile.local.yaml`, manual `az`/`gh` login, and (for Tier 2) real Azure resources. + +This skill is the **opt-in, run-locally** flow a PR submitter uses to check their change +for regressions. It: + +1. Resolves the current branch's PR link. +2. Maps the PR's changed files to the impacted scenario **tag set** (`cmd:*` / `tier:*`). +3. Enumerates and drives only the impacted scenarios through the tester. +4. Posts a per-scenario results comment back on the PR. + +It is cost- and side-effect-aware: Tier 0 is free/offline, Tier 1 needs Azure auth but +provisions nothing, and **Tier 2 incurs Azure cost and is only run after explicit user +confirmation**. + +> This skill drives scenarios **deliberately, with user consent**. That is different from +> the extension's `AGENTS.md` rule that coding agents must not invoke scenarios on their +> own during ordinary work — here the user has explicitly asked for a scenario run. + +{{ references/prerequisites.md }} + +{{ references/workflow.md }} + +{{ references/impact-mapping.md }} + +{{ references/running-scenarios.md }} + +{{ references/reporting.md }} + +## Exit Criteria + +- The current branch's PR was resolved (or the user supplied one / chose to skip the comment). +- The impacted scenario set was derived from the PR diff and **confirmed by the user** + (including an explicit cost acknowledgement before any Tier 2 run). +- Every selected scenario was driven to completion with a recorded PASS/FAIL, duration, and + any findings, and a `FINAL-REPORT.md` was written under `.reports//`. +- A results comment was posted on the PR (unless the user opted out), and any Tier 2 run was + followed by `2Z-teardown-down` so no Azure resources are left running. diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/agent-scenario-tests/references/impact-mapping.md new file mode 100644 index 00000000000..670b3625265 --- /dev/null +++ b/.github/skills/agent-scenario-tests/references/impact-mapping.md @@ -0,0 +1,72 @@ + +# Impact mapping — changed files → scenario tags + +Goal: from a PR's changed-file list, derive the **smallest** scenario tag set that still +covers the change, plus the tier ceiling for cost gating. + +All paths below are relative to `cli/azd/extensions/azure.ai.agents/`. + +## 1. Command source → `cmd:*` tag + +Files under `internal/cmd/` map to the command they implement: + +| Changed file (glob) | Tag(s) | Notes | +| --- | --- | --- | +| `internal/cmd/init*.go` | `cmd:init` | Includes `init_from_code*`, `init_from_templates*`, `init_models`, `init_locations`, `init_validate`, `init_copy`, `init_foundry_resources_helpers`. | +| `internal/cmd/show.go` | `cmd:show` | | +| `internal/cmd/invoke*.go` | `cmd:invoke` | `invoke.go`, `invoke_raw.go`. | +| `internal/cmd/run.go` | `cmd:run` | | +| `internal/cmd/session.go` | `cmd:sessions` | | +| `internal/cmd/files.go` | `cmd:files` | | +| `internal/cmd/monitor*.go` | `cmd:monitor` | `monitor.go`, `monitor_format.go`. | +| `internal/cmd/update.go` | `cmd:endpoint` | `update.go` defines `endpoint update`. | +| `internal/cmd/doctor*.go` | `cmd:doctor` | `doctor.go`, `doctor_format.go`. | +| `internal/cmd/sample*.go` | `cmd:sample` | `sample.go`, `sample_list.go`. | +| `internal/cmd/version.go` | `cmd:version` | | +| `internal/cmd/root.go` | `cmd:help` + broad | Touches the whole command tree — treat as broad (see §3). | +| `internal/cmd/listen.go` | — | gRPC host entrypoint; not scenario-testable. | + +## 2. Changed command with NO scenario coverage (gaps) + +These commands have **no** scenario in the suite yet. If the PR touches them, you cannot +run a regression check — **report the gap** and recommend the author add a scenario +(per the extension `AGENTS.md`), rather than silently passing: + +| Changed file (glob) | Uncovered command | +| --- | --- | +| `internal/cmd/eval*.go` | `eval` (init / run / update / list / show) | +| `internal/cmd/optimize*.go` | `optimize` (status / list / cancel / apply / deploy) | +| `internal/cmd/mcp.go` | `mcp start` (hidden/preview) | + +## 3. Shared / cross-cutting code → broaden + +Changes outside a single command file affect many flows. When the diff touches any of +these, broaden the impacted set (and ask the user how wide to go): + +| Changed file (glob) | Broaden to | +| --- | --- | +| `internal/cmd/helpers.go`, `internal/cmd/agent_context.go`, `internal/cmd/*_context.go` | All `cmd:*` for commands that resolve project/agent context — at minimum `cmd:init`, `cmd:invoke`, `cmd:show`, `cmd:doctor`. | +| `internal/cmd/root.go`, `internal/cmd/banner.go`, `internal/cmd/nextstep_output.go` | Run a Tier 0 smoke set (`tier:0`) across all commands. | +| `internal/pkg/**`, `internal/project/**`, `internal/exterrors/**` | Map by what the package feeds: parsers/manifests → `cmd:init`; deployment/project target → `cmd:provision` + `cmd:deploy` (Tier 2). When unclear, propose a Tier 0/1 sweep and ask before any Tier 2. | +| `go.mod` / `go.sum` / dependency bumps | Tier 0 smoke + ask whether a fuller sweep is warranted. | +| files **outside** `cli/azd/extensions/azure.ai.agents/` (e.g. `cli/azd/` core) | This skill is scoped to the agents extension; note that core changes may need core azd testing instead, and proceed only with the agents-relevant subset. | + +## 4. Tier ceiling (cost gate) + +From the impacted `cmd:*` set, decide the **highest tier to offer**: + +- Default to **Tier 0 + Tier 1** for any change to a covered command (free + auth-only). +- Offer **Tier 2** only when the change can plausibly affect cloud behavior — i.e. it + touches `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, + `cmd:show`, `cmd:run`, `cmd:doctor` provisioned paths, deployment/project code, or the + provision/deploy flow. Tier 2 always requires the explicit cost confirmation in + `workflow.md` Step 4. +- A pure Tier 0 change (e.g. `version.go`, help text, `sample list` formatting) should run + Tier 0 only. + +## 5. Translate tags → run list + +Combine the derived `cmd:*` tags with the chosen tier tags and call +`list_scenarios(tags=[...])`. Example: an `invoke.go` change approved for Tier 2 → +`list_scenarios(tags=["cmd:invoke"])`, then keep the Tier 0/1 results plus the Tier 2 +`22-*` scenarios, prefixed by `20-setup` and suffixed by `2Z-teardown`. diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md new file mode 100644 index 00000000000..20808b7ff2c --- /dev/null +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -0,0 +1,45 @@ +# Prerequisites + +Verify these before doing anything else. If a hard prerequisite is missing, stop and tell +the user exactly what to fix — do **not** try to work around it. + +### Repo location + +1. Locate the scenarios directory: + `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. + Resolve its absolute path; everything else is relative to it. +2. Note the **WSL path** of that directory for MCP tool arguments. On Windows hosts the + tester runs inside WSL, so a Windows path like + `C:\Repos\azure-dev\...\scenarios\00-version.yaml` must be passed as + `/mnt/c/Repos/azure-dev/.../scenarios/00-version.yaml`. On macOS/Linux use the native + absolute path. See `running-scenarios.md` § Path style. + +### Tooling + +| Requirement | Check | If missing | +| --- | --- | --- | +| `git` + `gh` CLIs | `gh auth status` | Ask the user to run `gh auth login`. | +| cli-interactive-tester MCP server | The `list_scenarios` / `start_session` MCP tools are available to you | Stop. Tell the user to register the cli-interactive-tester MCP server (see its README) and re-run. | +| `profile.local.yaml` | File exists in the scenarios dir | Stop. Tell the user to `cp profile.local.yaml.example profile.local.yaml` and set `prefix` + `subscription`. | + +### Auth (tier-dependent — only enforce for tiers actually selected) + +- **Tier 0** needs no auth. +- **Tier 1 / Tier 2** read from / write to Azure. A human must `az login` inside WSL + **before** the run (the agent cannot complete the browser sign-in). If the selected set + includes Tier 1/2, remind the user to `az login` first. +- **Manifest scenarios** (`10-init-from-manifest-url`, `10-init-flags-agent-name-model`) + download from GitHub and can fall back to the `gh` CLI; they need `gh auth login` inside + WSL. Their `pre` hook fails fast if it isn't set up. + +### Profiles + +The scenarios reference `{prefix}`, `{subscription}`, `{region}`, `{model}`, `{tenant}` +(optional) and `{shared_agent_name}` via placeholders. You must: + +1. Read both `profile.yaml` (checked-in defaults) and `profile.local.yaml` (developer + overrides) and **merge them, local overriding shared**. +2. Derive `shared_agent_name = "{prefix}-{shared_agent_suffix}"`. +3. Pass the merged map as `session_vars` on **every** `load_scenario`, `run_pre_hooks`, + `start_session`, and `run_post_hooks` call. Omitting it leaves placeholders unresolved + and the run executes against literal `{prefix}` strings. diff --git a/.github/skills/agent-scenario-tests/references/reporting.md b/.github/skills/agent-scenario-tests/references/reporting.md new file mode 100644 index 00000000000..186f7c66f9e --- /dev/null +++ b/.github/skills/agent-scenario-tests/references/reporting.md @@ -0,0 +1,51 @@ +# Reporting + +Two outputs: a local `FINAL-REPORT.md` artifact and a PR comment. + +## FINAL-REPORT.md + +Write to `/.reports//FINAL-REPORT.md` (the `.reports/` tree is +git-ignored). Include: + +- Run header: timestamp, PR number/URL, branch, base ref, the derived tag set, and the + tiers actually run. +- A per-tier table of scenarios with columns: `Scenario | Tier | Result | Duration | Findings`. +- A short "Coverage gaps" section listing any changed command(s) with no scenario (from + `impact-mapping.md` §2), so the author knows to add one. +- Links to the per-scenario `tester-reports//` folders for screenshots/HTML. + +## PR comment + +Post with `gh pr comment --body-file ` (use a temp file to preserve +formatting). Keep it scannable — full detail lives in the artifact. Suggested shape: + +```markdown +## 🧪 Agent scenario regression check + +**Branch:** `` → `` · **Run:** `` +**Impacted tags:** `cmd:invoke`, `cmd:sessions` · **Tiers run:** 0, 1, 2 + +| Scenario | Tier | Result | Duration | +| --- | --- | --- | --- | +| 00-version | 0 | ✅ PASS | 4s | +| 22-invoke-remote | 2 | ✅ PASS | 1m 12s | +| 22-invoke-new-session | 2 | ❌ FAIL | 1m 40s | + +**Findings** +- `22-invoke-new-session`: `--new-conversation` still recalled the prior name — memory + was not reset. (screenshot: …) + +**Coverage gaps:** this PR also touches `eval*.go`, which has no scenario — consider adding one. + +Run locally via the `agent-scenario-tests` skill. Not run in CI. +``` + +Rules: + +- Use ✅ PASS / ❌ FAIL (and ⚠️ for a scenario that completed but raised a non-fatal finding). +- **Never** soften a real regression to make the table green. A scenario that failed because + of the PR's change is a FAIL — report it and recommend fixing the code, not the scenario. +- If the user opted out of posting (or there's no PR), write only the artifact and print the + summary to the user instead. +- Mention any Tier 2 teardown status explicitly (e.g. "`2Z-teardown-down` ran — no resources + left provisioned") so the reader knows nothing is still costing money. diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md new file mode 100644 index 00000000000..243c589513c --- /dev/null +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -0,0 +1,72 @@ + +# Running scenarios through the tester + +This mirrors the scenarios `README.md` ("Driving conventions" and "Orchestrating a fleet +run"). Follow it so the run actually *tests* the CLI instead of papering over its bugs. + +## Path style (Windows → WSL) + +On Windows the tester drives CLIs through tmux **inside WSL**, and it resolves every +path-shaped MCP argument on the WSL side. Pass POSIX paths: + +| Orchestrator OS | Pass to MCP tools | +| --- | --- | +| Windows | `/mnt/c/Repos/azure-dev/.../scenarios/00-version.yaml` | +| macOS / Linux | native absolute path | + +This applies to `path:` on `load_scenario` / `run_pre_hooks` / `run_post_hooks` and to +`scenario_path:` on `start_session`. If `load_scenario` returns `Scenario file not found`, +the path style is almost certainly the cause — translate `C:\…` → `/mnt/c/…` and retry once +before fanning out. + +## Per-scenario loop + +For each selected scenario: + +1. `load_scenario(path=, session_vars=)` — also tells you whether + the scenario declares `pre`/`post` hooks. +2. If it has `pre` hooks: `run_pre_hooks(path=…, session_vars=…)`. Hooks run host-side, + sequentially, fail-fast (unless `continue_on_error: true`). +3. `start_session(scenario_path=…, session_vars=…, run_name=, output_dir=)`. + - `run_name` = the YAML filename without `.yaml` (e.g. `00-version`, `21-show-json`). + - For scenarios that start two sessions (`27-run-local-and-invoke-local`), suffix the + `run_name` with a role tag (`…-run`, `…-invoke`). + - `output_dir` = WSL path of `/.reports//tester-reports`. + Reuse the **same** `` across every scenario in the run. +4. Drive the session's `goals:` with `send_action` / `select_by_text` / screenshots, then + `finish_session`. +5. If it has `post` hooks: `run_post_hooks(path=…, session_vars=…)`. + +## Driving conventions (fail-loud) + +- **Don't verify/retry after a `select`.** Reading back the echo and "correcting" a pick + hides the very picker defect the test exists to catch. Send the action and let downstream + prompts surface any failure. +- **Treat a select miss as a hard failure.** `select_by_text` is fail-loud + (`ERROR during 'select': …`). Report a finding and stop that scenario — do **not** retry + with a different `choice_text`/`choice_index`. +- **Prefer `choice_text` over `choice_index`** (indices shift between releases). +- **Clear a pre-filled text field before typing** (e.g. the agent-name prompt); otherwise + your value *appends* to the default (`defaultyourvalue`). +- **Pause before the first cloud-creating action.** The Step 4 cost confirmation covers + this; never enter a Tier 2 provision flow without it. + +## Parallelism & ordering + +- **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), one + sub-agent per scenario, each with a distinct descriptive `session_id` (e.g. + `fleet-10-init-from-code`). No `instance_id` is needed — each scenario's `cwd` already + isolates itself (defaults to the `-main` suffix). +- **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's + parallel-readiness section for which scenarios support it. +- **Tier 2** (`serial-only`): never parallelize. Run `20-setup-deploy-shared-agent` first, + then `21-…2A-` serially (they share one deployed agent and mutate shared session/file/ + endpoint state), then `2Z-teardown-down` last. +- **Validate the recipe with one scenario before fanning out** — confirm `load_scenario` → + `start_session` → one `send_action` round-trips for a single Tier 0 scenario first. + +## Capture per scenario + +Record, for the report: the scenario stem, tier, PASS/FAIL, wall-clock **duration** +(`start_session` → `finish_session` incl. hooks, formatted `Hh Mm Ss`), and any +`report_finding` text (confusing UX, errors, doc mismatches). diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md new file mode 100644 index 00000000000..4280d8f111d --- /dev/null +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -0,0 +1,83 @@ +# Workflow + +### Step 1 — Verify prerequisites + +Run the checks in `prerequisites.md`. If a hard prerequisite is missing, stop with a clear +message. Don't auto-install or work around a missing MCP server or profile. + +### Step 2 — Resolve the PR + +```bash +gh pr view --json number,url,headRefName,baseRefName,title +``` + +- **PR found:** capture `number`, `url`, and `baseRefName` (the merge base for the diff). +- **No PR for the current branch:** ask the user via `ask_user` whether to (a) supply a PR + number/URL, (b) run against the local diff vs `origin/main` without posting a comment, or + (c) abort. + +### Step 3 — Compute the impacted scenario tag set + +1. Get the changed files: + + ```bash + gh pr diff --name-only # when a PR exists + # or, for a local-only run: + git diff --name-only origin/main...HEAD + ``` + +2. Map those files to scenario tags using `impact-mapping.md`. The result is: + - a set of `cmd:*` tags (which commands changed), + - the **highest tier** you should offer (cost gating), and + - any **coverage gaps** (changed commands that have *no* scenario yet — e.g. `eval`, + `optimize`, `mcp`). Surface gaps to the user; do not silently skip them. + +3. Enumerate matching scenarios via the tester: + + ```text + list_scenarios(root="", tags=[, ...]) + ``` + + `list_scenarios` filtering is **OR across tags, case-sensitive, exact match**. + +### Step 4 — Confirm the plan (cost gate) + +Show the user the concrete scenario list grouped by tier, plus estimated cost/auth needs, +and confirm via `ask_user` before running: + +- Always list the Tier 0 scenarios that will run (free). +- If the set includes **Tier 1**, confirm `az login` is done. +- If the set includes **Tier 2**, require an **explicit cost acknowledgement** ("Tier 2 + provisions real Azure resources and incurs cost — proceed?"). If the user declines Tier 2, + drop it and run only Tier 0/1. + +Pick one `` of the form `YYYYMMDD-HHMMSS` for the whole run. All artifacts go +under `/.reports//`. + +### Step 5 — Run the scenarios + +Drive each selected scenario per `running-scenarios.md`. Honor ordering: + +- **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each + with its own `cwd` (no `instance_id` needed for distinct scenarios). +- **Tier 2** is `serial-only` and order-dependent: `20-setup-deploy-shared-agent` **first**, + then the targeted `21-…2A-` scenarios **serially**, then `2Z-teardown-down` **last**. + +Record per scenario: PASS/FAIL, wall-clock duration (`Hh Mm Ss`), and any `report_finding` +entries. + +### Step 6 — Report + +Aggregate results into `.reports//FINAL-REPORT.md` and post a PR comment per +`reporting.md`. If a Tier 2 run started but was interrupted before `2Z-teardown`, run +`2Z-teardown-down` (or `20-setup`'s down hook) so no resources are orphaned, then report. + +### Step 7 — Stop conditions + +Stop and escalate to the user when: + +- a required prerequisite is missing (Step 1), +- the diff touches a changed command with **no** scenario coverage (note it in the report so + the user can author one — see the extension `AGENTS.md` guidance), or +- a scenario fails in a way that looks like a real product regression: report it as a FAIL + with the finding and do **not** edit the scenario to make it pass. From 5e57fe4642e8939af6f9507211d85506b2a3a082 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Tue, 9 Jun 2026 13:21:49 -0400 Subject: [PATCH 15/42] test(agents): add eval/optimize scenarios and invoke/init/doctor gap coverage Add 9 cli-interactive-tester scenarios closing the coverage gaps found in the PR #8524 review: - eval (cmd:eval): 00-eval-context-required (offline endpoint-required) and 28-eval-lifecycle (Tier 2 init/run/list/show against the shared agent). - optimize (cmd:optimize): 00-optimize-apply-requires-candidate (offline required-flag) and 29-optimize-submit-and-cancel (Tier 2, capped iteration). - invoke: 00-invoke-validate-protocol (offline unsupported-protocol) and 23-invoke-protocol-invocations (Tier 2 invocations memory semantics). - init: 00-init-validate-deploy-mode (offline value/required-flag validation) and 10-init-deploy-mode-container (Tier 1 container scaffold). - doctor: 00-doctor-partial-failure (mixed PASS+FAIL, exit 1). Add cmd:eval and cmd:optimize to the tag taxonomy, update the scenarios README tier tables, and update the agent-scenario-tests skill impact-mapping (eval and optimize are now covered Tier 2 commands, no longer listed as gaps). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../references/impact-mapping.md | 10 +++-- .../00-doctor-partial-failure.yaml | 32 ++++++++++++++++ .../00-eval-context-required.yaml | 26 +++++++++++++ .../00-init-validate-deploy-mode.yaml | 25 ++++++++++++ .../00-invoke-validate-protocol.yaml | 23 +++++++++++ .../00-optimize-apply-requires-candidate.yaml | 22 +++++++++++ .../10-init-deploy-mode-container.yaml | 38 +++++++++++++++++++ .../23-invoke-protocol-invocations.yaml | 37 ++++++++++++++++++ .../28-eval-lifecycle.yaml | 32 ++++++++++++++++ .../29-optimize-submit-and-cancel.yaml | 33 ++++++++++++++++ .../README.md | 11 +++++- 11 files changed, 284 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/agent-scenario-tests/references/impact-mapping.md index 670b3625265..aef60d48193 100644 --- a/.github/skills/agent-scenario-tests/references/impact-mapping.md +++ b/.github/skills/agent-scenario-tests/references/impact-mapping.md @@ -21,6 +21,8 @@ Files under `internal/cmd/` map to the command they implement: | `internal/cmd/monitor*.go` | `cmd:monitor` | `monitor.go`, `monitor_format.go`. | | `internal/cmd/update.go` | `cmd:endpoint` | `update.go` defines `endpoint update`. | | `internal/cmd/doctor*.go` | `cmd:doctor` | `doctor.go`, `doctor_format.go`. | +| `internal/cmd/eval*.go` | `cmd:eval` | `eval.go`, `eval_init.go`, `eval_run.go`, `eval_list.go`, `eval_show.go`, etc. Tier 2 (needs a deployed agent + Foundry endpoint). | +| `internal/cmd/optimize*.go` | `cmd:optimize` | `optimize.go`, `optimize_apply.go`, `optimize_status.go`, etc. Tier 2 (submits a cloud optimization job). | | `internal/cmd/sample*.go` | `cmd:sample` | `sample.go`, `sample_list.go`. | | `internal/cmd/version.go` | `cmd:version` | | | `internal/cmd/root.go` | `cmd:help` + broad | Touches the whole command tree — treat as broad (see §3). | @@ -34,8 +36,6 @@ run a regression check — **report the gap** and recommend the author add a sce | Changed file (glob) | Uncovered command | | --- | --- | -| `internal/cmd/eval*.go` | `eval` (init / run / update / list / show) | -| `internal/cmd/optimize*.go` | `optimize` (status / list / cancel / apply / deploy) | | `internal/cmd/mcp.go` | `mcp start` (hidden/preview) | ## 3. Shared / cross-cutting code → broaden @@ -58,8 +58,10 @@ From the impacted `cmd:*` set, decide the **highest tier to offer**: - Default to **Tier 0 + Tier 1** for any change to a covered command (free + auth-only). - Offer **Tier 2** only when the change can plausibly affect cloud behavior — i.e. it touches `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, - `cmd:show`, `cmd:run`, `cmd:doctor` provisioned paths, deployment/project code, or the - provision/deploy flow. Tier 2 always requires the explicit cost confirmation in + `cmd:show`, `cmd:run`, `cmd:eval`, `cmd:optimize`, `cmd:doctor` provisioned paths, + deployment/project code, or the provision/deploy flow. `cmd:eval` and `cmd:optimize` + are Tier 2-only (no offline happy path beyond their negative-path validation + scenarios). Tier 2 always requires the explicit cost confirmation in `workflow.md` Step 4. - A pure Tier 0 change (e.g. `version.go`, help text, `sample list` formatting) should run Tier 0 only. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml new file mode 100644 index 00000000000..1b827cb441d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml @@ -0,0 +1,32 @@ +# Tier 0 (offline) — `doctor` partial-failure path (mixed PASS + FAIL). +# +# Seeds a minimal name-only azure.yaml so the local checks produce a MIX of +# results: the grpc-extension and azure-yaml checks PASS, but later checks +# (e.g. environment-selected) FAIL, and remote checks are skipped. This must +# yield exit code 1 (at least one failure) with a clean, readable report — +# exercising the partial-failure path that the empty-dir scenario does not. +name: "doctor-partial-failure" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-doctor-partial-{instance}" +tags: ["tier:0", "cmd:doctor", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a name-only azure.yaml: enough for the azure-yaml check to PASS while +# downstream checks (no environment selected, no agent service) FAIL. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-partial-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + - run: "mkdir -p ~/working/azd-agents-doctor-partial-{instance} && printf 'name: azd-agents-doctor-partial\\n' > ~/working/azd-agents-doctor-partial-{instance}/azure.yaml" + cwd: "~/working" + name: "seed a minimal name-only azure.yaml" + +goals: + - "Run doctor in a directory containing only a minimal name-only azure.yaml. Wait for the check report to render." + - "Confirm the report shows a MIX of results: at least one check PASSES (e.g. the azd project/azure.yaml check) AND at least one check FAILS (e.g. no environment selected / no agent service detected)." + - "Confirm the command exits with code 1 (failure) — NOT 0 and NOT 2 (2 would mean all checks were skipped)." + - "Confirm the report renders cleanly with no panic or Go stack trace, and that failing checks include actionable remediation hints." + - "Take a screenshot of the doctor report." + - "Report a finding if doctor crashes, if a partial failure incorrectly reports exit 0 or 2, or if failing checks lack remediation guidance." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml new file mode 100644 index 00000000000..2a35012cb32 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml @@ -0,0 +1,26 @@ +# Tier 0 (offline) — `eval list` outside a project requires a Foundry endpoint. +# +# eval commands resolve a Foundry project endpoint. With no azd project to read +# it from, the command prompts for the endpoint. Submitting an empty value must +# fail with clear guidance (it must NOT hang, crash, or make a network call). +name: "eval-context-required" +command: "azd ai agent eval list" +cwd: "~/working/azd-agents-validate-eval-{instance}" +tags: ["tier:0", "cmd:eval", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Empty dir is a precondition: no azd project, so no endpoint can be auto-resolved. +pre: + - run: "rm -rf ~/working/azd-agents-validate-eval-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run 'eval list' in an empty directory with no azd project present." + - "When prompted for the Foundry project endpoint URL, submit an EMPTY value (just press Enter)." + - "Confirm the command exits non-zero with a clear message that a project endpoint is required (e.g. mentioning --project-endpoint or running from an azd project), rather than hanging or crashing." + - "Confirm it does NOT attempt a network call or print a stack trace." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, crashes, makes a network call before validating, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml new file mode 100644 index 00000000000..460b05fd9f0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml @@ -0,0 +1,25 @@ +# Tier 0 (offline) — `init --deploy-mode` value validation. +# +# --deploy-mode is validated early in init, before the wizard or any network +# call. An invalid value must fail fast with the allowed values. This covers the +# invalid-value path; the '--deploy-mode code --no-prompt' missing-flags path is +# validated in the same code (requires --runtime and --entry-point). +name: "init-validate-deploy-mode" +command: "azd ai agent init --deploy-mode bogus --no-prompt" +cwd: "~/working/azd-agents-validate-deploy-mode-val-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-validate-deploy-mode-val-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run 'init --deploy-mode bogus --no-prompt' in an empty directory." + - "Confirm it exits non-zero immediately with a clear message that --deploy-mode must be 'container' or 'code' (it must NOT hang or prompt despite --no-prompt)." + - "OPTIONAL second check: run 'azd ai agent init --deploy-mode code --no-prompt' and confirm it fails with a clear message that --runtime (and --entry-point) are required when using code deploy mode with --no-prompt." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, prompts despite --no-prompt, or the error does not state the allowed --deploy-mode values." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml new file mode 100644 index 00000000000..07aa92eec85 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml @@ -0,0 +1,23 @@ +# Tier 0 (offline) — `invoke --protocol` rejects an unsupported value. +# +# The protocol flag is validated before any agent resolution or network call, +# so a bogus value must fail fast with a message listing the supported protocols. +name: "invoke-validate-protocol" +command: "azd ai agent invoke --protocol bogus \"hello\"" +cwd: "~/working/azd-agents-validate-protocol-{instance}" +tags: ["tier:0", "cmd:invoke", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-validate-protocol-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run invoke with '--protocol bogus' and a prompt string." + - "Confirm it exits non-zero immediately with a clear 'unsupported protocol' message that lists the supported protocols (responses, invocations)." + - "Confirm it does NOT make a network call, prompt, or hang." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, makes a network call before validating, or the error does not name the supported protocols." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml new file mode 100644 index 00000000000..28ab26a247a --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml @@ -0,0 +1,22 @@ +# Tier 0 (offline) — `optimize` flag/required-argument validation. +# +# 'optimize apply' marks --candidate as required, so omitting it must fail fast +# with a clear cobra "required flag" error, before any prompt or network call. +name: "optimize-apply-requires-candidate" +command: "azd ai agent optimize apply" +cwd: "~/working/azd-agents-validate-optimize-{instance}" +tags: ["tier:0", "cmd:optimize", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-validate-optimize-{instance}" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run 'optimize apply' with no --candidate flag in an empty directory." + - "Confirm it exits non-zero immediately with a clear message that the required --candidate flag was not set (it must NOT hang, prompt, or make a network call)." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, prompts interactively, makes a network call, or the error does not identify the missing --candidate flag." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml new file mode 100644 index 00000000000..db6e292460b --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml @@ -0,0 +1,38 @@ +# Tier 1 (auth, scaffold only) — interactive container deploy mode. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# Targets the --deploy-mode container path, which builds/packages a container +# image (instead of prompting for entry-point + runtime as code-deploy does). +# Complements 10-init-deploy-mode-code.yaml. +name: "init-deploy-mode-container" +command: "azd ai agent init --deploy-mode container" +cwd: "~/working/azd-agents-t1-container-deploy-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a committed Python fixture so container-deploy has real source to package. +# Override the fixture location with AZD_AGENTS_FIXTURES if needed. +pre: + - run: "rm -rf ~/working/azd-agents-t1-container-deploy-{instance}" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-container-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-container-deploy-{instance}/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-container-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." + - "Wait for the tool to inspect the current directory's code with container-deploy mode selected (it should NOT prompt for an entry-point or runtime — those are code-deploy concerns)." + - "If an existing agent manifest is detected, confirm reuse." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect container-deploy mode (a Dockerfile or container build configuration is present rather than entry-point/runtime fields)." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if container mode incorrectly prompts for entry-point/runtime, fails to produce container build config, or the deploy mode is not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml new file mode 100644 index 00000000000..9a6bfaee4f0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml @@ -0,0 +1,37 @@ +# Tier 2 (cloud E2E) — `invoke --protocol invocations` and its memory semantics. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# +# NOTE: the shared agent is deployed with the "responses" protocol. This scenario +# exercises the invocations protocol path and its DISTINCT memory model: +# - memory is bound to the SESSION (reset with --new-session) +# - --new-conversation has NO effect under invocations (opposite of responses) +# If the deployed agent does not support invocations, the command should fail +# CLEANLY (clear protocol-mismatch error), which is itself a valid result to +# capture. Frame findings around clarity, not around forcing success. +name: "invoke-protocol-invocations" +command: "azd ai agent invoke --protocol invocations \"Remember the word 'banana'.\"" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run the invoke command with '--protocol invocations' against the deployed shared agent." + - "If the agent does not support the invocations protocol, confirm the command fails with a CLEAR protocol-mismatch error (not a 500, hang, or stack trace), capture it, and report a finding describing the message; then STOP." + - "If the invocations protocol IS supported and the first call succeeds, run a follow-up: 'azd ai agent invoke --protocol invocations \"What word did I ask you to remember?\"' and confirm the agent recalls 'banana' (session memory persists across calls within the same session)." + - "Run 'azd ai agent invoke --protocol invocations --new-conversation \"What word did I ask you to remember?\"' and confirm memory is STILL retained (under invocations, --new-conversation has NO effect; memory is bound to the session)." + - "Run 'azd ai agent invoke --protocol invocations --new-session \"What word did I ask you to remember?\"' and confirm memory is RESET (the agent no longer recalls 'banana')." + - "Take a screenshot of the relevant responses." + - "Report a finding if --new-conversation unexpectedly resets memory, if --new-session does NOT reset it, or if an unsupported protocol yields an unclear error." + +post: + - run: "echo 'invoke-protocol-invocations complete; no local teardown required'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "note: no local teardown required" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml new file mode 100644 index 00000000000..786049f8ddb --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml @@ -0,0 +1,32 @@ +# Tier 2 (cloud E2E) — `eval` lifecycle against the deployed shared agent. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Exercises eval init -> run -> list -> show. Uses a tiny sample budget to keep +# the run cheap and fast. The agent and Foundry project endpoint are auto-detected +# from the shared agent's azure.yaml. +name: "eval-lifecycle" +command: "azd ai agent eval init --max-samples 15 --no-wait" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:eval", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run 'eval init --max-samples 15 --no-wait' to kick off an evaluation run without blocking; the agent and project endpoint are auto-detected from azure.yaml." + - "If prompted for an evaluation dataset, accept the default or auto-generated dataset; if prompted for an eval model, accept the default." + - "Confirm the command reports that an evaluation run was created/submitted and prints an eval run identifier (because of --no-wait it should return without waiting for completion)." + - "Run 'azd ai agent eval list' and confirm the run just submitted appears in the list with a status." + - "Run 'azd ai agent eval show' (optionally passing the eval id from the list) and confirm it prints details/metrics for the run without crashing; a still-running or queued status is acceptable." + - "Take a screenshot of the eval list and eval show output." + - "Report a finding if eval init hangs despite --no-wait, the run id is not surfaced, eval list/show error out, or auth/endpoint errors occur." + +post: + - run: "echo 'eval-lifecycle complete; eval runs are managed in Foundry and incur no standing local resources'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "note: no local teardown required" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml new file mode 100644 index 00000000000..d9576601563 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -0,0 +1,33 @@ +# Tier 2 (cloud E2E) — submit an `optimize` job, then list / status / cancel it. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Optimization is an expensive iterative process, so this scenario caps it at a +# single iteration, submits with --no-wait, then immediately cancels to avoid +# incurring a full optimization run. The agent is auto-detected from azure.yaml. +name: "optimize-submit-and-cancel" +command: "azd ai agent optimize --eval-model {model} --max-iterations 1 --no-wait" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:optimize", "serial-only"] + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run 'optimize --eval-model {model} --max-iterations 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml." + - "If prompted for a dataset, accept the default or auto-generated dataset." + - "Confirm the command reports that an optimization job was submitted and prints a job identifier (because of --no-wait it should return without waiting for completion)." + - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." + - "Run 'azd ai agent optimize status ' using the job id and confirm it prints status details without crashing (a queued/running status is acceptable)." + - "Run 'azd ai agent optimize cancel ' using the job id to cancel the job, and confirm it reports the job was cancelled (or already terminal)." + - "Take a screenshot of the list / status / cancel output." + - "Report a finding if submit hangs despite --no-wait, the job id is not surfaced, list/status error out, or cancel fails to stop a still-running job." + +post: + - run: "echo 'optimize-submit-and-cancel complete; job was cancelled, no standing local resources'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "note: job cancelled, no local teardown required" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index b0ed50c84ef..7d66b3f1db5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -303,7 +303,12 @@ in any order, any time. | `00-doctor-local-only.yaml` | `doctor --local-only` | | `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | | `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | +| `00-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) | | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | +| `00-invoke-validate-protocol.yaml` | `invoke --protocol` unsupported-value error | +| `00-eval-context-required.yaml` | `eval list` outside a project requires a Foundry endpoint | +| `00-optimize-apply-requires-candidate.yaml` | `optimize apply` missing required `--candidate` | +| `00-doctor-partial-failure.yaml` | `doctor` mixed PASS+FAIL (exit 1) on a name-only `azure.yaml` | ### Tier 1 — Auth, scaffold only (prefix `10-`) Requires Azure login (reads subscriptions/Foundry projects) but **does not @@ -318,6 +323,7 @@ and verifies the generated files, then stops before `azd provision`. | `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | | `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | | `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | +| `10-init-deploy-mode-container.yaml` | `init --deploy-mode container` (container build config) | ### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost Provisions real resources. **Run order matters:** @@ -340,12 +346,15 @@ as their `cwd`. | `22-invoke-remote.yaml` | `invoke` (remote) | | `22-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | | `22-invoke-input-file.yaml` | `invoke -f ` | +| `23-invoke-protocol-invocations.yaml` | `invoke --protocol invocations` (session-bound memory; `--new-session` resets, `--new-conversation` no-op) | | `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | | `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | | `25-monitor-console.yaml` | `monitor` (console) | | `25-monitor-system.yaml` | `monitor --type system` | | `26-endpoint-update.yaml` | `endpoint update` | | `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | +| `28-eval-lifecycle.yaml` | `eval init/run/list/show` against the shared agent (small sample budget, `--no-wait`) | +| `29-optimize-submit-and-cancel.yaml` | `optimize` submit + `list`/`status`/`cancel` (capped at 1 iteration, `--no-wait`) | | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | @@ -362,7 +371,7 @@ grouping — colons are treated as ordinary characters by the filter): | Namespace | Values | Meaning | |---|---|---| | `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | -| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | | traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | **Examples** (the tool's `tags:` parameter is OR across the list): From 5a5298297d098e090e80a6f314143ddd6addbef7 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Tue, 9 Jun 2026 13:54:15 -0400 Subject: [PATCH 16/42] test(agents): fix init deploy-mode validation scenario to be reachable A local scenario run revealed that 00-init-validate-deploy-mode never exercised the --deploy-mode validation: in an empty directory with --no-prompt, init fails earlier with 'template selection requires interactive mode' because validateCodeDeployInput is only reached after an init method resolves. Reclassify Tier 0 -> Tier 1, rename to 10-init-validate-deploy-mode.yaml, and seed the from-code fixture so the from-code method resolves and the bogus --deploy-mode value is actually rejected ('--deploy-mode must be container or code'). Reaching the check scaffolds a starter template (network), hence Tier 1. Note the late-validation UX (template scaffolded before the flag is validated) as a report_finding. Update the README tier tables accordingly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../00-init-validate-deploy-mode.yaml | 25 ------------ .../10-init-validate-deploy-mode.yaml | 39 +++++++++++++++++++ .../README.md | 2 +- 3 files changed, 40 insertions(+), 26 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml deleted file mode 100644 index 460b05fd9f0..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-deploy-mode.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Tier 0 (offline) — `init --deploy-mode` value validation. -# -# --deploy-mode is validated early in init, before the wizard or any network -# call. An invalid value must fail fast with the allowed values. This covers the -# invalid-value path; the '--deploy-mode code --no-prompt' missing-flags path is -# validated in the same code (requires --runtime and --entry-point). -name: "init-validate-deploy-mode" -command: "azd ai agent init --deploy-mode bogus --no-prompt" -cwd: "~/working/azd-agents-validate-deploy-mode-val-{instance}" -tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] - -env: - AZD_DISABLE_AGENT_DETECT: "1" - -pre: - - run: "rm -rf ~/working/azd-agents-validate-deploy-mode-val-{instance}" - cwd: "~/working" - name: "reset to an empty working dir" - -goals: - - "Run 'init --deploy-mode bogus --no-prompt' in an empty directory." - - "Confirm it exits non-zero immediately with a clear message that --deploy-mode must be 'container' or 'code' (it must NOT hang or prompt despite --no-prompt)." - - "OPTIONAL second check: run 'azd ai agent init --deploy-mode code --no-prompt' and confirm it fails with a clear message that --runtime (and --entry-point) are required when using code deploy mode with --no-prompt." - - "Take a screenshot of the error output." - - "Report a finding if it hangs, prompts despite --no-prompt, or the error does not state the allowed --deploy-mode values." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml new file mode 100644 index 00000000000..5e2eec42f41 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml @@ -0,0 +1,39 @@ +# Tier 1 (auth, scaffold only) — `init --deploy-mode` value validation. +# +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. +# +# IMPORTANT: --deploy-mode is validated by validateCodeDeployInput, which is only +# reached AFTER init resolves an initialization METHOD. In an empty directory with +# --no-prompt there is no resolvable method, so init fails earlier with "template +# selection requires interactive mode" and the deploy-mode check never runs. To +# actually exercise the deploy-mode validation we seed a from-code fixture so the +# from-code method resolves, the starter template is scaffolded, and then the +# bogus --deploy-mode value is rejected. (Reaching the validation downloads a +# starter template, which is why this is Tier 1 rather than offline Tier 0.) +name: "init-validate-deploy-mode" +command: "azd ai agent init --deploy-mode bogus --no-prompt" +cwd: "~/working/azd-agents-validate-deploy-mode-{instance}" +tags: ["tier:1", "cmd:init", "negative-path", "parallel-safe"] + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a committed from-code fixture so init resolves the from-code method and +# reaches validateCodeDeployInput. Override the fixture location with +# AZD_AGENTS_FIXTURES if needed. +pre: + - run: "rm -rf ~/working/azd-agents-validate-deploy-mode-{instance}" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-validate-deploy-mode-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-validate-deploy-mode-{instance}/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "Run 'init --deploy-mode bogus --no-prompt' in the seeded from-code directory." + - "Allow init to resolve the from-code method and scaffold the starter template (it may print 'Template initialized')." + - "Confirm it then exits non-zero with a clear message that --deploy-mode must be 'container' or 'code' (suggestion: 'Specify --deploy-mode container or --deploy-mode code'). It must NOT hang or prompt despite --no-prompt, and must NOT provision any Azure resources." + - "OPTIONAL second check: in a freshly seeded copy of the fixture, run 'azd ai agent init --deploy-mode code --no-prompt' and confirm it fails with a clear message that --runtime (and --entry-point) are required when using code deploy mode with --no-prompt." + - "Take a screenshot of the error output." + - "STOP here — do NOT run 'azd provision'. Report a finding if it hangs, prompts despite --no-prompt, provisions resources, or the error does not state the allowed --deploy-mode values. Note as a minor finding that the starter template is scaffolded BEFORE the deploy-mode flag is validated (late validation)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 7d66b3f1db5..2711e042416 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -303,7 +303,6 @@ in any order, any time. | `00-doctor-local-only.yaml` | `doctor --local-only` | | `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | | `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | -| `00-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) | | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | | `00-invoke-validate-protocol.yaml` | `invoke --protocol` unsupported-value error | | `00-eval-context-required.yaml` | `eval list` outside a project requires a Foundry endpoint | @@ -323,6 +322,7 @@ and verifies the generated files, then stops before `azd provision`. | `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | | `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | | `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | +| `10-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) — seeds from-code so the deploy-mode check is reached | | `10-init-deploy-mode-container.yaml` | `init --deploy-mode container` (container build config) | ### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost From b36e5a3a468ff410500e3fcfdcd2b231051aee78 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Tue, 9 Jun 2026 14:54:44 -0400 Subject: [PATCH 17/42] test(agents): refine eval/optimize Tier 2 scenarios from live run findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full Tier 0+1+2 cli-interactive-tester run against a freshly deployed shared agent surfaced two scenario-accuracy issues (the CLI itself behaved correctly): - 28-eval-lifecycle: 'eval init --no-wait' is ASYNC — it submits dataset (datagen-*) and evaluator (evaluatorgen-*) generation jobs and writes eval.yaml, but does NOT create an eval 'run'. So 'eval list' legitimately shows 0 rows right after init and 'eval show' (no id) errors cleanly. Refined the header + goals to describe the async semantics and treat an empty list / eval-id-required message as expected rather than a failure. - 29-optimize-submit-and-cancel: the optimize command group is preview-gated per subscription. On a non-enrolled subscription both 'optimize' and 'optimize list' return a clean 400 SubscriptionNotRegistered (signup: aka.ms/ao/quickstart), so the submit->status->cancel lifecycle can't run. Documented the Agent Optimizer enrollment prerequisite and added a gating check that accepts the clean SubscriptionNotRegistered error as a valid outcome when enrollment is absent. Add cspell words: datagen, evaluatorgen, signup. All Tier 0 (13) and Tier 1 (2) scenarios and the Tier 2 setup/invoke/teardown passed; resources were fully torn down with azd down --force --purge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../extensions/azure.ai.agents/cspell.yaml | 4 +++ .../28-eval-lifecycle.yaml | 29 ++++++++++++------- .../29-optimize-submit-and-cancel.yaml | 18 +++++++++--- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index f6a93b191c0..3c6ab9b4ba4 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -4,6 +4,10 @@ words: - tavily - tvly - conncmd + # Eval / optimize async job ID prefixes and terms + - datagen + - evaluatorgen + - signup # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml index 786049f8ddb..9e3311cd736 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml @@ -1,9 +1,18 @@ # Tier 2 (cloud E2E) — `eval` lifecycle against the deployed shared agent. # # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -# Exercises eval init -> run -> list -> show. Uses a tiny sample budget to keep -# the run cheap and fast. The agent and Foundry project endpoint are auto-detected -# from the shared agent's azure.yaml. +# Exercises eval init -> list -> show. Uses a tiny sample budget to keep the run +# cheap and fast. The agent and Foundry project endpoint are auto-detected from +# the shared agent's azure.yaml. +# +# IMPORTANT — eval init is ASYNC: with --no-wait it submits dataset generation +# (datagen-*) and evaluator generation (evaluatorgen-*) jobs and returns +# immediately, surfacing those job IDs and writing eval.yaml. It does NOT create +# an eval "run" — `azd ai agent eval run` does that. So right after a --no-wait +# init, `eval list` will typically show 0 rows (generation still in progress) and +# `eval show` with no eval-id will error cleanly ("no eval-id provided and none +# found ..."). The goal here is to confirm each command EXECUTES and degrades +# gracefully, not that a run appears instantly. name: "eval-lifecycle" command: "azd ai agent eval init --max-samples 15 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" @@ -17,13 +26,13 @@ pre: continue_on_error: true goals: - - "Run 'eval init --max-samples 15 --no-wait' to kick off an evaluation run without blocking; the agent and project endpoint are auto-detected from azure.yaml." - - "If prompted for an evaluation dataset, accept the default or auto-generated dataset; if prompted for an eval model, accept the default." - - "Confirm the command reports that an evaluation run was created/submitted and prints an eval run identifier (because of --no-wait it should return without waiting for completion)." - - "Run 'azd ai agent eval list' and confirm the run just submitted appears in the list with a status." - - "Run 'azd ai agent eval show' (optionally passing the eval id from the list) and confirm it prints details/metrics for the run without crashing; a still-running or queued status is acceptable." - - "Take a screenshot of the eval list and eval show output." - - "Report a finding if eval init hangs despite --no-wait, the run id is not surfaced, eval list/show error out, or auth/endpoint errors occur." + - "Run 'eval init --max-samples 15 --no-wait'; the agent and project endpoint are auto-detected from azure.yaml. Accept the default eval suite name when prompted." + - "If asked how to provide the agent instruction, choose 'Type inline' and give a short description; decline including agent traces; when prompted for an eval/generation model, accept the deployed model default." + - "Confirm 'eval init' returns WITHOUT blocking (because of --no-wait) and surfaces async job identifiers — dataset generation (datagen-*) and evaluator generation (evaluatorgen-*) — and reports that eval.yaml was written." + - "Run 'azd ai agent eval list' and confirm it renders the eval table cleanly and exits 0. NOTE: because init is async, the list may legitimately show 0 rows immediately after a --no-wait init (generation still in progress, and no run has been created yet) — an empty list here is NOT a failure." + - "Run 'azd ai agent eval show' and confirm it either prints details for an eval-id or errors CLEANLY with a helpful message (e.g. 'no eval-id provided and none found ... run eval init first or pass an eval-id') and a non-zero exit — no crash or stack trace." + - "Take a screenshot of the eval init, list, and show output." + - "Report a finding ONLY if: eval init hangs despite --no-wait, the async job IDs are not surfaced, any of the commands crash / print a stack trace, or auth/endpoint errors occur. (An empty eval list or an eval-id-required message immediately after init is expected, not a failure.)" post: - run: "echo 'eval-lifecycle complete; eval runs are managed in Foundry and incur no standing local resources'" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml index d9576601563..f5dde47f2bd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -4,6 +4,16 @@ # Optimization is an expensive iterative process, so this scenario caps it at a # single iteration, submits with --no-wait, then immediately cancels to avoid # incurring a full optimization run. The agent is auto-detected from azure.yaml. +# +# ⚠️ ADDITIONAL PREREQUISITE — Agent Optimizer enrollment: the optimize command +# group is preview-gated PER SUBSCRIPTION. If the target subscription is not +# registered for Agent Optimizer, both 'optimize' (submit) and 'optimize list' +# return a clean HTTP 400 with error code 'SubscriptionNotRegistered' and a +# signup link (https://aka.ms/ao/quickstart), and the submit->status->cancel +# lifecycle cannot be exercised. To run the full happy path, first register the +# subscription for Agent Optimizer. When enrollment is absent, the CORRECT +# behavior is the clean gated error below (capture it and stop) — that is an +# acceptable outcome for this scenario, not a product failure. name: "optimize-submit-and-cancel" command: "azd ai agent optimize --eval-model {model} --max-iterations 1 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" @@ -17,14 +27,14 @@ pre: continue_on_error: true goals: - - "Run 'optimize --eval-model {model} --max-iterations 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml." - - "If prompted for a dataset, accept the default or auto-generated dataset." - - "Confirm the command reports that an optimization job was submitted and prints a job identifier (because of --no-wait it should return without waiting for completion)." + - "Run 'optimize --eval-model {model} --max-iterations 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." + - "GATING CHECK: if the command (or 'optimize list') returns a clean HTTP 400 with error code 'SubscriptionNotRegistered' (subscription not registered for Agent Optimizer, signup at https://aka.ms/ao/quickstart), this subscription is not enrolled. Confirm the error is CLEAN (structured error body, code, signup link, non-zero exit — no crash/hang), capture a screenshot, report it as an enrollment-gap finding, and STOP — the remaining list/status/cancel steps cannot run without enrollment." + - "If the job IS submitted successfully (subscription enrolled): confirm the command returns without blocking (because of --no-wait) and surfaces a job identifier." - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." - "Run 'azd ai agent optimize status ' using the job id and confirm it prints status details without crashing (a queued/running status is acceptable)." - "Run 'azd ai agent optimize cancel ' using the job id to cancel the job, and confirm it reports the job was cancelled (or already terminal)." - "Take a screenshot of the list / status / cancel output." - - "Report a finding if submit hangs despite --no-wait, the job id is not surfaced, list/status error out, or cancel fails to stop a still-running job." + - "Report a finding if submit hangs despite --no-wait, the job id is not surfaced (when enrolled), list/status error out unexpectedly, cancel fails to stop a still-running job, or any command crashes / prints a stack trace rather than a clean error." post: - run: "echo 'optimize-submit-and-cancel complete; job was cancelled, no standing local resources'" From 904defe69e777eb479cd91576bb4e3736d7f5dea Mon Sep 17 00:00:00 2001 From: JianW <49907914+v1212@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:08:25 +0800 Subject: [PATCH 18/42] Add test scenarios for delete, endpoint show, and code download (#8591) Add 6 new cli-interactive-tester scenarios covering three recently merged commands: Tier 0 (offline help validation): - 00-delete-help.yaml: validates azd ai agent delete --help output - 00-endpoint-show-help.yaml: validates azd ai agent endpoint show --help output - 00-code-download-help.yaml: validates azd ai agent code download --help output Tier 2 (cloud E2E, run between 2A-doctor and 2Z-teardown): - 2B-endpoint-show.yaml: shows endpoint config (table + JSON output) - 2C-code-download.yaml: negative-path test (container agent returns AgentNotCodeBased) - 2D-delete.yaml: deletes agent with --force, confirms removal via show All scenarios tested locally: 6/6 PASS. Co-authored-by: Jian Wu Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../00-code-download-help.yaml | 12 +++++++++ .../00-delete-help.yaml | 12 +++++++++ .../00-endpoint-show-help.yaml | 12 +++++++++ .../2B-endpoint-show.yaml | 21 ++++++++++++++++ .../2C-code-download.yaml | 25 +++++++++++++++++++ .../2D-delete.yaml | 22 ++++++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml new file mode 100644 index 00000000000..7d47cdef583 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent code download --help` prints usage. +name: "code-download-help" +command: "azd ai agent code download --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:code", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and flags (--dest, --version, --zip)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml new file mode 100644 index 00000000000..cbdab56b8ce --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent delete --help` prints usage. +name: "delete-help" +command: "azd ai agent delete --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:delete", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and flags (--force, --version)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml new file mode 100644 index 00000000000..d0b598108f0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent endpoint show --help` prints usage. +name: "endpoint-show-help" +command: "azd ai agent endpoint show --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:endpoint", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and supported output formats (json, table)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml new file mode 100644 index 00000000000..756108a70a2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `endpoint show` displays the deployed agent's endpoint config. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "endpoint-show" +command: "azd ai agent endpoint show" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:endpoint", "serial-only"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the endpoint configuration table to render (auto-resolves agent from azure.yaml)." + - "Confirm the output shows protocol(s) configured on the agent (e.g. 'responses')." + - "Confirm the endpoint URL or base path is displayed." + - "Run the command again with '--output json' and confirm valid JSON is returned with protocol and endpoint fields." + - "Take a screenshot of both the table and JSON outputs." + - "Report a finding if the command errors, shows no endpoint config, or returns unexpected empty fields for a deployed agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml new file mode 100644 index 00000000000..e2cd3087dd8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml @@ -0,0 +1,25 @@ +# Tier 2 (cloud E2E) — `code download` against a container-based agent (negative path). +# +# The shared agent uses Docker (container) deploy mode, not code_configuration. +# `code download` is only supported for code-based agents. This scenario verifies +# the CLI returns a clear, actionable error (AgentNotCodeBased / 409) rather than +# a confusing or generic failure. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "code-download-not-code-based" +command: "azd ai agent code download --dest ~/working/azd-agents-shared/code-download-test" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:code", "serial-only", "negative-path"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run code download against the shared agent (which is container-based, NOT code-based)." + - "Confirm the command returns a non-zero exit code and a clear error message indicating the agent is not code-based (e.g. 'AgentNotCodeBased' or 'not a code-based agent')." + - "Confirm the error message is user-friendly and actionable — not a raw stack trace or generic HTTP error." + - "Take a screenshot of the error output." + - "Report a finding if the command succeeds unexpectedly, hangs, panics, or gives a confusing/generic error message instead of a clear 'not code-based' explanation." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml new file mode 100644 index 00000000000..3bb1deaf2ed --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — `delete` removes the shared agent from Foundry. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# NOTE: This MUST run AFTER all other Tier 2 scenarios that use the shared agent +# and BEFORE 2Z-teardown-down.yaml. Ordering: 2A → 2B → 2C → 2D → 2Z. +name: "delete" +command: "azd ai agent delete --force" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:delete", "serial-only"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the delete command to complete (auto-resolves agent from azure.yaml, --force skips confirmation and handles active sessions)." + - "Confirm the command reports the agent was deleted successfully (exit 0)." + - "After deletion, run 'azd ai agent show' in the same directory and confirm it returns an error or indicates the agent no longer exists." + - "Take a screenshot of the delete result and the post-delete show output." + - "Report a finding if delete fails, hangs, or if 'show' still reports the agent as active after deletion." From 69538a68d95df6182246798dd24dad452adc553b Mon Sep 17 00:00:00 2001 From: JianW <49907914+v1212@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:07:19 +0800 Subject: [PATCH 19/42] fix: replace hardcoded fixture fallback with fail-fast :? syntax (#8606) The 4 Tier 1 scenarios that copy fixtures used a hardcoded /mnt/c/Repos/... fallback path (Travis's machine). Replace with bash :? expansion so that missing AZD_AGENTS_FIXTURES fails immediately with a clear message instead of a cryptic 'No such file or directory'. Co-authored-by: Jian Wu Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../10-init-deploy-mode-code.yaml | 2 +- .../10-init-deploy-mode-container.yaml | 2 +- .../cli-interactive-tester-scenarios/10-init-from-code.yaml | 2 +- .../10-init-validate-deploy-mode.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index cd46adb445c..9b0b2566339 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -19,7 +19,7 @@ pre: - run: "rm -rf ~/working/azd-agents-t1-code-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml index db6e292460b..7201f055f45 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml @@ -19,7 +19,7 @@ pre: - run: "rm -rf ~/working/azd-agents-t1-container-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-container-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-container-deploy-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-container-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-container-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 84fbcbb5b47..6d05e6181c7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -18,7 +18,7 @@ pre: - run: "rm -rf ~/working/azd-agents-t1-from-code-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml index 5e2eec42f41..8df6f5f6fb2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml @@ -26,7 +26,7 @@ pre: - run: "rm -rf ~/working/azd-agents-validate-deploy-mode-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-validate-deploy-mode-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-validate-deploy-mode-{instance}/" + - run: "mkdir -p ~/working/azd-agents-validate-deploy-mode-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-validate-deploy-mode-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" From a42a0d66816d5ebf4bb4abbda5ee16a71f34cd03 Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 13 Jul 2026 12:41:34 -0700 Subject: [PATCH 20/42] Add setup-wsl.sh for native Linux azd builds in WSL Adds a setup script that cross-compiles azd core and the azure.ai.agents extension as native Linux/amd64 binaries and installs them in WSL. This eliminates git safe.directory, TTY detection, and file-locking errors caused by running Windows azd.exe via WSL interop. Also updates: - Scenarios README: documents the WSL setup workflow - Skill prerequisites: adds native binary check for Windows hosts - Skill running-scenarios: adds timestamp postfix to session_ids for collision avoidance across parallel agent sessions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/prerequisites.md | 1 + .../references/running-scenarios.md | 8 +- .../README.md | 22 ++++++ .../setup-wsl.sh | 77 +++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index 20808b7ff2c..f1f54ea88ec 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -21,6 +21,7 @@ the user exactly what to fix — do **not** try to work around it. | `git` + `gh` CLIs | `gh auth status` | Ask the user to run `gh auth login`. | | cli-interactive-tester MCP server | The `list_scenarios` / `start_session` MCP tools are available to you | Stop. Tell the user to register the cli-interactive-tester MCP server (see its README) and re-run. | | `profile.local.yaml` | File exists in the scenarios dir | Stop. Tell the user to `cp profile.local.yaml.example profile.local.yaml` and set `prefix` + `subscription`. | +| Native Linux `azd` in WSL (Windows only) | `azd version` inside the tester returns a dev build, not a Windows `.exe` interop version | Stop. Tell the user to run `bash setup-wsl.sh` from the scenarios directory inside WSL. Symlinking to `azd.exe` does not work (causes git safe.directory, TTY, and file-locking errors). | ### Auth (tier-dependent — only enforce for tiers actually selected) diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index 243c589513c..cc90a1db529 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -54,9 +54,11 @@ For each selected scenario: ## Parallelism & ordering - **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), one - sub-agent per scenario, each with a distinct descriptive `session_id` (e.g. - `fleet-10-init-from-code`). No `instance_id` is needed — each scenario's `cwd` already - isolates itself (defaults to the `-main` suffix). + sub-agent per scenario, each with a distinct descriptive `session_id` **postfixed with a + timestamp** (e.g. `fleet-10-init-from-code-1752434100`). The timestamp (Unix epoch + seconds) prevents collisions when multiple agent sessions drive the tester concurrently. + No `instance_id` is needed — each scenario's `cwd` already isolates itself (defaults to + the `-main` suffix). - **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's parallel-readiness section for which scenarios support it. - **Tier 2** (`serial-only`): never parallelize. Run `20-setup-deploy-shared-agent` first, diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 2711e042416..9c249286300 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -109,6 +109,28 @@ WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: path style is almost certainly the cause — translate `C:\…` to `/mnt/c/…` and retry one call before fanning out. +### Installing azd in WSL (Windows developers) + +The scenarios must run **native Linux binaries** inside WSL. Symlinking to +`azd.exe` on the Windows side does not work — it causes `git safe.directory` +errors, TTY detection failures, and file locking issues. + +To build and install your local dev code as native Linux binaries in WSL: + +```bash +# From inside WSL: +cd /mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios +bash setup-wsl.sh +``` + +This script: +1. Cross-compiles `azd` core (`linux/amd64`) → `/usr/local/bin/azd` +2. Cross-compiles the extension (`linux/amd64`) → `~/.azd/extensions/azure.ai.agents/` +3. Prints version confirmation + +**Re-run `setup-wsl.sh` after every local code change** you want to test. +Requires the Go toolchain installed in WSL. + ## Authentication Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh new file mode 100644 index 00000000000..803fa9fce76 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# setup-wsl.sh — Build and install native Linux azd + extension for WSL testing. +# +# Run this from inside WSL (or via `wsl bash setup-wsl.sh` from Windows) after +# making local code changes. It cross-compiles native Linux/amd64 binaries from +# the repo source so the cli-interactive-tester drives your dev build directly. +# +# Prerequisites: +# - Go toolchain installed in WSL (or accessible via PATH) +# - Git installed in WSL +# +# Usage: +# cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios +# bash setup-wsl.sh +# +# What it does: +# 1. Builds azd core (linux/amd64) → /usr/local/bin/azd +# 2. Builds the azure.ai.agents extension (linux/amd64) → ~/.azd/extensions/ +# 3. Prints version confirmation + +set -euo pipefail + +# Resolve paths relative to this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXTENSION_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +AZD_DIR="$(cd "$EXTENSION_DIR/../.." && pwd)" +REPO_ROOT="$(cd "$AZD_DIR/../.." && pwd)" + +echo "=== setup-wsl.sh ===" +echo " Repo root: $REPO_ROOT" +echo " azd source: $AZD_DIR" +echo " Extension src: $EXTENSION_DIR" +echo "" + +# --- Step 1: Build azd core --- +echo "▸ Building azd core (linux/amd64)..." + +COMMIT=$(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") +VERSION="0.0.0-dev.0" +LDFLAGS="-X 'github.com/azure/azure-dev/cli/azd/internal.Version=${VERSION} (commit ${COMMIT})'" + +GOOS=linux GOARCH=amd64 go build \ + -ldflags="$LDFLAGS" \ + -o /usr/local/bin/azd \ + "$AZD_DIR" + +echo " ✓ Installed /usr/local/bin/azd" +echo "" + +# --- Step 2: Build the extension --- +echo "▸ Building azure.ai.agents extension (linux/amd64)..." + +EXTENSION_INSTALL_DIR="$HOME/.azd/extensions/azure.ai.agents" +mkdir -p "$EXTENSION_INSTALL_DIR" + +EXT_COMMIT=$(cd "$EXTENSION_DIR" && git rev-parse HEAD 2>/dev/null || echo "unknown") +EXT_BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) +EXT_VERSION=$(cat "$EXTENSION_DIR/version.txt" 2>/dev/null || echo "0.0.0-dev") +VERSION_PATH="azureaiagent/internal/version" + +GOOS=linux GOARCH=amd64 go build \ + -ldflags="-X '${VERSION_PATH}.Version=${EXT_VERSION}' -X '${VERSION_PATH}.Commit=${EXT_COMMIT}' -X '${VERSION_PATH}.BuildDate=${EXT_BUILD_DATE}'" \ + -o "$EXTENSION_INSTALL_DIR/azure-ai-agents-linux-amd64" \ + "$EXTENSION_DIR" + +# Copy extension.yaml (azd needs it to discover the extension) +cp "$EXTENSION_DIR/extension.yaml" "$EXTENSION_INSTALL_DIR/extension.yaml" + +echo " ✓ Installed $EXTENSION_INSTALL_DIR/azure-ai-agents-linux-amd64" +echo "" + +# --- Step 3: Verify --- +echo "▸ Verifying installation..." +echo " azd version: $(azd version 2>&1 | head -1)" +echo " extension: $(azd ai agent version 2>&1 | grep -i version | head -1)" +echo "" +echo "=== Done. WSL is ready for scenario testing. ===" From 7088035c59f5ab21bee383d3a4da0e9e919a3729 Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 13 Jul 2026 12:43:07 -0700 Subject: [PATCH 21/42] Update default test model to gpt-5.4-mini Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../tests/cli-interactive-tester-scenarios/profile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml index 30ef5c004d4..ebfc9bbace2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml @@ -16,7 +16,7 @@ region: "East US 2" # Default model deployment chosen during init / referenced by --model flags. # gpt-4.1-mini is cheap and fast — appropriate for tests. -model: "gpt-4.1-mini" +model: "gpt-5.4-mini" # Suffix appended to {prefix} to form the Tier 2 shared agent's name (and # therefore the subdir name that `azd ai agent init` scaffolds into under From a674c2d6189ed530dfab537129be383424c3b759 Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 13 Jul 2026 12:55:23 -0700 Subject: [PATCH 22/42] Add no-retry rule to driving conventions Failed scenarios must not be retried unless the scenario's goals explicitly instruct it. Retrying masks flaky behavior and undermines the test suite's value as a regression signal. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../agent-scenario-tests/references/running-scenarios.md | 4 ++++ .../tests/cli-interactive-tester-scenarios/README.md | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index cc90a1db529..b19c05ff0a5 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -45,6 +45,10 @@ For each selected scenario: - **Treat a select miss as a hard failure.** `select_by_text` is fail-loud (`ERROR during 'select': …`). Report a finding and stop that scenario — do **not** retry with a different `choice_text`/`choice_index`. +- **Never retry a failed scenario.** If a scenario fails (command errors, unexpected output, + non-zero exit), report the finding and move on. Do **not** re-run the scenario hoping for + a different result — unless the scenario's `goals:` explicitly instruct a retry. Retrying + masks flaky behavior and makes the test suite unreliable as a regression signal. - **Prefer `choice_text` over `choice_index`** (indices shift between releases). - **Clear a pre-filled text field before typing** (e.g. the agent-name prompt); otherwise your value *appends* to the default (`defaultyourvalue`). diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 9c249286300..58b82e23f8e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -264,6 +264,11 @@ its bugs: fail-loud: a missing target raises `LookupError`, surfaced as `ERROR during 'select': …`. **Report a finding and stop** — do not retry with a different `choice_text`/`choice_index` to work around it. +- **Never retry a failed scenario.** If a scenario fails (command errors, + unexpected output, non-zero exit), report the finding and move on. Do **not** + re-run the scenario hoping for a different result — unless the scenario's + `goals:` explicitly instruct a retry. Retrying masks flaky behavior and makes + the test suite unreliable as a regression signal. - **Prefer `choice_text` over `choice_index`** when the label is stable (indices shift between releases). - **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent From 125c78cbeb29085eeac1564420d7a1b922765f68 Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 13 Jul 2026 13:01:40 -0700 Subject: [PATCH 23/42] fix: setup-wsl.sh build from module root and use full commit hash - Use subshell cd into module directory before go build (fixes 'outside main module' error when script is run from the scenarios directory) - Use full 40-char commit hash (git rev-parse HEAD) instead of --short, matching the required version format ' (commit )' Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../cli-interactive-tester-scenarios/setup-wsl.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh index 803fa9fce76..07028637baf 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh @@ -35,14 +35,14 @@ echo "" # --- Step 1: Build azd core --- echo "▸ Building azd core (linux/amd64)..." -COMMIT=$(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") +COMMIT=$(cd "$REPO_ROOT" && git rev-parse HEAD 2>/dev/null || echo "0000000000000000000000000000000000000000") VERSION="0.0.0-dev.0" LDFLAGS="-X 'github.com/azure/azure-dev/cli/azd/internal.Version=${VERSION} (commit ${COMMIT})'" -GOOS=linux GOARCH=amd64 go build \ +(cd "$AZD_DIR" && GOOS=linux GOARCH=amd64 go build \ -ldflags="$LDFLAGS" \ -o /usr/local/bin/azd \ - "$AZD_DIR" + .) echo " ✓ Installed /usr/local/bin/azd" echo "" @@ -58,10 +58,10 @@ EXT_BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) EXT_VERSION=$(cat "$EXTENSION_DIR/version.txt" 2>/dev/null || echo "0.0.0-dev") VERSION_PATH="azureaiagent/internal/version" -GOOS=linux GOARCH=amd64 go build \ +(cd "$EXTENSION_DIR" && GOOS=linux GOARCH=amd64 go build \ -ldflags="-X '${VERSION_PATH}.Version=${EXT_VERSION}' -X '${VERSION_PATH}.Commit=${EXT_COMMIT}' -X '${VERSION_PATH}.BuildDate=${EXT_BUILD_DATE}'" \ -o "$EXTENSION_INSTALL_DIR/azure-ai-agents-linux-amd64" \ - "$EXTENSION_DIR" + .) # Copy extension.yaml (azd needs it to discover the extension) cp "$EXTENSION_DIR/extension.yaml" "$EXTENSION_INSTALL_DIR/extension.yaml" From 2fa694120b00804071e2969c02f423ddee96e489 Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 13 Jul 2026 14:02:44 -0700 Subject: [PATCH 24/42] Auto-rebuild WSL binaries before every scenario run The skill workflow now includes Step 1b which executes setup-wsl.sh via the tester at the start of every run. This guarantees WSL always has the latest local code compiled as native Linux binaries, eliminating stale binary issues when code changes between runs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/prerequisites.md | 2 +- .../agent-scenario-tests/references/workflow.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index f1f54ea88ec..62c875175b7 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -21,7 +21,7 @@ the user exactly what to fix — do **not** try to work around it. | `git` + `gh` CLIs | `gh auth status` | Ask the user to run `gh auth login`. | | cli-interactive-tester MCP server | The `list_scenarios` / `start_session` MCP tools are available to you | Stop. Tell the user to register the cli-interactive-tester MCP server (see its README) and re-run. | | `profile.local.yaml` | File exists in the scenarios dir | Stop. Tell the user to `cp profile.local.yaml.example profile.local.yaml` and set `prefix` + `subscription`. | -| Native Linux `azd` in WSL (Windows only) | `azd version` inside the tester returns a dev build, not a Windows `.exe` interop version | Stop. Tell the user to run `bash setup-wsl.sh` from the scenarios directory inside WSL. Symlinking to `azd.exe` does not work (causes git safe.directory, TTY, and file-locking errors). | +| Native Linux `azd` in WSL (Windows only) | `azd version` inside the tester returns a dev build, not a Windows `.exe` interop version | The skill automatically runs `setup-wsl.sh` (Step 1b) to rebuild from source. If you need to run it manually: `bash setup-wsl.sh` from the scenarios directory inside WSL. Symlinking to `azd.exe` does not work (causes git safe.directory, TTY, and file-locking errors). | ### Auth (tier-dependent — only enforce for tiers actually selected) diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md index 4280d8f111d..f30365ac107 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -5,6 +5,22 @@ Run the checks in `prerequisites.md`. If a hard prerequisite is missing, stop with a clear message. Don't auto-install or work around a missing MCP server or profile. +### Step 1b — Rebuild WSL binaries (Windows only) + +Before running any scenarios, rebuild the native Linux `azd` and extension binaries from the +current repo source so the tester always exercises the latest local code. Execute +`setup-wsl.sh` inside WSL via the tester: + +```text +start_session(command="bash /mnt/c//setup-wsl.sh", + cwd="/mnt/c//cli/azd/extensions/azure.ai.agents", + session_id="setup-wsl-", + run_name="setup-wsl") +``` + +Wait for it to print "Done. WSL is ready for scenario testing." and then `finish_session`. +If the build fails, stop and report the build error — do not proceed with stale binaries. + ### Step 2 — Resolve the PR ```bash From 145106ff34eac91bac260952e2a7739db827a8cb Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 14 Jul 2026 09:42:59 -0700 Subject: [PATCH 25/42] Strict goal-matching rule for driving conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario goals are the contract — a scenario PASSES only when the product's actual behavior matches what the goals describe. The driving agent must verify goals were met, not rationalize why they weren't. Added 'never adapt around broken goals' rule: if goals reference a command/flag that doesn't exist or expect output that doesn't appear, fail the scenario instead of working around it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/running-scenarios.md | 11 +++++++++++ .../cli-interactive-tester-scenarios/README.md | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index b19c05ff0a5..4d66f27128a 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -39,6 +39,13 @@ For each selected scenario: ## Driving conventions (fail-loud) +- **The scenario goals are the contract.** A scenario PASSES only when the product's actual + behavior matches what the goals describe. If the goals say "expect error X" and the + product prints a different error (even a reasonable one), that is a FAIL. If the goals + reference a flag or subcommand that no longer exists, that is a FAIL. The driving agent's + job is to **verify** goals were met, not to **rationalize** why they weren't. Do not mark + a scenario as PASSED with an "observation" when the goals were not achieved — observations + are for incidental notes on scenarios that genuinely passed all their goals. - **Don't verify/retry after a `select`.** Reading back the echo and "correcting" a pick hides the very picker defect the test exists to catch. Send the action and let downstream prompts surface any failure. @@ -49,6 +56,10 @@ For each selected scenario: non-zero exit), report the finding and move on. Do **not** re-run the scenario hoping for a different result — unless the scenario's `goals:` explicitly instruct a retry. Retrying masks flaky behavior and makes the test suite unreliable as a regression signal. +- **Never adapt around broken goals.** If the goals instruct you to run a command or flag + that does not exist, or expect output that does not appear, fail the scenario. Do not + substitute an alternative command, skip the broken step, or invent a workaround. The + scenario must be updated by a human — the driving agent must not silently patch over it. - **Prefer `choice_text` over `choice_index`** (indices shift between releases). - **Clear a pre-filled text field before typing** (e.g. the agent-name prompt); otherwise your value *appends* to the default (`defaultyourvalue`). diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 58b82e23f8e..d4e7c76d766 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -257,6 +257,14 @@ These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving ag should follow them so the runs actually *test* the CLI instead of papering over its bugs: +- **The scenario goals are the contract.** A scenario PASSES only when the + product's actual behavior matches what the goals describe. If the goals say + "expect error X" and the product prints a different error (even a reasonable + one), that is a FAIL. If the goals reference a flag or subcommand that no + longer exists, that is a FAIL. The driving agent's job is to **verify** goals + were met, not to **rationalize** why they weren't. Do not mark a scenario as + PASSED with an "observation" when the goals were not achieved — observations + are for incidental notes on scenarios that genuinely passed all their goals. - **Don't verify/retry after a `select`.** These runs exist to catch picker bugs; reading back the echo and "correcting" a pick hides the very defect the test is for. Send the action and let downstream prompts surface any failure. @@ -269,6 +277,11 @@ its bugs: re-run the scenario hoping for a different result — unless the scenario's `goals:` explicitly instruct a retry. Retrying masks flaky behavior and makes the test suite unreliable as a regression signal. +- **Never adapt around broken goals.** If the goals instruct you to run a + command or flag that does not exist, or expect output that does not appear, + fail the scenario. Do not substitute an alternative command, skip the broken + step, or invent a workaround. The scenario must be updated by a human — the + driving agent must not silently patch over it. - **Prefer `choice_text` over `choice_index`** when the label is stable (indices shift between releases). - **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent From 66d0eac5c3b0899fdd1cc1fc22c2aa7e9980faf3 Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 14 Jul 2026 10:43:17 -0700 Subject: [PATCH 26/42] Add timestamp postfix to shared_agent_name for run isolation The shared_agent_name derivation now includes a compact timestamp (MMDDHHmm) so concurrent runs on the same machine get unique Azure resource names and working directories, preventing deployment conflicts. Format: {prefix}-{shared_agent_suffix}-{ts} Example: alice-basic-responses-07141038 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/prerequisites.md | 5 ++++- .../20-setup-deploy-shared-agent.yaml | 13 +++++++------ .../cli-interactive-tester-scenarios/README.md | 14 +++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index 62c875175b7..e05852add27 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -40,7 +40,10 @@ The scenarios reference `{prefix}`, `{subscription}`, `{region}`, `{model}`, `{t 1. Read both `profile.yaml` (checked-in defaults) and `profile.local.yaml` (developer overrides) and **merge them, local overriding shared**. -2. Derive `shared_agent_name = "{prefix}-{shared_agent_suffix}"`. +2. Derive `shared_agent_name = "{prefix}-{shared_agent_suffix}-{ts}"` where `{ts}` is + a compact timestamp of the form `MMDDHHmm` (e.g. `07141038`). This isolates + concurrent runs so two agents on the same machine don't collide on Azure resource + names or working directories. 3. Pass the merged map as `session_vars` on **every** `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` call. Omitting it leaves placeholders unresolved and the run executes against literal `{prefix}` strings. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 94234d7c8b1..8f622de8ec9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -3,12 +3,13 @@ # ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared # and scaffolds the project into a subdirectory named after the agent, so the # deployed project lives in ~/working/azd-agents-shared/{shared_agent_name} -# (where {shared_agent_name} = {prefix}-{shared_agent_suffix}, e.g. -# "alice-basic-responses") and is reused by the targeted scenarios. The agent -# name MUST be exactly that value so the subdirectory path is deterministic -# and the reuse scenarios can find it. Run 2Z-teardown-down.yaml LAST to -# clean up. All `{name}` placeholders come from the merged profile -# (`profile.yaml` + `profile.local.yaml`). +# (where {shared_agent_name} = {prefix}-{shared_agent_suffix}-{ts}, e.g. +# "alice-basic-responses-07141038") and is reused by the targeted scenarios. +# The agent name MUST be exactly that value so the subdirectory path is +# deterministic and the reuse scenarios can find it. Run +# 2Z-teardown-down.yaml LAST to clean up. All `{name}` placeholders come +# from the merged profile (`profile.yaml` + `profile.local.yaml`) plus the +# orchestrator-derived `shared_agent_name`. name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index d4e7c76d766..c28513060a6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -19,7 +19,8 @@ cp profile.local.yaml.example profile.local.yaml Then ask Copilot CLI to load a scenario and accomplish its goals. The orchestrator must **load both profile files, merge them (local overrides -shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass +shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}-{ts}` +(where `{ts}` is a compact timestamp like `MMDDHHmm`, e.g. `07141038`), and pass the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` call** — the scenario YAMLs reference those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, @@ -42,7 +43,8 @@ a set of test scenarios for the cli-interactive-tester. I want you to use the cl and take screenshots at each step. First, read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge -them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}". +them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}-{ts}" +where {ts} is a compact timestamp (MMDDHHmm, e.g. 07141038). Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / run_post_hooks call — the scenarios reference {prefix}, {subscription}, {region}, {model}, {tenant} (optional), and {shared_agent_name} placeholders. @@ -435,7 +437,8 @@ and tags=["cmd:init"] to enumerate the matching scenarios. Then read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge them (local overrides shared); also derive -shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +shared_agent_name = "{prefix}-{shared_agent_suffix}-{ts}" where {ts} is a compact +timestamp (MMDDHHmm, e.g. 07141038). Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / run_post_hooks call. @@ -505,7 +508,7 @@ Variables exposed to scenarios via `session_vars`: | `{region}` | `profile.yaml` | `East US 2` | | | `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | | `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | -| `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}` | Tier 2 subdirectory name — orchestrator must compute and pass alongside the others | +| `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}-{ts}` | Tier 2 subdirectory name — orchestrator must compute (with `{ts}` = `MMDDHHmm` compact timestamp) and pass alongside the others. The timestamp isolates concurrent runs. | **Bootstrap (one-time per checkout):** @@ -515,7 +518,8 @@ cp profile.local.yaml.example profile.local.yaml ``` The orchestrator must load both files, merge (local overrides shared), derive -`shared_agent_name`, and pass the merged map as `session_vars=` on every +`shared_agent_name` (= `{prefix}-{shared_agent_suffix}-{ts}` where `{ts}` is a +compact `MMDDHHmm` timestamp), and pass the merged map as `session_vars=` on every `load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. Failing to thread `session_vars` leaves `{prefix}` etc. unresolved in goals and the run will execute against literal placeholder strings. From 7d3e9f1009896e002e73fe66c8172c3e3118fae3 Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 14 Jul 2026 14:43:57 -0700 Subject: [PATCH 27/42] Fix scenario issues from run 20260714-100000 - Picker navigation: update goals to test wrapping behavior (intentional) instead of expecting bounded clamping - fixtures_dir: replace AZD_AGENTS_FIXTURES env var with {fixtures_dir} session_var auto-derived by the orchestrator (4 scenarios updated) - Container deploy: clarify that startup command prompt is expected; only entry-point/runtime prompts are code-deploy-only - Make fixtures_dir derivation platform-neutral (not WSL-specific) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/prerequisites.md | 13 ++++++++++--- .../00-init-picker-navigation.yaml | 5 +++-- .../10-init-deploy-mode-code.yaml | 3 +-- .../10-init-deploy-mode-container.yaml | 10 +++++----- .../10-init-from-code.yaml | 5 ++--- .../10-init-validate-deploy-mode.yaml | 5 ++--- .../cli-interactive-tester-scenarios/README.md | 17 +++++++++++------ 7 files changed, 34 insertions(+), 24 deletions(-) diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index e05852add27..7569f143afa 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -44,6 +44,13 @@ The scenarios reference `{prefix}`, `{subscription}`, `{region}`, `{model}`, `{t a compact timestamp of the form `MMDDHHmm` (e.g. `07141038`). This isolates concurrent runs so two agents on the same machine don't collide on Azure resource names or working directories. -3. Pass the merged map as `session_vars` on **every** `load_scenario`, `run_pre_hooks`, - `start_session`, and `run_post_hooks` call. Omitting it leaves placeholders unresolved - and the run executes against literal `{prefix}` strings. +3. Derive `fixtures_dir` = the tester-side absolute path of the `fixtures/` subdirectory + inside the scenarios directory. On Windows (where the tester runs inside WSL) this is + the WSL-translated path (e.g. `/mnt/c/Repos/azure-dev/.../fixtures`); on native + Linux/macOS it is the regular absolute path. Apply the same path-style logic used for + scenario paths (see `running-scenarios.md` § Path style). + Scenario pre-hooks use `{fixtures_dir}` to locate test fixture files. +4. Pass the merged map (including `shared_agent_name` and `fixtures_dir`) as `session_vars` + on **every** `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` + call. Omitting it leaves placeholders unresolved and the run executes against literal + `{prefix}` strings. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index 6e11833c869..4b19625349f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -30,8 +30,9 @@ goals: - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." - "Type a string that matches nothing; confirm the list shows an empty/no-match state and does not crash." - "Clear the filter (backspace) and confirm the full list returns." - - "Press Down many times past the end of the list; confirm selection stays bounded and does not error." + - "Press Down until 'See all templates...' is selected (it should be the last item in the list)." + - "Press Down one more time; confirm the picker wraps back to the first item in the list." - "If a prompt mentions a '?' hint, press '?' and confirm a helpful hint is shown." - "Take a screenshot at each interesting state." - "Press Ctrl-C (or Escape at the top-level prompt) to abort. Confirm it exits cleanly without a stack trace and without leaving a half-written azure.yaml." - - "Report a finding for any picker glitch: filter not recovering, crash on no-match, unbounded scrolling, unhelpful hints, or a messy/abrupt abort." + - "Report a finding for any picker glitch: filter not recovering, crash on no-match, wrap not working correctly, unhelpful hints, or a messy/abrupt abort." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 9b0b2566339..0ebf5db1576 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -14,12 +14,11 @@ env: AZD_DISABLE_AGENT_DETECT: "1" # Seed a committed Python fixture so code-deploy has real source to package. -# Override the fixture location with AZD_AGENTS_FIXTURES if needed. pre: - run: "rm -rf ~/working/azd-agents-t1-code-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"{fixtures_dir}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml index 7201f055f45..d18046f1e15 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml @@ -3,7 +3,8 @@ # Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; # no cost incurred. # Targets the --deploy-mode container path, which builds/packages a container -# image (instead of prompting for entry-point + runtime as code-deploy does). +# image. Container mode may prompt for a startup command (Dockerfile CMD) but +# should NOT prompt for entry-point + runtime (those are code-deploy concerns). # Complements 10-init-deploy-mode-code.yaml. name: "init-deploy-mode-container" command: "azd ai agent init --deploy-mode container" @@ -14,19 +15,18 @@ env: AZD_DISABLE_AGENT_DETECT: "1" # Seed a committed Python fixture so container-deploy has real source to package. -# Override the fixture location with AZD_AGENTS_FIXTURES if needed. pre: - run: "rm -rf ~/working/azd-agents-t1-container-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-container-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-container-deploy-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-container-deploy-{instance} && cp -r \"{fixtures_dir}/from-code/.\" ~/working/azd-agents-t1-container-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-container-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - - "Wait for the tool to inspect the current directory's code with container-deploy mode selected (it should NOT prompt for an entry-point or runtime — those are code-deploy concerns)." + - "Wait for the tool to inspect the current directory's code with container-deploy mode selected. Container mode may prompt for a startup command (the Dockerfile CMD, e.g. 'python main.py') — that is expected. However, it should NOT prompt for the separate entry-point or runtime prompts that code-deploy uses (those are distinct, code-deploy-only prompts)." - "If an existing agent manifest is detected, confirm reuse." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." @@ -35,4 +35,4 @@ goals: - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect container-deploy mode (a Dockerfile or container build configuration is present rather than entry-point/runtime fields)." - "Take a screenshot of the completed init output." - - "STOP here — do NOT run 'azd provision'. Report a finding if container mode incorrectly prompts for entry-point/runtime, fails to produce container build config, or the deploy mode is not persisted." + - "STOP here — do NOT run 'azd provision'. Report a finding if container mode incorrectly prompts for entry-point/runtime (the separate code-deploy prompts), fails to produce container build config, or the deploy mode is not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 6d05e6181c7..1611ab5b7ab 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -4,8 +4,7 @@ # no cost incurred. # Precondition: the cwd should already contain agent source code (and ideally an # agent manifest). The pre hooks seed a committed Python fixture so this is -# guaranteed and the run is idempotent. Override the fixture location with -# AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. +# guaranteed and the run is idempotent. name: "init-from-code" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-from-code-{instance}" @@ -18,7 +17,7 @@ pre: - run: "rm -rf ~/working/azd-agents-t1-from-code-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" + - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"{fixtures_dir}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml index 8df6f5f6fb2..6e0c30735ad 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml @@ -20,13 +20,12 @@ env: AZD_DISABLE_AGENT_DETECT: "1" # Seed a committed from-code fixture so init resolves the from-code method and -# reaches validateCodeDeployInput. Override the fixture location with -# AZD_AGENTS_FIXTURES if needed. +# reaches validateCodeDeployInput. pre: - run: "rm -rf ~/working/azd-agents-validate-deploy-mode-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-validate-deploy-mode-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:?Set AZD_AGENTS_FIXTURES to the WSL path of your fixtures directory (e.g. /mnt/d/w1/azure-dev/.../tests/cli-interactive-tester-scenarios/fixtures)}/from-code/.\" ~/working/azd-agents-validate-deploy-mode-{instance}/" + - run: "mkdir -p ~/working/azd-agents-validate-deploy-mode-{instance} && cp -r \"{fixtures_dir}/from-code/.\" ~/working/azd-agents-validate-deploy-mode-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index c28513060a6..366b9821d06 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -509,6 +509,7 @@ Variables exposed to scenarios via `session_vars`: | `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | | `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | | `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}-{ts}` | Tier 2 subdirectory name — orchestrator must compute (with `{ts}` = `MMDDHHmm` compact timestamp) and pass alongside the others. The timestamp isolates concurrent runs. | +| `{fixtures_dir}` | derived by orchestrator | `/fixtures` | Tester-side absolute path to the `fixtures/` subdirectory (WSL-translated on Windows, native on Linux/macOS); used by pre-hooks to seed test fixture files | **Bootstrap (one-time per checkout):** @@ -519,7 +520,9 @@ cp profile.local.yaml.example profile.local.yaml The orchestrator must load both files, merge (local overrides shared), derive `shared_agent_name` (= `{prefix}-{shared_agent_suffix}-{ts}` where `{ts}` is a -compact `MMDDHHmm` timestamp), and pass the merged map as `session_vars=` on every +compact `MMDDHHmm` timestamp) and `fixtures_dir` (= the tester-side absolute path +of the `fixtures/` subdirectory — WSL-translated on Windows, native on +Linux/macOS), and pass the merged map as `session_vars=` on every `load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. Failing to thread `session_vars` leaves `{prefix}` etc. unresolved in goals and the run will execute against literal placeholder strings. @@ -581,15 +584,17 @@ How they're used here: `app.py`). The existing-code scenarios copy it into the working dir via a `pre` hook, then select "Use the code in the current directory" at the init prompt. -The hook references the fixture by absolute path with an overridable env var: +The hook references the fixture via the `{fixtures_dir}` session variable, which +the orchestrator auto-derives from the scenarios directory path: ```sh -cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/." "$cwd" +cp -r "{fixtures_dir}/from-code/." "$cwd" ``` -If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view -of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this -`fixtures/` directory before running the existing-code scenarios. +The orchestrator computes `fixtures_dir` as the tester-side absolute path of the +`fixtures/` subdirectory inside the scenarios directory (WSL-translated on Windows, +native on Linux/macOS) and passes it as a `session_var` alongside the other profile +variables. ## Re-running scenarios (idempotency) From 04d6d1fd86281a1ce24c6084e476e31fbf250d4b Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 16 Jul 2026 17:07:21 -0700 Subject: [PATCH 28/42] fix: update test scenarios to match current extension API - eval init -> eval generate, --max-iterations -> --max-candidates (stale flags) - Remove agent.yaml references; agent definition now inline in azure.yaml - Rename 10-init-from-manifest-url -> 10-init-from-azure-yaml-url with correct URL - Update deploy-mode scenarios: code deploy auto-selected for Python templates - Flip 2C-code-download from negative to positive path (code deploy is default) - Rewrite 26-endpoint-update pre-hook to inject agentCard into azure.yaml - Fix 2B-endpoint-show goals (command shows protocols/version/auth, not URL) - Update 10-init-flags-agent-name-model with correct azure.yaml URL Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../10-init-deploy-mode-code.yaml | 2 +- .../10-init-deploy-mode-container.yaml | 2 +- .../10-init-flags-agent-name-model.yaml | 20 +++++----- ....yaml => 10-init-from-azure-yaml-url.yaml} | 30 ++++++++------- .../10-init-template-python.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 6 ++- .../26-endpoint-update.yaml | 37 +++++++++++++------ .../28-eval-lifecycle.yaml | 20 +++++----- .../29-optimize-submit-and-cancel.yaml | 6 +-- .../2B-endpoint-show.yaml | 4 +- .../2C-code-download.yaml | 26 +++++++------ 11 files changed, 89 insertions(+), 66 deletions(-) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-from-manifest-url.yaml => 10-init-from-azure-yaml-url.yaml} (63%) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 0ebf5db1576..122414a42b5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -34,6 +34,6 @@ goals: - "If asked for a location/region, select '{region}'." - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." + - "Verify the scaffold: confirm azure.yaml reflects code-deploy mode — the azure.ai.agent service entry should have codeConfiguration with the chosen entry point and runtime embedded in its properties, and a .agentignore file should control ZIP packaging." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if the entry-point or runtime prompts are missing, confusing, or not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml index d18046f1e15..c41089d44ba 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml @@ -33,6 +33,6 @@ goals: - "If asked for a location/region, select '{region}'." - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect container-deploy mode (a Dockerfile or container build configuration is present rather than entry-point/runtime fields)." + - "Verify the scaffold: confirm azure.yaml reflects container-deploy mode — the azure.ai.agent service entry should have docker/Dockerfile configuration in its properties rather than codeConfiguration with entry-point/runtime fields." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if container mode incorrectly prompts for entry-point/runtime (the separate code-deploy prompts), fails to produce container build config, or the deploy mode is not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 81a29eb4e44..86dab30a67a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,13 +1,17 @@ -# Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. +# Tier 1 (auth, scaffold only) — init from a remote azure.yaml with explicit --agent-name and --model. # # Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; # no cost incurred. -# Also requires GitHub login: `gh auth login` (manifest download can fall back to +# Also requires GitHub login: `gh auth login` (azure.yaml download can fall back to # the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails # fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. +# +# NOTE: With `-m` pointing to a Python azure.yaml, code deploy is auto-selected +# (no deploy-mode prompt). The --agent-name and --model flags should still override +# defaults in the generated project. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/azure.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" cwd: "~/working/azd-agents-t1-flags-{instance}" tags: ["tier:1", "cmd:init", "parallel-safe"] @@ -19,21 +23,19 @@ env: pre: - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" cwd: "~/working" - name: "require gh auth login (manifest download)" + name: "require gh auth login (azure.yaml download)" - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - - "Wait for the manifest to download and parse." - - "When asked how to deploy, select 'Container' (hosted agent)." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "Wait for the azure.yaml to download and parse." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - "If asked for a location/region, select '{region}'." - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the manifest defaults." + - "Verify the overrides: confirm azure.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the sample defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml similarity index 63% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml index e326894f897..49ce5685e03 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml @@ -1,14 +1,18 @@ -# Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. +# Tier 1 (auth, scaffold only) — init from a remote azure.yaml URL. # # Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; # no cost incurred. -# Also requires GitHub login: `gh auth login`. Downloading the manifest (and its +# Also requires GitHub login: `gh auth login`. Downloading the azure.yaml (and its # sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub # API is rate-limited, which would otherwise drop into an interactive gh login # mid-run. The pre hook fails fast if gh is not authenticated. -name: "init-from-manifest-url" -command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" -cwd: "~/working/azd-agents-t1-manifest-{instance}" +# +# NOTE: The `-m` flag now accepts an azure.yaml URL (unified project manifest) +# in addition to legacy agent.manifest.yaml files. With a Python project URL and +# userProvidedManifest=true, code deploy is auto-selected (no deploy-mode prompt). +name: "init-from-azure-yaml-url" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/azure.yaml" +cwd: "~/working/azd-agents-t1-from-url-{instance}" tags: ["tier:1", "cmd:init", "parallel-safe"] env: @@ -19,22 +23,20 @@ env: pre: - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" cwd: "~/working" - name: "require gh auth login (manifest download)" - - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" + name: "require gh auth login (azure.yaml download)" + - run: "rm -rf ~/working/azd-agents-t1-from-url-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - - "Wait for the tool to fetch and parse the manifest from the provided URL." - - "When asked how to deploy, select 'Container' (hosted agent)." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "Wait for the tool to fetch and parse the azure.yaml from the provided URL." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - "If asked for a location/region, select '{region}'." - - "When asked to select a model, choose '{model}' (or accept the manifest's model if one is pinned)." + - "When asked to select a model, choose '{model}' (or accept the sample's model if one is pinned)." - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." + - "Verify the scaffold: confirm azure.yaml exists in the project directory and reflects the sample's agent definition." - "Take a screenshot of the completed init output." - - "STOP here — do NOT run 'azd provision'. Report a finding if the manifest fails to download/parse, or if any field is dropped during scaffold." + - "STOP here — do NOT run 'azd provision'. Report a finding if the URL fails to download/parse, or if any field is dropped during scaffold." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index da845744fc7..43cce403902 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -32,6 +32,6 @@ goals: - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service with the agent definition (name, model, instructions) embedded inline in the service entry properties." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding for any confusing prompt, wrong default, or scaffold problem." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 8f622de8ec9..d10b8067fbd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -10,6 +10,10 @@ # 2Z-teardown-down.yaml LAST to clean up. All `{name}` placeholders come # from the merged profile (`profile.yaml` + `profile.local.yaml`) plus the # orchestrator-derived `shared_agent_name`. +# +# NOTE: The Python "Basic Responses" template auto-selects CODE deploy mode +# (no deploy-mode prompt appears). The resulting agent is code-based, which +# means `code download` will succeed against it. name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" @@ -39,13 +43,11 @@ goals: - "Select Python as the language." - "Select the 'Basic Responses' template from the list." - "When prompted for the AGENT NAME, set it to EXACTLY '{shared_agent_name}' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named '{shared_agent_name}'." - - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - "If asked for a location/region, select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Change directory into the new '{shared_agent_name}' subdirectory (run 'cd {shared_agent_name}') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." - "Run 'azd provision' (from inside the '{shared_agent_name}' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 0ac9922bb4b..7fb8214e937 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -12,24 +12,39 @@ pre: cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true - # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors - # ("nothing to update") if neither is defined. The Basic Responses template - # defines neither, so inject a minimal agent_card (idempotently) before the run - # so there is something to patch. + # `endpoint update` reads agentEndpoint/agentCard from the azure.yaml service + # properties (via LoadAgentDefinition) and errors ("nothing to update") if + # neither is defined. The Basic Responses template defines neither, so inject a + # minimal agentCard into the azure.yaml service properties (idempotently) before + # the run so there is something to patch. - run: | - f="$(find ~/working/azd-agents-shared/{shared_agent_name} -name agent.yaml | head -1)" - if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then - printf '\nagent_card:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" - echo "Injected agent_card into $f" + f="~/working/azd-agents-shared/{shared_agent_name}/azure.yaml" + if [ -f "$f" ] && ! grep -q 'agentCard:' "$f"; then + python3 -c " +import sys, yaml +with open(sys.argv[1]) as fh: + doc = yaml.safe_load(fh) +for svc in doc.get('services', {}).values(): + if svc.get('host') == 'azure.ai.agent': + props = svc.setdefault('config', {}) + props['agentCard'] = { + 'description': '{prefix} endpoint-update test card', + 'skills': [{'id': '{prefix}-echo', 'name': 'Echo', 'description': 'Echoes input back'}] + } + break +with open(sys.argv[1], 'w') as fh: + yaml.dump(doc, fh, default_flow_style=False, sort_keys=False) +print('Injected agentCard into azure.yaml') +" "$f" else - echo "agent_card already present or agent.yaml not found" + echo "agentCard already present or azure.yaml not found" fi cwd: "~/working/azd-agents-shared/{shared_agent_name}" - name: "inject agent_card so there is something to patch" + name: "inject agentCard into azure.yaml so there is something to patch" continue_on_error: true goals: - - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agent_card was injected into agent.yaml during setup so the patch has content." + - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agentCard was injected into azure.yaml service properties during setup so the patch has content." - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." - "Take a screenshot of the update result and the post-update 'show' output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml index 9e3311cd736..ded03a75c77 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml @@ -5,16 +5,16 @@ # cheap and fast. The agent and Foundry project endpoint are auto-detected from # the shared agent's azure.yaml. # -# IMPORTANT — eval init is ASYNC: with --no-wait it submits dataset generation +# IMPORTANT — eval generate is ASYNC: with --no-wait it submits dataset generation # (datagen-*) and evaluator generation (evaluatorgen-*) jobs and returns # immediately, surfacing those job IDs and writing eval.yaml. It does NOT create # an eval "run" — `azd ai agent eval run` does that. So right after a --no-wait -# init, `eval list` will typically show 0 rows (generation still in progress) and -# `eval show` with no eval-id will error cleanly ("no eval-id provided and none -# found ..."). The goal here is to confirm each command EXECUTES and degrades +# generate, `eval list` will typically show 0 rows (generation still in progress) +# and `eval show` with no eval-id will error cleanly ("no eval-id provided and +# none found ..."). The goal here is to confirm each command EXECUTES and degrades # gracefully, not that a run appears instantly. name: "eval-lifecycle" -command: "azd ai agent eval init --max-samples 15 --no-wait" +command: "azd ai agent eval generate --max-samples 15 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:eval", "serial-only"] @@ -26,13 +26,13 @@ pre: continue_on_error: true goals: - - "Run 'eval init --max-samples 15 --no-wait'; the agent and project endpoint are auto-detected from azure.yaml. Accept the default eval suite name when prompted." + - "Run 'eval generate --max-samples 15 --no-wait'; the agent and project endpoint are auto-detected from azure.yaml. Accept the default eval suite name when prompted." - "If asked how to provide the agent instruction, choose 'Type inline' and give a short description; decline including agent traces; when prompted for an eval/generation model, accept the deployed model default." - - "Confirm 'eval init' returns WITHOUT blocking (because of --no-wait) and surfaces async job identifiers — dataset generation (datagen-*) and evaluator generation (evaluatorgen-*) — and reports that eval.yaml was written." - - "Run 'azd ai agent eval list' and confirm it renders the eval table cleanly and exits 0. NOTE: because init is async, the list may legitimately show 0 rows immediately after a --no-wait init (generation still in progress, and no run has been created yet) — an empty list here is NOT a failure." + - "Confirm 'eval generate' returns WITHOUT blocking (because of --no-wait) and surfaces async job identifiers — dataset generation (datagen-*) and evaluator generation (evaluatorgen-*) — and reports that eval.yaml was written." + - "Run 'azd ai agent eval list' and confirm it renders the eval table cleanly and exits 0. NOTE: because generate is async, the list may legitimately show 0 rows immediately after a --no-wait generate (generation still in progress, and no run has been created yet) — an empty list here is NOT a failure." - "Run 'azd ai agent eval show' and confirm it either prints details for an eval-id or errors CLEANLY with a helpful message (e.g. 'no eval-id provided and none found ... run eval init first or pass an eval-id') and a non-zero exit — no crash or stack trace." - - "Take a screenshot of the eval init, list, and show output." - - "Report a finding ONLY if: eval init hangs despite --no-wait, the async job IDs are not surfaced, any of the commands crash / print a stack trace, or auth/endpoint errors occur. (An empty eval list or an eval-id-required message immediately after init is expected, not a failure.)" + - "Take a screenshot of the eval generate, list, and show output." + - "Report a finding ONLY if: eval generate hangs despite --no-wait, the async job IDs are not surfaced, any of the commands crash / print a stack trace, or auth/endpoint errors occur. (An empty eval list or an eval-id-required message immediately after generate is expected, not a failure.)" post: - run: "echo 'eval-lifecycle complete; eval runs are managed in Foundry and incur no standing local resources'" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml index f5dde47f2bd..d4ef4112fe8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -2,7 +2,7 @@ # # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. # Optimization is an expensive iterative process, so this scenario caps it at a -# single iteration, submits with --no-wait, then immediately cancels to avoid +# single candidate, submits with --no-wait, then immediately cancels to avoid # incurring a full optimization run. The agent is auto-detected from azure.yaml. # # ⚠️ ADDITIONAL PREREQUISITE — Agent Optimizer enrollment: the optimize command @@ -15,7 +15,7 @@ # behavior is the clean gated error below (capture it and stop) — that is an # acceptable outcome for this scenario, not a product failure. name: "optimize-submit-and-cancel" -command: "azd ai agent optimize --eval-model {model} --max-iterations 1 --no-wait" +command: "azd ai agent optimize --eval-model {model} --max-candidates 1 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:optimize", "serial-only"] @@ -27,7 +27,7 @@ pre: continue_on_error: true goals: - - "Run 'optimize --eval-model {model} --max-iterations 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." + - "Run 'optimize --eval-model {model} --max-candidates 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." - "GATING CHECK: if the command (or 'optimize list') returns a clean HTTP 400 with error code 'SubscriptionNotRegistered' (subscription not registered for Agent Optimizer, signup at https://aka.ms/ao/quickstart), this subscription is not enrolled. Confirm the error is CLEAN (structured error body, code, signup link, non-zero exit — no crash/hang), capture a screenshot, report it as an enrollment-gap finding, and STOP — the remaining list/status/cancel steps cannot run without enrollment." - "If the job IS submitted successfully (subscription enrolled): confirm the command returns without blocking (because of --no-wait) and surfaces a job identifier." - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml index 756108a70a2..d5a4b88e24b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml @@ -15,7 +15,7 @@ pre: goals: - "Wait for the endpoint configuration table to render (auto-resolves agent from azure.yaml)." - "Confirm the output shows protocol(s) configured on the agent (e.g. 'responses')." - - "Confirm the endpoint URL or base path is displayed." - - "Run the command again with '--output json' and confirm valid JSON is returned with protocol and endpoint fields." + - "Confirm the version selector and authorization sections are displayed (may show defaults if not explicitly configured)." + - "Run the command again with '--output json' and confirm valid JSON is returned with 'name', 'agent_endpoint', and 'agent_card' fields." - "Take a screenshot of both the table and JSON outputs." - "Report a finding if the command errors, shows no endpoint config, or returns unexpected empty fields for a deployed agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml index e2cd3087dd8..7ddb4b6684d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml @@ -1,25 +1,27 @@ -# Tier 2 (cloud E2E) — `code download` against a container-based agent (negative path). +# Tier 2 (cloud E2E) — `code download` against a code-based agent (positive path). # -# The shared agent uses Docker (container) deploy mode, not code_configuration. -# `code download` is only supported for code-based agents. This scenario verifies -# the CLI returns a clear, actionable error (AgentNotCodeBased / 409) rather than -# a confusing or generic failure. +# The shared agent uses code deploy mode (auto-selected for Python templates). +# `code download` retrieves the deployed source. This scenario verifies the CLI +# downloads the agent code and extracts it into the destination directory. # # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -name: "code-download-not-code-based" +name: "code-download" command: "azd ai agent code download --dest ~/working/azd-agents-shared/code-download-test" cwd: "~/working/azd-agents-shared/{shared_agent_name}" -tags: ["tier:2", "cmd:code", "serial-only", "negative-path"] +tags: ["tier:2", "cmd:code", "serial-only"] pre: - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true + - run: "rm -rf ~/working/azd-agents-shared/code-download-test" + name: "clean destination directory" + continue_on_error: true goals: - - "Run code download against the shared agent (which is container-based, NOT code-based)." - - "Confirm the command returns a non-zero exit code and a clear error message indicating the agent is not code-based (e.g. 'AgentNotCodeBased' or 'not a code-based agent')." - - "Confirm the error message is user-friendly and actionable — not a raw stack trace or generic HTTP error." - - "Take a screenshot of the error output." - - "Report a finding if the command succeeds unexpectedly, hangs, panics, or gives a confusing/generic error message instead of a clear 'not code-based' explanation." + - "Run code download against the shared agent (which is code-based)." + - "Confirm the command exits successfully (zero exit code)." + - "Confirm that agent source files are downloaded into the destination directory (~/working/azd-agents-shared/code-download-test). Run 'ls ~/working/azd-agents-shared/code-download-test' to verify files exist (e.g. app.py, requirements.txt, or similar Python source files)." + - "Take a screenshot of the successful download output and the directory listing." + - "Report a finding if the command fails, hangs, returns an error about the agent not being code-based, or downloads an empty directory." From b1143d6040292ac74a1bdb19590a056d2036930e Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 17 Jul 2026 08:54:37 -0700 Subject: [PATCH 29/42] fix: address run-20260716 scenario failures - 10-init-template-python: fix scaffold verification goal (no 'instructions' or literal 'model' in schema; model is via env var). Remove stale Container deploy goal (auto-selects code for templates). - 10-init-template-dotnet: same Container deploy fix (.NET also auto-selects code) - 26-endpoint-update: rewrite pre-hook with sed (embedded Python broke YAML parser) - 00-sample-list-text: relax goal for terminal buffer overflow (JSON scenario covers full data verification) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../00-sample-list-text.yaml | 4 ++-- .../10-init-template-dotnet.yaml | 4 ++-- .../10-init-template-python.yaml | 6 ++--- .../26-endpoint-update.yaml | 22 +++++-------------- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml index 76323265ec5..481b6b18a27 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -6,6 +6,6 @@ tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Wait for the curated sample catalog to render as human-readable text." - - "Confirm at least one sample entry is shown with a name/title and a manifest or repo reference usable with 'azd ai agent init -m' or 'azd init -t'." + - "Confirm at least one sample entry is visible with a name/title and a manifest or repo reference usable with 'azd ai agent init -m' or 'azd init -t'. NOTE: the output may overflow the terminal buffer — verifying the full list is not required; confirm the command completed successfully (clean exit, no error) and that visible entries are well-formatted." - "Take a screenshot of the catalog output." - - "Report a finding if the list is empty, truncated, or the command errors." + - "Report a finding if the command errors, shows zero entries, or the formatting is broken (missing columns, garbled text)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 74b4bf1e555..b4b427efb24 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -21,13 +21,13 @@ goals: - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." - - "When asked how to deploy, select 'Container' (hosted agent)." + - "Deploy mode is auto-selected as 'code' for .NET templates — no deploy-mode prompt should appear. Report a finding if one does." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - "If asked for a location/region, select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - - "If asked for container/resource size, select 'Small'." + - "If asked for container/resource size, select 'Small'. NOTE: with code deploy auto-selected, this prompt is unlikely." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." - "Take a screenshot of the completed init output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index 43cce403902..6b6f29a34b2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -24,14 +24,14 @@ goals: - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." - - "When asked how to deploy, select 'Container' (hosted agent)." + - "Deploy mode is auto-selected as 'code' for Python templates — no deploy-mode prompt should appear. Report a finding if one does." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - "If asked for a location/region, select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - - "If asked for container/resource size, select 'Small'." + - "If asked for container/resource size, select 'Small'. NOTE: with code deploy auto-selected, this prompt is unlikely." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service with the agent definition (name, model, instructions) embedded inline in the service entry properties." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service with the agent definition embedded inline (agent name, kind, protocols, and environmentVariables referencing AZURE_AI_MODEL_DEPLOYMENT_NAME for the model)." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding for any confusing prompt, wrong default, or scaffold problem." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 7fb8214e937..94ed5d8d269 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -18,24 +18,12 @@ pre: # minimal agentCard into the azure.yaml service properties (idempotently) before # the run so there is something to patch. - run: | - f="~/working/azd-agents-shared/{shared_agent_name}/azure.yaml" + f="$HOME/working/azd-agents-shared/{shared_agent_name}/azure.yaml" if [ -f "$f" ] && ! grep -q 'agentCard:' "$f"; then - python3 -c " -import sys, yaml -with open(sys.argv[1]) as fh: - doc = yaml.safe_load(fh) -for svc in doc.get('services', {}).values(): - if svc.get('host') == 'azure.ai.agent': - props = svc.setdefault('config', {}) - props['agentCard'] = { - 'description': '{prefix} endpoint-update test card', - 'skills': [{'id': '{prefix}-echo', 'name': 'Echo', 'description': 'Echoes input back'}] - } - break -with open(sys.argv[1], 'w') as fh: - yaml.dump(doc, fh, default_flow_style=False, sort_keys=False) -print('Injected agentCard into azure.yaml') -" "$f" + # Append agentCard under the config key of the azure.ai.agent service. + # Find the "config:" line under the agent service and append after it. + sed -i '/^ config:/a\ agentCard:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"' "$f" + echo "Injected agentCard into azure.yaml" else echo "agentCard already present or azure.yaml not found" fi From 3e673bff295a9c28bb04540138732d247a23a30b Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 17 Jul 2026 12:54:16 -0700 Subject: [PATCH 30/42] fix: address run-20260717 scenario failures - Location select: use 'search for and select' wording (8 scenarios) to guide text-based matching instead of fragile positional indexing - endpoint-update: replace sed with awk for reliable multi-line agentCard injection into azure.yaml - optimize: add --evaluator coherence flag (required by validate) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../10-init-deploy-mode-code.yaml | 2 +- .../10-init-deploy-mode-container.yaml | 2 +- .../10-init-flags-agent-name-model.yaml | 2 +- .../10-init-from-azure-yaml-url.yaml | 2 +- .../cli-interactive-tester-scenarios/10-init-from-code.yaml | 2 +- .../10-init-template-dotnet.yaml | 2 +- .../10-init-template-python.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 2 +- .../26-endpoint-update.yaml | 6 +++--- .../29-optimize-submit-and-cancel.yaml | 4 ++-- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 122414a42b5..91987f4cbc4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -31,7 +31,7 @@ goals: - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml reflects code-deploy mode — the azure.ai.agent service entry should have codeConfiguration with the chosen entry point and runtime embedded in its properties, and a .agentignore file should control ZIP packaging." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml index c41089d44ba..f112788e53b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml @@ -30,7 +30,7 @@ goals: - "If an existing agent manifest is detected, confirm reuse." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml reflects container-deploy mode — the azure.ai.agent service entry should have docker/Dockerfile configuration in its properties rather than codeConfiguration with entry-point/runtime fields." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 86dab30a67a..4ffdd490e6f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -33,7 +33,7 @@ goals: - "Wait for the azure.yaml to download and parse." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the overrides: confirm azure.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the sample defaults." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml index 49ce5685e03..6b8c96d531b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml @@ -33,7 +33,7 @@ goals: - "Wait for the tool to fetch and parse the azure.yaml from the provided URL." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "When asked to select a model, choose '{model}' (or accept the sample's model if one is pinned)." - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "Wait for initialization to complete — look for 'Next:' or a success message." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 1611ab5b7ab..1b1290a1dae 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -28,7 +28,7 @@ goals: - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' in the output." - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index b4b427efb24..627738f8ff2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -24,7 +24,7 @@ goals: - "Deploy mode is auto-selected as 'code' for .NET templates — no deploy-mode prompt should appear. Report a finding if one does." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'. NOTE: with code deploy auto-selected, this prompt is unlikely." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index 6b6f29a34b2..674d60e180a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -27,7 +27,7 @@ goals: - "Deploy mode is auto-selected as 'code' for Python templates — no deploy-mode prompt should appear. Report a finding if one does." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'. NOTE: with code deploy auto-selected, this prompt is unlikely." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index d10b8067fbd..1935ee11d88 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -45,7 +45,7 @@ goals: - "When prompted for the AGENT NAME, set it to EXACTLY '{shared_agent_name}' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named '{shared_agent_name}'." - "If asked to select an Azure AI Foundry project, create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." - - "If asked for a location/region, select '{region}'." + - "If asked for a location/region, search for and select '{region}'." - "When asked to select a model, choose '{model}'." - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "Wait for initialization to complete — look for 'Next:' or a success message." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 94ed5d8d269..6928583cfd8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -20,9 +20,9 @@ pre: - run: | f="$HOME/working/azd-agents-shared/{shared_agent_name}/azure.yaml" if [ -f "$f" ] && ! grep -q 'agentCard:' "$f"; then - # Append agentCard under the config key of the azure.ai.agent service. - # Find the "config:" line under the agent service and append after it. - sed -i '/^ config:/a\ agentCard:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"' "$f" + # Insert agentCard block after the "config:" line using awk. + # awk is more reliable than sed for multi-line insertion. + awk '/^ config:/{print; print " agentCard:"; print " description: \"{prefix} endpoint-update test card\""; print " skills:"; print " - id: \"{prefix}-echo\""; print " name: \"Echo\""; print " description: \"Echoes input back\""; next}1' "$f" > "${f}.tmp" && mv "${f}.tmp" "$f" echo "Injected agentCard into azure.yaml" else echo "agentCard already present or azure.yaml not found" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml index d4ef4112fe8..bb50e83ee04 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -15,7 +15,7 @@ # behavior is the clean gated error below (capture it and stop) — that is an # acceptable outcome for this scenario, not a product failure. name: "optimize-submit-and-cancel" -command: "azd ai agent optimize --eval-model {model} --max-candidates 1 --no-wait" +command: "azd ai agent optimize --eval-model {model} --evaluator coherence --max-candidates 1 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:optimize", "serial-only"] @@ -27,7 +27,7 @@ pre: continue_on_error: true goals: - - "Run 'optimize --eval-model {model} --max-candidates 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." + - "Run 'optimize --eval-model {model} --evaluator coherence --max-candidates 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." - "GATING CHECK: if the command (or 'optimize list') returns a clean HTTP 400 with error code 'SubscriptionNotRegistered' (subscription not registered for Agent Optimizer, signup at https://aka.ms/ao/quickstart), this subscription is not enrolled. Confirm the error is CLEAN (structured error body, code, signup link, non-zero exit — no crash/hang), capture a screenshot, report it as an enrollment-gap finding, and STOP — the remaining list/status/cancel steps cannot run without enrollment." - "If the job IS submitted successfully (subscription enrolled): confirm the command returns without blocking (because of --no-wait) and surfaces a job identifier." - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." From 4c0abbfb2e20a0897af967d30794b86c1cd7cd75 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 17 Jul 2026 14:45:03 -0700 Subject: [PATCH 31/42] fix: tighten observation goals for sample-list and optimize - sample-list-json-filters: assert --type agent returns non-empty results - optimize-submit-and-cancel: broaden gating check to any clean HTTP 400 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../00-sample-list-json-filters.yaml | 2 +- .../29-optimize-submit-and-cancel.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml index 55711345ff0..2976c0e75c7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -7,7 +7,7 @@ tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." - "Run: azd ai agent sample list --language python --output json. Confirm results are filtered to python samples only." - - "Run: azd ai agent sample list --type agent --output json. Confirm results only include agent-type templates." + - "Run: azd ai agent sample list --type agent --output json. Confirm results only include agent-type templates and that the result set is non-empty (at least one agent-type sample should exist in the catalog)." - "Run: azd ai agent sample list --featured-only --output json. Confirm only featured samples are returned (a subset of the full list)." - "Take a screenshot after each command." - "Report a finding if any command produces invalid JSON, ignores its filter, or errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml index bb50e83ee04..69ab1a93f96 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -28,7 +28,7 @@ pre: goals: - "Run 'optimize --eval-model {model} --evaluator coherence --max-candidates 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." - - "GATING CHECK: if the command (or 'optimize list') returns a clean HTTP 400 with error code 'SubscriptionNotRegistered' (subscription not registered for Agent Optimizer, signup at https://aka.ms/ao/quickstart), this subscription is not enrolled. Confirm the error is CLEAN (structured error body, code, signup link, non-zero exit — no crash/hang), capture a screenshot, report it as an enrollment-gap finding, and STOP — the remaining list/status/cancel steps cannot run without enrollment." + - "GATING CHECK: if the command returns a clean HTTP 400 (e.g. error code 'SubscriptionNotRegistered', 'train_dataset.version required', or another API validation error), confirm the error is CLEAN (structured error body, non-zero exit — no crash/hang), capture a screenshot, report it as an observation finding describing the specific error, and STOP — the remaining list/status/cancel steps cannot run. This is an acceptable outcome, not a product failure." - "If the job IS submitted successfully (subscription enrolled): confirm the command returns without blocking (because of --no-wait) and surfaces a job identifier." - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." - "Run 'azd ai agent optimize status ' using the job id and confirm it prints status details without crashing (a queued/running status is acceptable)." From 22e67d348d3266aace608f845cd41e33e8dc4c15 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 17 Jul 2026 14:45:35 -0700 Subject: [PATCH 32/42] fix: optimize gating check should only accept SubscriptionNotRegistered Other HTTP 400 errors (e.g. train_dataset.version required) are unexpected API errors and should be reported as bugs, not observations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../29-optimize-submit-and-cancel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml index 69ab1a93f96..46335b5bca2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml @@ -28,7 +28,7 @@ pre: goals: - "Run 'optimize --eval-model {model} --evaluator coherence --max-candidates 1 --no-wait' to submit an optimization job without blocking; the agent is auto-detected from azure.yaml. Work through the prompts (use the project eval.yaml, accept the baseline instruction file, skip skills/tools, provide a dataset, accept the optimization model)." - - "GATING CHECK: if the command returns a clean HTTP 400 (e.g. error code 'SubscriptionNotRegistered', 'train_dataset.version required', or another API validation error), confirm the error is CLEAN (structured error body, non-zero exit — no crash/hang), capture a screenshot, report it as an observation finding describing the specific error, and STOP — the remaining list/status/cancel steps cannot run. This is an acceptable outcome, not a product failure." + - "GATING CHECK: if the command (or 'optimize list') returns an HTTP 400 with error code 'SubscriptionNotRegistered' (subscription not registered for Agent Optimizer, signup at https://aka.ms/ao/quickstart), this subscription is not enrolled. Confirm the error is CLEAN (structured error body, code, signup link, non-zero exit — no crash/hang), capture a screenshot, report it as an enrollment-gap observation, and STOP — the remaining list/status/cancel steps cannot run without enrollment. Any OTHER HTTP 400 (e.g. 'train_dataset.version required') is an unexpected API error — report it as a bug finding." - "If the job IS submitted successfully (subscription enrolled): confirm the command returns without blocking (because of --no-wait) and surfaces a job identifier." - "Run 'azd ai agent optimize list' and confirm the just-submitted job appears with a status." - "Run 'azd ai agent optimize status ' using the job id and confirm it prints status details without crashing (a queued/running status is acceptable)." From 39ecddd436aee209eae929fd9001e4d6ecac7b00 Mon Sep 17 00:00:00 2001 From: trangevi Date: Wed, 22 Jul 2026 16:51:46 -0600 Subject: [PATCH 33/42] docs: address PR review comments for README, profile, and skill refs - profile.yaml: fix stale gpt-4.1-mini comment (value is gpt-5.4-mini) - README: fix model default in profile table, soften tier 0 'offline' claim (sample-list needs network), extend tier 2 range to 2D, add 2B/2C/2D rows, add cmd:code and cmd:delete to tag taxonomy, fix eval description - impact-mapping: add code/delete command mappings, add agent_endpoint.go - workflow.md: remove stale eval/optimize coverage gap refs, extend range - running-scenarios.md: extend serial range to 2D - reporting.md: update sample report (eval has scenarios now) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/impact-mapping.md | 4 +++- .../agent-scenario-tests/references/reporting.md | 2 +- .../references/running-scenarios.md | 2 +- .../agent-scenario-tests/references/workflow.md | 6 +++--- .../cli-interactive-tester-scenarios/README.md | 14 +++++++++----- .../cli-interactive-tester-scenarios/profile.yaml | 2 +- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/agent-scenario-tests/references/impact-mapping.md index aef60d48193..bd5878a7a56 100644 --- a/.github/skills/agent-scenario-tests/references/impact-mapping.md +++ b/.github/skills/agent-scenario-tests/references/impact-mapping.md @@ -24,6 +24,8 @@ Files under `internal/cmd/` map to the command they implement: | `internal/cmd/eval*.go` | `cmd:eval` | `eval.go`, `eval_init.go`, `eval_run.go`, `eval_list.go`, `eval_show.go`, etc. Tier 2 (needs a deployed agent + Foundry endpoint). | | `internal/cmd/optimize*.go` | `cmd:optimize` | `optimize.go`, `optimize_apply.go`, `optimize_status.go`, etc. Tier 2 (submits a cloud optimization job). | | `internal/cmd/sample*.go` | `cmd:sample` | `sample.go`, `sample_list.go`. | +| `internal/cmd/code*.go` | `cmd:code` | `code.go` (code download). | +| `internal/cmd/delete*.go` | `cmd:delete` | `delete.go` (agent deletion). | | `internal/cmd/version.go` | `cmd:version` | | | `internal/cmd/root.go` | `cmd:help` + broad | Touches the whole command tree — treat as broad (see §3). | | `internal/cmd/listen.go` | — | gRPC host entrypoint; not scenario-testable. | @@ -45,7 +47,7 @@ these, broaden the impacted set (and ask the user how wide to go): | Changed file (glob) | Broaden to | | --- | --- | -| `internal/cmd/helpers.go`, `internal/cmd/agent_context.go`, `internal/cmd/*_context.go` | All `cmd:*` for commands that resolve project/agent context — at minimum `cmd:init`, `cmd:invoke`, `cmd:show`, `cmd:doctor`. | +| `internal/cmd/helpers.go`, `internal/cmd/agent_context.go`, `internal/cmd/agent_endpoint.go`, `internal/cmd/*_context.go` | All `cmd:*` for commands that resolve project/agent context — at minimum `cmd:init`, `cmd:invoke`, `cmd:show`, `cmd:doctor`. | | `internal/cmd/root.go`, `internal/cmd/banner.go`, `internal/cmd/nextstep_output.go` | Run a Tier 0 smoke set (`tier:0`) across all commands. | | `internal/pkg/**`, `internal/project/**`, `internal/exterrors/**` | Map by what the package feeds: parsers/manifests → `cmd:init`; deployment/project target → `cmd:provision` + `cmd:deploy` (Tier 2). When unclear, propose a Tier 0/1 sweep and ask before any Tier 2. | | `go.mod` / `go.sum` / dependency bumps | Tier 0 smoke + ask whether a fuller sweep is warranted. | diff --git a/.github/skills/agent-scenario-tests/references/reporting.md b/.github/skills/agent-scenario-tests/references/reporting.md index 186f7c66f9e..e5b9e24a84b 100644 --- a/.github/skills/agent-scenario-tests/references/reporting.md +++ b/.github/skills/agent-scenario-tests/references/reporting.md @@ -35,7 +35,7 @@ formatting). Keep it scannable — full detail lives in the artifact. Suggested - `22-invoke-new-session`: `--new-conversation` still recalled the prior name — memory was not reset. (screenshot: …) -**Coverage gaps:** this PR also touches `eval*.go`, which has no scenario — consider adding one. +**Coverage gaps:** this PR also touches `mcp.go`, which has no scenario — consider adding one. Run locally via the `agent-scenario-tests` skill. Not run in CI. ``` diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index 4d66f27128a..d7f7f4f6e7a 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -77,7 +77,7 @@ For each selected scenario: - **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's parallel-readiness section for which scenarios support it. - **Tier 2** (`serial-only`): never parallelize. Run `20-setup-deploy-shared-agent` first, - then `21-…2A-` serially (they share one deployed agent and mutate shared session/file/ + then `21-…2D-` serially (they share one deployed agent and mutate shared session/file/ endpoint state), then `2Z-teardown-down` last. - **Validate the recipe with one scenario before fanning out** — confirm `load_scenario` → `start_session` → one `send_action` round-trips for a single Tier 0 scenario first. diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md index f30365ac107..9bf7493132d 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -45,8 +45,8 @@ gh pr view --json number,url,headRefName,baseRefName,title 2. Map those files to scenario tags using `impact-mapping.md`. The result is: - a set of `cmd:*` tags (which commands changed), - the **highest tier** you should offer (cost gating), and - - any **coverage gaps** (changed commands that have *no* scenario yet — e.g. `eval`, - `optimize`, `mcp`). Surface gaps to the user; do not silently skip them. + - any **coverage gaps** (changed commands that have *no* scenario yet — e.g. `mcp`). + Surface gaps to the user; do not silently skip them. 3. Enumerate matching scenarios via the tester: @@ -77,7 +77,7 @@ Drive each selected scenario per `running-scenarios.md`. Honor ordering: - **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each with its own `cwd` (no `instance_id` needed for distinct scenarios). - **Tier 2** is `serial-only` and order-dependent: `20-setup-deploy-shared-agent` **first**, - then the targeted `21-…2A-` scenarios **serially**, then `2Z-teardown-down` **last**. + then the targeted `21-…2D-` scenarios **serially**, then `2Z-teardown-down` **last**. Record per scenario: PASS/FAIL, wall-clock duration (`Hh Mm Ss`), and any `report_finding` entries. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 366b9821d06..e2d5e206d8f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -332,7 +332,8 @@ command(s) under test — see [Tags](#tags) for the full taxonomy and how to filter via `list_scenarios`. ### Tier 0 — Offline (prefix `00-`) -No Azure auth, no network resource creation. Fast and deterministic. Safe to run +No Azure auth, no network resource creation (except `sample list`, which fetches +the public template catalog). Fast and mostly deterministic. Safe to run in any order, any time. | File | Targets | @@ -371,7 +372,7 @@ and verifies the generated files, then stops before `azd provision`. Provisions real resources. **Run order matters:** 1. `20-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. -2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). +2. Any `21-`…`2D-` targeted scenario (reuse the deployed agent). 3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` @@ -395,9 +396,12 @@ as their `cwd`. | `25-monitor-system.yaml` | `monitor --type system` | | `26-endpoint-update.yaml` | `endpoint update` | | `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | -| `28-eval-lifecycle.yaml` | `eval init/run/list/show` against the shared agent (small sample budget, `--no-wait`) | +| `28-eval-lifecycle.yaml` | `eval generate/list/show` against the shared agent (small sample budget, `--no-wait`) | | `29-optimize-submit-and-cancel.yaml` | `optimize` submit + `list`/`status`/`cancel` (capped at 1 iteration, `--no-wait`) | | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | +| `2B-endpoint-show.yaml` | `endpoint show` (agent endpoint details) | +| `2C-code-download.yaml` | `code download` (positive-path: downloads agent source code) | +| `2D-delete.yaml` | `delete` (destroys the shared agent — run before teardown) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | ## Tags @@ -413,7 +417,7 @@ grouping — colons are treated as ordinary characters by the filter): | Namespace | Values | Meaning | |---|---|---| | `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | -| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help`, `cmd:code`, `cmd:delete` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | | traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | **Examples** (the tool's `tags:` parameter is OR across the list): @@ -506,7 +510,7 @@ Variables exposed to scenarios via `session_vars`: | `{subscription}` | `profile.local.yaml` | **required** | subscription display name | | `{tenant}` | `profile.local.yaml` | optional, no default | only consumed by the `az login` guidance above; when unset, drop `--tenant` and rely on the user's default tenant | | `{region}` | `profile.yaml` | `East US 2` | | -| `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | +| `{model}` | `profile.yaml` | `gpt-5.4-mini` | cheap/fast for tests | | `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | | `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}-{ts}` | Tier 2 subdirectory name — orchestrator must compute (with `{ts}` = `MMDDHHmm` compact timestamp) and pass alongside the others. The timestamp isolates concurrent runs. | | `{fixtures_dir}` | derived by orchestrator | `/fixtures` | Tester-side absolute path to the `fixtures/` subdirectory (WSL-translated on Windows, native on Linux/macOS); used by pre-hooks to seed test fixture files | diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml index ebfc9bbace2..71f2585df48 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml @@ -15,7 +15,7 @@ region: "East US 2" # Default model deployment chosen during init / referenced by --model flags. -# gpt-4.1-mini is cheap and fast — appropriate for tests. +# gpt-5.4-mini is cheap and fast — appropriate for tests. model: "gpt-5.4-mini" # Suffix appended to {prefix} to form the Tier 2 shared agent's name (and From d790f42dfc0fd256e6f75b337dd4054e8ece4634 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 23 Jul 2026 14:30:20 -0600 Subject: [PATCH 34/42] fix: rewrite setup-wsl.sh to use azd x tooling for reliable dev-build registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the script manually copied the extension binary without registering it in azd's config, which meant azd could silently fall back to the published registry version instead of the dev build. Now the script uses the official azd extension tooling: 1. Builds azd core with sudo install to /usr/local/bin 2. Ensures microsoft.azd.extensions is available (auto-installs if needed) 3. azd x build → azd x pack --bundle → azd extension install 4. Strict version verification that fails the script if dev build isn't active Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../README.md | 11 ++- .../setup-wsl.sh | 99 +++++++++++++++---- 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index e2d5e206d8f..68e01cc5ac6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -127,11 +127,16 @@ bash setup-wsl.sh This script: 1. Cross-compiles `azd` core (`linux/amd64`) → `/usr/local/bin/azd` -2. Cross-compiles the extension (`linux/amd64`) → `~/.azd/extensions/azure.ai.agents/` -3. Prints version confirmation +2. Ensures the extensions dev kit (`microsoft.azd.extensions`) is installed +3. Builds, packages, and installs the `azure.ai.agents` extension from source + using `azd x build` → `azd x pack --bundle` → `azd extension install` +4. Verifies both azd and the extension report expected dev versions + +The script properly registers the extension in azd's config, so it will always +use your dev build — never the published registry version. **Re-run `setup-wsl.sh` after every local code change** you want to test. -Requires the Go toolchain installed in WSL. +Requires the Go toolchain and sudo access in WSL. ## Authentication diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh index 07028637baf..61a59b59576 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh @@ -2,12 +2,13 @@ # setup-wsl.sh — Build and install native Linux azd + extension for WSL testing. # # Run this from inside WSL (or via `wsl bash setup-wsl.sh` from Windows) after -# making local code changes. It cross-compiles native Linux/amd64 binaries from -# the repo source so the cli-interactive-tester drives your dev build directly. +# making local code changes. It compiles native Linux/amd64 binaries from the +# repo source so the cli-interactive-tester drives your dev build directly. # # Prerequisites: # - Go toolchain installed in WSL (or accessible via PATH) # - Git installed in WSL +# - sudo access (for installing azd to /usr/local/bin) # # Usage: # cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios @@ -15,8 +16,9 @@ # # What it does: # 1. Builds azd core (linux/amd64) → /usr/local/bin/azd -# 2. Builds the azure.ai.agents extension (linux/amd64) → ~/.azd/extensions/ -# 3. Prints version confirmation +# 2. Ensures the azd extensions dev kit (microsoft.azd.extensions) is installed +# 3. Builds + packages + installs the azure.ai.agents extension from source +# 4. Verifies the dev version is running set -euo pipefail @@ -32,6 +34,17 @@ echo " azd source: $AZD_DIR" echo " Extension src: $EXTENSION_DIR" echo "" +# --- Prerequisites --- +if ! command -v go &>/dev/null; then + echo "ERROR: Go toolchain not found. Install Go in WSL first." >&2 + exit 1 +fi + +if ! sudo -n true 2>/dev/null; then + echo "NOTE: sudo access is needed to install azd to /usr/local/bin." + echo " You may be prompted for your password." +fi + # --- Step 1: Build azd core --- echo "▸ Building azd core (linux/amd64)..." @@ -41,37 +54,81 @@ LDFLAGS="-X 'github.com/azure/azure-dev/cli/azd/internal.Version=${VERSION} (com (cd "$AZD_DIR" && GOOS=linux GOARCH=amd64 go build \ -ldflags="$LDFLAGS" \ - -o /usr/local/bin/azd \ + -o /tmp/azd-dev-build \ .) +sudo install -m 755 /tmp/azd-dev-build /usr/local/bin/azd +rm -f /tmp/azd-dev-build + echo " ✓ Installed /usr/local/bin/azd" echo "" -# --- Step 2: Build the extension --- +# --- Step 2: Ensure microsoft.azd.extensions is available --- +echo "▸ Checking for azd extensions dev kit (microsoft.azd.extensions)..." + +if azd x version &>/dev/null; then + echo " ✓ microsoft.azd.extensions is already installed" +else + echo " → Installing microsoft.azd.extensions from registry..." + azd extension install microsoft.azd.extensions --no-prompt + echo " ✓ Installed microsoft.azd.extensions" +fi +echo "" + +# --- Step 3: Build extension from source --- echo "▸ Building azure.ai.agents extension (linux/amd64)..." +azd x build -C "$EXTENSION_DIR" +echo " ✓ Extension built" +echo "" -EXTENSION_INSTALL_DIR="$HOME/.azd/extensions/azure.ai.agents" -mkdir -p "$EXTENSION_INSTALL_DIR" +# --- Step 4: Package as bundle --- +echo "▸ Packaging extension bundle..." +azd x pack --bundle -C "$EXTENSION_DIR" -EXT_COMMIT=$(cd "$EXTENSION_DIR" && git rev-parse HEAD 2>/dev/null || echo "unknown") -EXT_BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# Find the generated bundle zip EXT_VERSION=$(cat "$EXTENSION_DIR/version.txt" 2>/dev/null || echo "0.0.0-dev") -VERSION_PATH="azureaiagent/internal/version" +BUNDLE_ZIP="$EXTENSION_DIR/azure-ai-agents_${EXT_VERSION}.zip" -(cd "$EXTENSION_DIR" && GOOS=linux GOARCH=amd64 go build \ - -ldflags="-X '${VERSION_PATH}.Version=${EXT_VERSION}' -X '${VERSION_PATH}.Commit=${EXT_COMMIT}' -X '${VERSION_PATH}.BuildDate=${EXT_BUILD_DATE}'" \ - -o "$EXTENSION_INSTALL_DIR/azure-ai-agents-linux-amd64" \ - .) +if [ ! -f "$BUNDLE_ZIP" ]; then + echo "ERROR: Expected bundle not found at $BUNDLE_ZIP" >&2 + echo " Check the output above for packaging errors." >&2 + exit 1 +fi -# Copy extension.yaml (azd needs it to discover the extension) -cp "$EXTENSION_DIR/extension.yaml" "$EXTENSION_INSTALL_DIR/extension.yaml" +echo " ✓ Bundle created: $BUNDLE_ZIP" +echo "" -echo " ✓ Installed $EXTENSION_INSTALL_DIR/azure-ai-agents-linux-amd64" +# --- Step 5: Install from bundle --- +echo "▸ Installing extension from bundle..." +azd extension install "$BUNDLE_ZIP" --force --no-prompt +echo " ✓ Extension installed and registered" echo "" -# --- Step 3: Verify --- +# Clean up the bundle zip +rm -f "$BUNDLE_ZIP" + +# --- Step 6: Verify --- echo "▸ Verifying installation..." -echo " azd version: $(azd version 2>&1 | head -1)" -echo " extension: $(azd ai agent version 2>&1 | grep -i version | head -1)" + +AZD_VER=$(azd version 2>&1 | head -1) +echo " azd version: $AZD_VER" + +if ! echo "$AZD_VER" | grep -q "$VERSION"; then + echo "ERROR: azd version does not contain expected dev version '$VERSION'" >&2 + echo " Got: $AZD_VER" >&2 + echo " This suggests the dev build was not installed correctly." >&2 + exit 1 +fi + +EXT_VER=$(azd ai agent version 2>&1) +echo " extension: $EXT_VER" + +if ! echo "$EXT_VER" | grep -qi "version"; then + echo "ERROR: Failed to get extension version. Is it properly registered?" >&2 + echo " Got: $EXT_VER" >&2 + echo " Try 'azd extension list' to check installed extensions." >&2 + exit 1 +fi + echo "" echo "=== Done. WSL is ready for scenario testing. ===" From 8f24eac40be6fcce905dea0a326c4bc6019a2962 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 23 Jul 2026 15:35:34 -0600 Subject: [PATCH 35/42] fix: harden endpoint-update scenario hooks and goals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove continue_on_error from agentCard injection hook (if injection fails, the scenario cannot succeed — let it fail fast) - Add pre-update version capture goal so the driving agent records the version BEFORE running endpoint update, enabling a concrete comparison Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../cli-interactive-tester-scenarios/26-endpoint-update.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 6928583cfd8..4e631418aa1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -29,9 +29,9 @@ pre: fi cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "inject agentCard into azure.yaml so there is something to patch" - continue_on_error: true goals: + - "BEFORE running endpoint update, run 'azd ai agent show' first and note the current agent version number — you will need it to verify the update did not create a new version." - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agentCard was injected into azure.yaml service properties during setup so the patch has content." - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." From 56f4913e99270cb710fadff5d4a4dec161e6ec2b Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 23 Jul 2026 15:48:25 -0600 Subject: [PATCH 36/42] fix: replace agent.manifest.yaml references with azure.yaml - Update mutually-exclusive validation scenario to use azure.yaml - Remove legacy agent.manifest.yaml mention from init-from-url comment Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../00-init-validate-mutually-exclusive.yaml | 2 +- .../10-init-from-azure-yaml-url.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index fafecf07551..b810c91ada4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -4,7 +4,7 @@ # BOTH a positional manifest argument AND -m/--manifest, which the command # rejects up front (CodeConflictingArguments) before any wizard or network call. name: "init-validate-mutually-exclusive" -command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" +command: "azd ai agent init azure.yaml -m https://example.com/azure.yaml" cwd: "~/working/azd-agents-validate-{instance}" tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml index 6b8c96d531b..a07819750dd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml @@ -7,8 +7,7 @@ # API is rate-limited, which would otherwise drop into an interactive gh login # mid-run. The pre hook fails fast if gh is not authenticated. # -# NOTE: The `-m` flag now accepts an azure.yaml URL (unified project manifest) -# in addition to legacy agent.manifest.yaml files. With a Python project URL and +# NOTE: The `-m` flag accepts an azure.yaml URL (unified project manifest). With a Python project URL and # userProvidedManifest=true, code deploy is auto-selected (no deploy-mode prompt). name: "init-from-azure-yaml-url" command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/azure.yaml" From ac1289d252aafb17d2ad29579829193039ff071f Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 23 Jul 2026 15:54:11 -0600 Subject: [PATCH 37/42] fix: add MMDDH to cspell word list for timestamp format patterns Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- cli/azd/extensions/azure.ai.agents/cspell.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 6ffab8210ec..58cb60140cd 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -12,6 +12,7 @@ words: - signup # Terms - Reprompt + - MMDDH # Azure region names - australiaeast - brazilsouth From 4ff3698eb229aa1643cfd0363e42f5d4043818e5 Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 28 Jul 2026 14:26:34 -0700 Subject: [PATCH 38/42] Restructure scenario files into tier subdirectories Move scenario YAML files from a flat directory into tier0/, tier1/, tier2/ subdirectories. Rename files from the old prefix scheme (00-, 10-, 2x-) to a structured format: {tier}.{two-digit-number}-{test-name}.yaml. This makes it immediately obvious which tier and test number a file belongs to, and allows each tier to scale independently without running out of identifiers. Updated all references in README.md, skill reference files, and docs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/skills/agent-scenario-tests/SKILL.md | 2 +- .../references/impact-mapping.md | 2 +- .../references/prerequisites.md | 6 +- .../references/reporting.md | 10 +- .../references/running-scenarios.md | 14 +- .../references/workflow.md | 8 +- .../README.md | 167 +++++++++--------- .../0.01-version.yaml} | 0 .../0.02-help-root.yaml} | 0 .../0.03-sample-list-text.yaml} | 0 .../0.04-sample-list-json-filters.yaml} | 0 .../0.05-doctor-empty-dir.yaml} | 0 .../0.06-doctor-local-only.yaml} | 0 .../0.07-doctor-partial-failure.yaml} | 0 ....08-init-validate-mutually-exclusive.yaml} | 0 ...0.09-init-validate-no-prompt-missing.yaml} | 0 .../0.10-init-picker-navigation.yaml} | 0 .../0.11-invoke-validate-protocol.yaml} | 0 .../0.12-eval-context-required.yaml} | 0 ...13-optimize-apply-requires-candidate.yaml} | 0 .../0.14-endpoint-show-help.yaml} | 0 .../0.15-code-download-help.yaml} | 0 .../0.16-delete-help.yaml} | 0 .../1.01-init-template-python.yaml} | 0 .../1.02-init-template-dotnet.yaml} | 0 .../1.03-init-from-azure-yaml-url.yaml} | 0 .../1.04-init-from-code.yaml} | 0 .../1.05-init-flags-agent-name-model.yaml} | 0 .../1.06-init-deploy-mode-code.yaml} | 0 .../1.07-init-deploy-mode-container.yaml} | 0 .../1.08-init-validate-deploy-mode.yaml} | 0 .../2.00-setup-deploy-shared-agent.yaml} | 0 .../{21-show.yaml => tier2/2.01-show.yaml} | 0 .../2.02-show-json.yaml} | 0 .../2.03-invoke-remote.yaml} | 0 .../2.04-invoke-new-session.yaml} | 0 .../2.05-invoke-input-file.yaml} | 0 .../2.06-invoke-protocol-invocations.yaml} | 0 .../2.07-sessions-lifecycle.yaml} | 0 .../2.08-files-lifecycle.yaml} | 0 .../2.09-monitor-console.yaml} | 0 .../2.10-monitor-system.yaml} | 0 .../2.11-endpoint-update.yaml} | 0 .../2.12-run-local-and-invoke-local.yaml} | 0 .../2.13-eval-lifecycle.yaml} | 0 .../2.14-optimize-submit-and-cancel.yaml} | 0 .../2.15-doctor-provisioned-all-pass.yaml} | 0 .../2.16-endpoint-show.yaml} | 0 .../2.17-code-download.yaml} | 0 .../2.18-delete.yaml} | 0 .../2.99-teardown-down.yaml} | 0 51 files changed, 106 insertions(+), 103 deletions(-) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-version.yaml => tier0/0.01-version.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-help-root.yaml => tier0/0.02-help-root.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-sample-list-text.yaml => tier0/0.03-sample-list-text.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-sample-list-json-filters.yaml => tier0/0.04-sample-list-json-filters.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-doctor-empty-dir.yaml => tier0/0.05-doctor-empty-dir.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-doctor-local-only.yaml => tier0/0.06-doctor-local-only.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-doctor-partial-failure.yaml => tier0/0.07-doctor-partial-failure.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-init-validate-mutually-exclusive.yaml => tier0/0.08-init-validate-mutually-exclusive.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-init-validate-no-prompt-missing.yaml => tier0/0.09-init-validate-no-prompt-missing.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-init-picker-navigation.yaml => tier0/0.10-init-picker-navigation.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-invoke-validate-protocol.yaml => tier0/0.11-invoke-validate-protocol.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-eval-context-required.yaml => tier0/0.12-eval-context-required.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-optimize-apply-requires-candidate.yaml => tier0/0.13-optimize-apply-requires-candidate.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-endpoint-show-help.yaml => tier0/0.14-endpoint-show-help.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-code-download-help.yaml => tier0/0.15-code-download-help.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{00-delete-help.yaml => tier0/0.16-delete-help.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-template-python.yaml => tier1/1.01-init-template-python.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-template-dotnet.yaml => tier1/1.02-init-template-dotnet.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-from-azure-yaml-url.yaml => tier1/1.03-init-from-azure-yaml-url.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-from-code.yaml => tier1/1.04-init-from-code.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-flags-agent-name-model.yaml => tier1/1.05-init-flags-agent-name-model.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-deploy-mode-code.yaml => tier1/1.06-init-deploy-mode-code.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-deploy-mode-container.yaml => tier1/1.07-init-deploy-mode-container.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{10-init-validate-deploy-mode.yaml => tier1/1.08-init-validate-deploy-mode.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{20-setup-deploy-shared-agent.yaml => tier2/2.00-setup-deploy-shared-agent.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{21-show.yaml => tier2/2.01-show.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{21-show-json.yaml => tier2/2.02-show-json.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{22-invoke-remote.yaml => tier2/2.03-invoke-remote.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{22-invoke-new-session.yaml => tier2/2.04-invoke-new-session.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{22-invoke-input-file.yaml => tier2/2.05-invoke-input-file.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{23-invoke-protocol-invocations.yaml => tier2/2.06-invoke-protocol-invocations.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{23-sessions-lifecycle.yaml => tier2/2.07-sessions-lifecycle.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{24-files-lifecycle.yaml => tier2/2.08-files-lifecycle.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{25-monitor-console.yaml => tier2/2.09-monitor-console.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{25-monitor-system.yaml => tier2/2.10-monitor-system.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{26-endpoint-update.yaml => tier2/2.11-endpoint-update.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{27-run-local-and-invoke-local.yaml => tier2/2.12-run-local-and-invoke-local.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{28-eval-lifecycle.yaml => tier2/2.13-eval-lifecycle.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{29-optimize-submit-and-cancel.yaml => tier2/2.14-optimize-submit-and-cancel.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{2A-doctor-provisioned-all-pass.yaml => tier2/2.15-doctor-provisioned-all-pass.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{2B-endpoint-show.yaml => tier2/2.16-endpoint-show.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{2C-code-download.yaml => tier2/2.17-code-download.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{2D-delete.yaml => tier2/2.18-delete.yaml} (100%) rename cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/{2Z-teardown-down.yaml => tier2/2.99-teardown-down.yaml} (100%) diff --git a/.github/skills/agent-scenario-tests/SKILL.md b/.github/skills/agent-scenario-tests/SKILL.md index 5f36a321f22..e19f2735a8c 100644 --- a/.github/skills/agent-scenario-tests/SKILL.md +++ b/.github/skills/agent-scenario-tests/SKILL.md @@ -70,4 +70,4 @@ confirmation**. - Every selected scenario was driven to completion with a recorded PASS/FAIL, duration, and any findings, and a `FINAL-REPORT.md` was written under `.reports//`. - A results comment was posted on the PR (unless the user opted out), and any Tier 2 run was - followed by `2Z-teardown-down` so no Azure resources are left running. + followed by `2.99-teardown-down` so no Azure resources are left running. diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/agent-scenario-tests/references/impact-mapping.md index bd5878a7a56..9053ed767c5 100644 --- a/.github/skills/agent-scenario-tests/references/impact-mapping.md +++ b/.github/skills/agent-scenario-tests/references/impact-mapping.md @@ -73,4 +73,4 @@ From the impacted `cmd:*` set, decide the **highest tier to offer**: Combine the derived `cmd:*` tags with the chosen tier tags and call `list_scenarios(tags=[...])`. Example: an `invoke.go` change approved for Tier 2 → `list_scenarios(tags=["cmd:invoke"])`, then keep the Tier 0/1 results plus the Tier 2 -`22-*` scenarios, prefixed by `20-setup` and suffixed by `2Z-teardown`. +`2.*-invoke-*` scenarios, prefixed by `2.00-setup` and suffixed by `2.99-teardown`. diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index 7569f143afa..a59a15e027e 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -10,8 +10,8 @@ the user exactly what to fix — do **not** try to work around it. Resolve its absolute path; everything else is relative to it. 2. Note the **WSL path** of that directory for MCP tool arguments. On Windows hosts the tester runs inside WSL, so a Windows path like - `C:\Repos\azure-dev\...\scenarios\00-version.yaml` must be passed as - `/mnt/c/Repos/azure-dev/.../scenarios/00-version.yaml`. On macOS/Linux use the native + `C:\Repos\azure-dev\...\scenarios\tier0\0.01-version.yaml` must be passed as + `/mnt/c/Repos/azure-dev/.../scenarios/tier0/0.01-version.yaml`. On macOS/Linux use the native absolute path. See `running-scenarios.md` § Path style. ### Tooling @@ -29,7 +29,7 @@ the user exactly what to fix — do **not** try to work around it. - **Tier 1 / Tier 2** read from / write to Azure. A human must `az login` inside WSL **before** the run (the agent cannot complete the browser sign-in). If the selected set includes Tier 1/2, remind the user to `az login` first. -- **Manifest scenarios** (`10-init-from-manifest-url`, `10-init-flags-agent-name-model`) +- **Manifest scenarios** (`1.03-init-from-azure-yaml-url`, `1.05-init-flags-agent-name-model`) download from GitHub and can fall back to the `gh` CLI; they need `gh auth login` inside WSL. Their `pre` hook fails fast if it isn't set up. diff --git a/.github/skills/agent-scenario-tests/references/reporting.md b/.github/skills/agent-scenario-tests/references/reporting.md index e5b9e24a84b..e0b68a79380 100644 --- a/.github/skills/agent-scenario-tests/references/reporting.md +++ b/.github/skills/agent-scenario-tests/references/reporting.md @@ -27,12 +27,12 @@ formatting). Keep it scannable — full detail lives in the artifact. Suggested | Scenario | Tier | Result | Duration | | --- | --- | --- | --- | -| 00-version | 0 | ✅ PASS | 4s | -| 22-invoke-remote | 2 | ✅ PASS | 1m 12s | -| 22-invoke-new-session | 2 | ❌ FAIL | 1m 40s | +| 0.01-version | 0 | ✅ PASS | 4s | +| 2.03-invoke-remote | 2 | ✅ PASS | 1m 12s | +| 2.04-invoke-new-session | 2 | ❌ FAIL | 1m 40s | **Findings** -- `22-invoke-new-session`: `--new-conversation` still recalled the prior name — memory +- `2.04-invoke-new-session`: `--new-conversation` still recalled the prior name — memory was not reset. (screenshot: …) **Coverage gaps:** this PR also touches `mcp.go`, which has no scenario — consider adding one. @@ -47,5 +47,5 @@ Rules: of the PR's change is a FAIL — report it and recommend fixing the code, not the scenario. - If the user opted out of posting (or there's no PR), write only the artifact and print the summary to the user instead. -- Mention any Tier 2 teardown status explicitly (e.g. "`2Z-teardown-down` ran — no resources +- Mention any Tier 2 teardown status explicitly (e.g. "`2.99-teardown-down` ran — no resources left provisioned") so the reader knows nothing is still costing money. diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index d7f7f4f6e7a..8a1bace40bd 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -11,7 +11,7 @@ path-shaped MCP argument on the WSL side. Pass POSIX paths: | Orchestrator OS | Pass to MCP tools | | --- | --- | -| Windows | `/mnt/c/Repos/azure-dev/.../scenarios/00-version.yaml` | +| Windows | `/mnt/c/Repos/azure-dev/.../scenarios/tier0/0.01-version.yaml` | | macOS / Linux | native absolute path | This applies to `path:` on `load_scenario` / `run_pre_hooks` / `run_post_hooks` and to @@ -28,8 +28,8 @@ For each selected scenario: 2. If it has `pre` hooks: `run_pre_hooks(path=…, session_vars=…)`. Hooks run host-side, sequentially, fail-fast (unless `continue_on_error: true`). 3. `start_session(scenario_path=…, session_vars=…, run_name=, output_dir=)`. - - `run_name` = the YAML filename without `.yaml` (e.g. `00-version`, `21-show-json`). - - For scenarios that start two sessions (`27-run-local-and-invoke-local`), suffix the + - `run_name` = the YAML filename without `.yaml` (e.g. `0.01-version`, `2.02-show-json`). + - For scenarios that start two sessions (`2.12-run-local-and-invoke-local`), suffix the `run_name` with a role tag (`…-run`, `…-invoke`). - `output_dir` = WSL path of `/.reports//tester-reports`. Reuse the **same** `` across every scenario in the run. @@ -70,15 +70,15 @@ For each selected scenario: - **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), one sub-agent per scenario, each with a distinct descriptive `session_id` **postfixed with a - timestamp** (e.g. `fleet-10-init-from-code-1752434100`). The timestamp (Unix epoch + timestamp** (e.g. `fleet-1.04-init-from-code-1752434100`). The timestamp (Unix epoch seconds) prevents collisions when multiple agent sessions drive the tester concurrently. No `instance_id` is needed — each scenario's `cwd` already isolates itself (defaults to the `-main` suffix). - **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's parallel-readiness section for which scenarios support it. -- **Tier 2** (`serial-only`): never parallelize. Run `20-setup-deploy-shared-agent` first, - then `21-…2D-` serially (they share one deployed agent and mutate shared session/file/ - endpoint state), then `2Z-teardown-down` last. +- **Tier 2** (`serial-only`): never parallelize. Run `2.00-setup-deploy-shared-agent` first, + then `2.01-`…`2.18-` serially (they share one deployed agent and mutate shared session/file/ + endpoint state), then `2.99-teardown-down` last. - **Validate the recipe with one scenario before fanning out** — confirm `load_scenario` → `start_session` → one `send_action` round-trips for a single Tier 0 scenario first. diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md index 9bf7493132d..758e153410e 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -76,8 +76,8 @@ Drive each selected scenario per `running-scenarios.md`. Honor ordering: - **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each with its own `cwd` (no `instance_id` needed for distinct scenarios). -- **Tier 2** is `serial-only` and order-dependent: `20-setup-deploy-shared-agent` **first**, - then the targeted `21-…2D-` scenarios **serially**, then `2Z-teardown-down` **last**. +- **Tier 2** is `serial-only` and order-dependent: `2.00-setup-deploy-shared-agent` **first**, + then the targeted `2.01-`…`2.18-` scenarios **serially**, then `2.99-teardown-down` **last**. Record per scenario: PASS/FAIL, wall-clock duration (`Hh Mm Ss`), and any `report_finding` entries. @@ -85,8 +85,8 @@ entries. ### Step 6 — Report Aggregate results into `.reports//FINAL-REPORT.md` and post a PR comment per -`reporting.md`. If a Tier 2 run started but was interrupted before `2Z-teardown`, run -`2Z-teardown-down` (or `20-setup`'s down hook) so no resources are orphaned, then report. +`reporting.md`. If a Tier 2 run started but was interrupted before `2.99-teardown`, run +`2.99-teardown-down` (or `2.00-setup`'s down hook) so no resources are orphaned, then report. ### Step 7 — Stop conditions diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 68e01cc5ac6..f43a1ed3310 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -85,7 +85,7 @@ Implications: isolation (the `{instance}` suffix keeps concurrent runs of the same scenario apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they - operate on the same deployed agent. `20-setup` runs `init` in that shared dir, + operate on the same deployed agent. `2.00-setup` runs `init` in that shared dir, which scaffolds the project into a subdirectory named after the agent, so the deployed project actually lives in `~/working/azd-agents-shared/{shared_agent_name}` (where `{shared_agent_name} = {prefix}-{shared_agent_suffix}` from your @@ -104,7 +104,7 @@ WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: | Orchestrator OS | Pass to MCP tools | Don't pass | | --- | --- | --- | -| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml` | `C:\Repos\azure-dev\...\00-version.yaml` | +| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.01-version.yaml` | `C:\Repos\azure-dev\...\tier0\0.01-version.yaml` | | macOS / Linux | native absolute path | — | **Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the @@ -163,14 +163,14 @@ This opens the interactive sign-in flow and then: 2. **Subscription selection** — back in the terminal, select the `{subscription}` subscription. -Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL +Tier 0 (`tier0/`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of them reuse that session credential. ### GitHub login (manifest scenarios) -The manifest scenarios (`10-init-from-manifest-url`, -`10-init-flags-agent-name-model`) download an agent manifest — and its sibling +The manifest scenarios (`1.03-init-from-azure-yaml-url`, +`1.05-init-flags-agent-name-model`) download an agent manifest — and its sibling files — from a public GitHub repo. The CLI first tries the anonymous GitHub API, but when that's rate-limited (60 req/hr) it falls back to the `gh` CLI, which would otherwise drop into an **interactive GitHub login** mid-run. Like @@ -202,15 +202,15 @@ advantage of both where it's safe. - **Tier 1 resource names** are suffixed with `-{instance}` too (via the RESOURCE NAMING goal and the `--agent-name` flag), so parallel instances don't collide on Azure resource names. - - **`27-run-local-and-invoke-local`** declares `allocate_ports: [agent]` and + - **`2.12-run-local-and-invoke-local`** declares `allocate_ports: [agent]` and binds `azd ai agent run`/`invoke --local` to `--port {agent}`. A port pool is shared across every `start_session` with the same `scenario_path`, so the `run` and `invoke` sessions find each other; parallel local runs each get a distinct port instead of colliding on the default `8088`. -- **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), - plus `20-setup` and `2Z-teardown`, all share the one deployed agent under +- **Single-instance by design:** the **Tier 2 reuse scenarios** (`2.01-`…`2.18-`), + plus `2.00-setup` and `2.99-teardown`, all share the one deployed agent under `~/working/azd-agents-shared` (the project itself lives in the - `{shared_agent_name}` subdirectory created by `20-setup`). They are + `{shared_agent_name}` subdirectory created by `2.00-setup`). They are **not** parameterized with `{instance}` (doing so would break the shared-agent assumption) and should be run serially. @@ -225,18 +225,18 @@ fan-out primitive for the shape of the run: - **Different scenarios in parallel** (the common case for a full Tier 0/1 sweep): give each sub-agent a distinct, descriptive `session_id` — e.g. - `fleet-00-version`, `fleet-10-init-from-code` — and call `start_session` with + `fleet-0.01-version`, `fleet-1.04-init-from-code` — and call `start_session` with the scenario's own `cwd`. **No `instance_id` is needed**: each scenario's `cwd` already isolates itself via the `{instance}` substitution, which defaults to `"main"` when `instance_id` is omitted. - **Same scenario N times in parallel:** use `instance_id="1"`, `"2"`, … per call. See [Parallel-readiness](#parallel-readiness--port-allocation) for which scenarios are authored to support this. -- **Tier 2 ordering is fixed**, not parallel-friendly. Run `20-setup` first, - then the targeted `21-…2A-` scenarios **serially** (they share one deployed +- **Tier 2 ordering is fixed**, not parallel-friendly. Run `2.00-setup` first, + then the targeted `2.01-`…`2.18-` scenarios **serially** (they share one deployed agent and mutate shared state — sessions, files, endpoint configuration — - so parallel runs interfere), then `2Z-teardown` last. See the - [Tier 2](#tier-2--cloud-end-to-end-prefix-2x---%EF%B8%8F-incurs-azure-cost) + so parallel runs interfere), then `2.99-teardown` last. See the + [Tier 2](#tier-2--cloud-end-to-end-tier2---%EF%B8%8F-incurs-azure-cost) section. ### Operational guardrails for the orchestrator @@ -299,13 +299,13 @@ its bugs: irreversible-ish; confirm with the user before entering an `init`/`provision` flow that creates real resources (especially when running in parallel). - **Pass `run_name=` to every `start_session` call.** The - scenario stem is the YAML filename without `.yaml` (e.g. `00-version`, - `21-show-json`, `27-run-local-and-invoke-local`). Without `run_name` the + scenario stem is the YAML filename without `.yaml` (e.g. `0.01-version`, + `2.02-show-json`, `2.12-run-local-and-invoke-local`). Without `run_name` the tester auto-names the run folder `agent_YYYYMMDD_HHMMSS`, which makes archived runs in `.reports//tester-reports/` hard to cross-reference with the scenario list. For scenarios that start two sessions - (e.g. `27-run-local-and-invoke-local`), suffix the run_name with a role tag - (`27-run-local-and-invoke-local-run`, `27-run-local-and-invoke-local-invoke`) + (e.g. `2.12-run-local-and-invoke-local`), suffix the run_name with a role tag + (`2.12-run-local-and-invoke-local-run`, `2.12-run-local-and-invoke-local-invoke`) so each session gets its own clearly named folder. - **Pass `output_dir` to every `start_session` call** so the tester writes screenshots and HTML reports directly into this repo's archive layout @@ -336,78 +336,81 @@ scenario also carries a `tags:` list that exposes the same axes plus the command(s) under test — see [Tags](#tags) for the full taxonomy and how to filter via `list_scenarios`. -### Tier 0 — Offline (prefix `00-`) +### Tier 0 — Offline (`tier0/`) No Azure auth, no network resource creation (except `sample list`, which fetches the public template catalog). Fast and mostly deterministic. Safe to run in any order, any time. | File | Targets | |------|---------| -| `00-version.yaml` | `version` | -| `00-help-root.yaml` | root help / command discovery | -| `00-sample-list-text.yaml` | `sample list` (text) | -| `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | -| `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | -| `00-doctor-local-only.yaml` | `doctor --local-only` | -| `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | -| `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | -| `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | -| `00-invoke-validate-protocol.yaml` | `invoke --protocol` unsupported-value error | -| `00-eval-context-required.yaml` | `eval list` outside a project requires a Foundry endpoint | -| `00-optimize-apply-requires-candidate.yaml` | `optimize apply` missing required `--candidate` | -| `00-doctor-partial-failure.yaml` | `doctor` mixed PASS+FAIL (exit 1) on a name-only `azure.yaml` | - -### Tier 1 — Auth, scaffold only (prefix `10-`) +| `tier0/0.01-version.yaml` | `version` | +| `tier0/0.02-help-root.yaml` | root help / command discovery | +| `tier0/0.03-sample-list-text.yaml` | `sample list` (text) | +| `tier0/0.04-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | +| `tier0/0.05-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | +| `tier0/0.06-doctor-local-only.yaml` | `doctor --local-only` | +| `tier0/0.07-doctor-partial-failure.yaml` | `doctor` mixed PASS+FAIL (exit 1) on a name-only `azure.yaml` | +| `tier0/0.08-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | +| `tier0/0.09-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | +| `tier0/0.10-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | +| `tier0/0.11-invoke-validate-protocol.yaml` | `invoke --protocol` unsupported-value error | +| `tier0/0.12-eval-context-required.yaml` | `eval list` outside a project requires a Foundry endpoint | +| `tier0/0.13-optimize-apply-requires-candidate.yaml` | `optimize apply` missing required `--candidate` | +| `tier0/0.14-endpoint-show-help.yaml` | `endpoint show --help` | +| `tier0/0.15-code-download-help.yaml` | `code download --help` | +| `tier0/0.16-delete-help.yaml` | `delete --help` | + +### Tier 1 — Auth, scaffold only (`tier1/`) Requires Azure login (reads subscriptions/Foundry projects) but **does not provision** any resources and incurs no cost. Each completes a project scaffold and verifies the generated files, then stops before `azd provision`. | File | Targets | |------|---------| -| `10-init-template-python.yaml` | `init` new-from-template, Python | -| `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | -| `10-init-from-manifest-url.yaml` | `init -m ` (needs `gh auth login`) | -| `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | -| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | -| `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | -| `10-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) — seeds from-code so the deploy-mode check is reached | -| `10-init-deploy-mode-container.yaml` | `init --deploy-mode container` (container build config) | - -### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost +| `tier1/1.01-init-template-python.yaml` | `init` new-from-template, Python | +| `tier1/1.02-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | +| `tier1/1.03-init-from-azure-yaml-url.yaml` | `init -m ` (needs `gh auth login`) | +| `tier1/1.04-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | +| `tier1/1.05-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | +| `tier1/1.06-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | +| `tier1/1.07-init-deploy-mode-container.yaml` | `init --deploy-mode container` (container build config) | +| `tier1/1.08-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) — seeds from-code so the deploy-mode check is reached | + +### Tier 2 — Cloud end-to-end (`tier2/`) — ⚠️ incurs Azure cost Provisions real resources. **Run order matters:** -1. `20-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. -2. Any `21-`…`2D-` targeted scenario (reuse the deployed agent). -3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. +1. `tier2/2.00-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. +2. Any `2.01-`…`2.18-` targeted scenario (reuse the deployed agent). +3. `tier2/2.99-teardown-down.yaml` **last** — `azd down --force --purge`. All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` -so they operate on the same deployed agent. `20-setup` runs `init` there, which +so they operate on the same deployed agent. `2.00-setup` runs `init` there, which scaffolds the project into the `{shared_agent_name}` subdirectory; the reuse and teardown scenarios run with `~/working/azd-agents-shared/{shared_agent_name}` as their `cwd`. | File | Targets | |------|---------| -| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` + `azd deploy` (SETUP) | -| `21-show.yaml` | `show` (table) | -| `21-show-json.yaml` | `show --output json` | -| `22-invoke-remote.yaml` | `invoke` (remote) | -| `22-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | -| `22-invoke-input-file.yaml` | `invoke -f ` | -| `23-invoke-protocol-invocations.yaml` | `invoke --protocol invocations` (session-bound memory; `--new-session` resets, `--new-conversation` no-op) | -| `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | -| `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | -| `25-monitor-console.yaml` | `monitor` (console) | -| `25-monitor-system.yaml` | `monitor --type system` | -| `26-endpoint-update.yaml` | `endpoint update` | -| `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | -| `28-eval-lifecycle.yaml` | `eval generate/list/show` against the shared agent (small sample budget, `--no-wait`) | -| `29-optimize-submit-and-cancel.yaml` | `optimize` submit + `list`/`status`/`cancel` (capped at 1 iteration, `--no-wait`) | -| `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | -| `2B-endpoint-show.yaml` | `endpoint show` (agent endpoint details) | -| `2C-code-download.yaml` | `code download` (positive-path: downloads agent source code) | -| `2D-delete.yaml` | `delete` (destroys the shared agent — run before teardown) | -| `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | +| `tier2/2.00-setup-deploy-shared-agent.yaml` | `init` + `azd provision` + `azd deploy` (SETUP) | +| `tier2/2.01-show.yaml` | `show` (table) | +| `tier2/2.02-show-json.yaml` | `show --output json` | +| `tier2/2.03-invoke-remote.yaml` | `invoke` (remote) | +| `tier2/2.04-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | +| `tier2/2.05-invoke-input-file.yaml` | `invoke -f ` | +| `tier2/2.06-invoke-protocol-invocations.yaml` | `invoke --protocol invocations` (session-bound memory; `--new-session` resets, `--new-conversation` no-op) | +| `tier2/2.07-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | +| `tier2/2.08-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | +| `tier2/2.09-monitor-console.yaml` | `monitor` (console) | +| `tier2/2.10-monitor-system.yaml` | `monitor --type system` | +| `tier2/2.11-endpoint-update.yaml` | `endpoint update` | +| `tier2/2.12-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | +| `tier2/2.13-eval-lifecycle.yaml` | `eval generate/list/show` against the shared agent (small sample budget, `--no-wait`) | +| `tier2/2.14-optimize-submit-and-cancel.yaml` | `optimize` submit + `list`/`status`/`cancel` (capped at 1 iteration, `--no-wait`) | +| `tier2/2.15-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | +| `tier2/2.16-endpoint-show.yaml` | `endpoint show` (agent endpoint details) | +| `tier2/2.17-code-download.yaml` | `code download` (positive-path: downloads agent source code) | +| `tier2/2.18-delete.yaml` | `delete` (destroys the shared agent — run before teardown) | +| `tier2/2.99-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | ## Tags @@ -422,7 +425,7 @@ grouping — colons are treated as ordinary characters by the filter): | Namespace | Values | Meaning | |---|---|---| | `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | -| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help`, `cmd:code`, `cmd:delete` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help`, `cmd:code`, `cmd:delete` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `2.12-run-local-and-invoke-local` runs both `run` and `invoke --local`; `2.00-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | | traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | **Examples** (the tool's `tags:` parameter is OR across the list): @@ -455,7 +458,7 @@ For each scenario returned by list_scenarios: load it, run any pre hooks, start the session and accomplish the goals (take screenshots at each step), finish the session, run any post hooks. The Tier 0/1 `init` scenarios are parallel-safe (also tagged `parallel-safe`); fan them out via fleet mode. -The Tier 2 `init` scenario (`20-setup-deploy-shared-agent`) is `serial-only` +The Tier 2 `init` scenario (`2.00-setup-deploy-shared-agent`) is `serial-only` — run it on its own and only if I confirm I want to spend on Azure resources. ``` @@ -568,21 +571,21 @@ How they're used here: re-runs start clean. (`start_session` recreates the dir, so removing it is enough; the doctor/init scenarios just need an empty dir.) - **`pre` fixture seed** — the existing-code scenarios - (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python + (`1.04-init-from-code`, `1.06-init-deploy-mode-code`) also copy a committed Python fixture into the dir so the source exists before the wizard's "Use the code in the current directory" flow inspects it (see [Fixtures](#fixtures)). -- **`pre` gh-auth guard** — the manifest scenarios (`10-init-from-manifest-url`, - `10-init-flags-agent-name-model`) run `gh auth status` and fail fast if GitHub +- **`pre` gh-auth guard** — the manifest scenarios (`1.03-init-from-azure-yaml-url`, + `1.05-init-flags-agent-name-model`) run `gh auth status` and fail fast if GitHub CLI isn't authenticated, because downloading the manifest can fall back to the `gh` CLI (and an interactive login) when the anonymous GitHub API is rate-limited. Run `gh auth login` first (see [Authentication](#authentication)). -- **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs +- **`pre` idempotent setup (Tier 2)** — `2.00-setup-deploy-shared-agent` first runs `azd down --force --purge` if a leftover project exists in the shared dir (so it never orphans live Azure resources), then clears the dir. The down hook uses `timeout: 900` and `continue_on_error: true`. -- **`pre` precondition guard (Tier 2 reuse)** — `21-…2A` print a clear "run - 20-setup first" warning if the shared agent isn't deployed (non-fatal). -- **`post` cleanup** — `2Z-teardown-down` clears the shared working dir after the +- **`pre` precondition guard (Tier 2 reuse)** — `2.01-`…`2.18` print a clear "run + 2.00-setup first" warning if the shared agent isn't deployed (non-fatal). +- **`post` cleanup** — `2.99-teardown-down` clears the shared working dir after the in-session `azd down` completes. ## Fixtures @@ -614,16 +617,16 @@ be run back to back in any order within a tier: - Tier 0/1 stateful scenarios **pre-wipe** their own `cwd`. Cleanup is pre-wipe **only** (no `post` delete), so the generated scaffold stays on disk for inspection after a run while the next run still starts clean. -- The shared Tier 2 dir is reset by `20-setup`'s `pre` hook, which **downs any +- The shared Tier 2 dir is reset by `2.00-setup`'s `pre` hook, which **downs any leftover deployed project first** to avoid orphaning live Azure resources (this also sidesteps the resource-name hash collision behind - [#8360](https://github.com/Azure/azure-dev/issues/8360)). `2Z-teardown-down` + [#8360](https://github.com/Azure/azure-dev/issues/8360)). `2.99-teardown-down` additionally clears the dir in a `post` hook. - Read-only scenarios (`version`, `--help`, `sample list`) run in `/tmp`, hold no state, and declare no hooks. -> If a Tier 2 run is interrupted before `2Z-teardown`, just re-run -> `20-setup-deploy-shared-agent` — its `pre` hook downs any live project in the +> If a Tier 2 run is interrupted before `2.99-teardown`, just re-run +> `2.00-setup-deploy-shared-agent` — its `pre` hook downs any live project in the > shared dir before redeploying, so resources won't be orphaned. ## Notes @@ -631,7 +634,7 @@ be run back to back in any order within a tier: - `files` and `sessions` are exercised as one lifecycle scenario per command group (rather than one file per subcommand) to avoid cross-scenario ordering dependencies — still one command at a time. -- `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` +- `azd ai agent run` blocks the terminal; `2.12-run-local-and-invoke-local.yaml` uses two sessions (one to run, one to invoke `--local`) that share an allocated `{agent}` port (see [Parallel-readiness](#parallel-readiness--port-allocation)). diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.01-version.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.01-version.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.02-help-root.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.02-help-root.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.03-sample-list-text.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.03-sample-list-text.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.04-sample-list-json-filters.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.04-sample-list-json-filters.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.05-doctor-empty-dir.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.05-doctor-empty-dir.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.06-doctor-local-only.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.06-doctor-local-only.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.07-doctor-partial-failure.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-partial-failure.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.07-doctor-partial-failure.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.08-init-validate-mutually-exclusive.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.08-init-validate-mutually-exclusive.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.09-init-validate-no-prompt-missing.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.09-init-validate-no-prompt-missing.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.10-init-picker-navigation.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.10-init-picker-navigation.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.11-invoke-validate-protocol.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-invoke-validate-protocol.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.11-invoke-validate-protocol.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.12-eval-context-required.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-eval-context-required.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.12-eval-context-required.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.13-optimize-apply-requires-candidate.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-optimize-apply-requires-candidate.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.13-optimize-apply-requires-candidate.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.14-endpoint-show-help.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.14-endpoint-show-help.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.15-code-download-help.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.15-code-download-help.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.16-delete-help.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.16-delete-help.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-azure-yaml-url.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.05-init-flags-agent-name-model.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.05-init-flags-agent-name-model.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-container.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.08-init-validate-deploy-mode.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-validate-deploy-mode.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.08-init-validate-deploy-mode.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.00-setup-deploy-shared-agent.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.00-setup-deploy-shared-agent.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-invoke-protocol-invocations.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-lifecycle.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-and-cancel.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.99-teardown-down.yaml similarity index 100% rename from cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml rename to cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.99-teardown-down.yaml From 34082ed0aa5ad60c8cf9d351511768c43874d3b4 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 30 Jul 2026 15:58:16 -0700 Subject: [PATCH 39/42] Add Tier 1b verify-deploy scenarios, requires field, and pin agent names - Add tier1b/ directory with 7 deploy scenarios (1b.01-1b.07) that verify Tier 1 init scaffolds actually deploy and produce a working agent - Each Tier 1b scenario uses requires: to depend on its Tier 1 prerequisite - Pin deterministic agent names in Tier 1 scenarios 1.01-1.04, 1.06-1.07 so Tier 1b can use predictable cwd paths - Add requires: to Tier 2 scenarios 2.01-2.18 (all require 2.00 setup) - Update skill files: running-scenarios.md (requires check + SKIPPED status), workflow.md (Tier 1b ordering), impact-mapping.md (verify-deploy tag), reporting.md (SKIPPED status), SKILL.md (Tier 1b in overview/exit criteria) - Update README: Tier 1b section, requires field docs, verify-deploy tag, updated fleet prompt and tag examples Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/skills/agent-scenario-tests/SKILL.md | 17 +++-- .../references/impact-mapping.md | 9 +++ .../references/reporting.md | 12 +++- .../references/running-scenarios.md | 21 +++++- .../references/workflow.md | 11 ++- .../README.md | 69 ++++++++++++++++--- .../tier1/1.01-init-template-python.yaml | 3 +- .../tier1/1.02-init-template-dotnet.yaml | 1 + .../tier1/1.03-init-from-azure-yaml-url.yaml | 1 + .../tier1/1.04-init-from-code.yaml | 1 + .../tier1/1.06-init-deploy-mode-code.yaml | 1 + .../1.07-init-deploy-mode-container.yaml | 1 + .../tier1b/1b.01-deploy-template-python.yaml | 43 ++++++++++++ .../tier1b/1b.02-deploy-template-dotnet.yaml | 41 +++++++++++ .../1b.03-deploy-from-azure-yaml-url.yaml | 41 +++++++++++ .../tier1b/1b.04-deploy-from-code.yaml | 41 +++++++++++ .../1b.05-deploy-flags-agent-name-model.yaml | 41 +++++++++++ .../tier1b/1b.06-deploy-deploy-mode-code.yaml | 42 +++++++++++ .../1b.07-deploy-deploy-mode-container.yaml | 42 +++++++++++ .../tier2/2.01-show.yaml | 1 + .../tier2/2.02-show-json.yaml | 1 + .../tier2/2.03-invoke-remote.yaml | 1 + .../tier2/2.04-invoke-new-session.yaml | 1 + .../tier2/2.05-invoke-input-file.yaml | 1 + .../2.06-invoke-protocol-invocations.yaml | 3 +- .../tier2/2.07-sessions-lifecycle.yaml | 1 + .../tier2/2.08-files-lifecycle.yaml | 1 + .../tier2/2.09-monitor-console.yaml | 1 + .../tier2/2.10-monitor-system.yaml | 1 + .../tier2/2.11-endpoint-update.yaml | 1 + .../2.12-run-local-and-invoke-local.yaml | 1 + .../tier2/2.13-eval-lifecycle.yaml | 3 +- .../2.14-optimize-submit-and-cancel.yaml | 3 +- .../2.15-doctor-provisioned-all-pass.yaml | 1 + .../tier2/2.16-endpoint-show.yaml | 1 + .../tier2/2.17-code-download.yaml | 1 + .../tier2/2.18-delete.yaml | 1 + 37 files changed, 435 insertions(+), 27 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.01-deploy-template-python.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.02-deploy-template-dotnet.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.03-deploy-from-azure-yaml-url.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.04-deploy-from-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.05-deploy-flags-agent-name-model.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.06-deploy-deploy-mode-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.07-deploy-deploy-mode-container.yaml diff --git a/.github/skills/agent-scenario-tests/SKILL.md b/.github/skills/agent-scenario-tests/SKILL.md index e19f2735a8c..c5bb65760e6 100644 --- a/.github/skills/agent-scenario-tests/SKILL.md +++ b/.github/skills/agent-scenario-tests/SKILL.md @@ -45,8 +45,9 @@ for regressions. It: 4. Posts a per-scenario results comment back on the PR. It is cost- and side-effect-aware: Tier 0 is free/offline, Tier 1 needs Azure auth but -provisions nothing, and **Tier 2 incurs Azure cost and is only run after explicit user -confirmation**. +provisions nothing, **Tier 1b** (`verify-deploy`) provisions per-scenario Azure resources to +verify Tier 1 scaffolds actually deploy, and **Tier 2** incurs Azure cost for cloud-feature +testing — both Tier 1b and Tier 2 are only run after explicit user confirmation. > This skill drives scenarios **deliberately, with user consent**. That is different from > the extension's `AGENTS.md` rule that coding agents must not invoke scenarios on their @@ -66,8 +67,10 @@ confirmation**. - The current branch's PR was resolved (or the user supplied one / chose to skip the comment). - The impacted scenario set was derived from the PR diff and **confirmed by the user** - (including an explicit cost acknowledgement before any Tier 2 run). -- Every selected scenario was driven to completion with a recorded PASS/FAIL, duration, and - any findings, and a `FINAL-REPORT.md` was written under `.reports//`. -- A results comment was posted on the PR (unless the user opted out), and any Tier 2 run was - followed by `2.99-teardown-down` so no Azure resources are left running. + (including an explicit cost acknowledgement before any Tier 1b or Tier 2 run). +- Every selected scenario was driven to completion with a recorded PASS/FAIL/SKIPPED, duration, + and any findings, and a `FINAL-REPORT.md` was written under `.reports//`. +- Scenarios with a `requires:` field whose prerequisite did not PASS are marked ⏭️ SKIPPED + (not FAIL) with a clear reason. +- A results comment was posted on the PR (unless the user opted out), and any Tier 1b/Tier 2 + run was followed by appropriate teardown so no Azure resources are left running. diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/agent-scenario-tests/references/impact-mapping.md index 9053ed767c5..5750791589d 100644 --- a/.github/skills/agent-scenario-tests/references/impact-mapping.md +++ b/.github/skills/agent-scenario-tests/references/impact-mapping.md @@ -58,6 +58,10 @@ these, broaden the impacted set (and ask the user how wide to go): From the impacted `cmd:*` set, decide the **highest tier to offer**: - Default to **Tier 0 + Tier 1** for any change to a covered command (free + auth-only). +- Offer **Tier 1b** (`verify-deploy`) when the change touches `cmd:init` code — these + scenarios verify that init scaffolds actually deploy. Since they reuse Tier 1 scaffolds, + any init change warrants running them. Tier 1b requires cost acknowledgement (provisions + real Azure resources) but is `parallel-safe`. - Offer **Tier 2** only when the change can plausibly affect cloud behavior — i.e. it touches `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:show`, `cmd:run`, `cmd:eval`, `cmd:optimize`, `cmd:doctor` provisioned paths, @@ -74,3 +78,8 @@ Combine the derived `cmd:*` tags with the chosen tier tags and call `list_scenarios(tags=[...])`. Example: an `invoke.go` change approved for Tier 2 → `list_scenarios(tags=["cmd:invoke"])`, then keep the Tier 0/1 results plus the Tier 2 `2.*-invoke-*` scenarios, prefixed by `2.00-setup` and suffixed by `2.99-teardown`. + +For **Tier 1b** (`verify-deploy`): when `cmd:init` is impacted and the user approves +cost-incurring tiers, include `list_scenarios(tags=["verify-deploy"])`. These scenarios +have `requires:` fields — the orchestrator will check prerequisites at runtime and SKIP +any whose Tier 1 prerequisite did not PASS. diff --git a/.github/skills/agent-scenario-tests/references/reporting.md b/.github/skills/agent-scenario-tests/references/reporting.md index e0b68a79380..1341c0689b4 100644 --- a/.github/skills/agent-scenario-tests/references/reporting.md +++ b/.github/skills/agent-scenario-tests/references/reporting.md @@ -23,15 +23,19 @@ formatting). Keep it scannable — full detail lives in the artifact. Suggested ## 🧪 Agent scenario regression check **Branch:** `` → `` · **Run:** `` -**Impacted tags:** `cmd:invoke`, `cmd:sessions` · **Tiers run:** 0, 1, 2 +**Impacted tags:** `cmd:init`, `cmd:invoke` · **Tiers run:** 0, 1, 1b, 2 | Scenario | Tier | Result | Duration | | --- | --- | --- | --- | | 0.01-version | 0 | ✅ PASS | 4s | +| 1.01-init-template-python | 1 | ✅ PASS | 22s | +| 1b.01-deploy-template-python | 1b | ✅ PASS | 3m 45s | +| 1b.02-deploy-template-dotnet | 1b | ⏭️ SKIPPED | — | | 2.03-invoke-remote | 2 | ✅ PASS | 1m 12s | | 2.04-invoke-new-session | 2 | ❌ FAIL | 1m 40s | **Findings** +- `1b.02-deploy-template-dotnet`: ⏭️ SKIPPED — prerequisite `tier1/1.02-init-template-dotnet.yaml` failed. - `2.04-invoke-new-session`: `--new-conversation` still recalled the prior name — memory was not reset. (screenshot: …) @@ -42,7 +46,11 @@ formatting). Keep it scannable — full detail lives in the artifact. Suggested Rules: -- Use ✅ PASS / ❌ FAIL (and ⚠️ for a scenario that completed but raised a non-fatal finding). +- Use ✅ PASS / ❌ FAIL / ⏭️ SKIPPED (and ⚠️ for a scenario that completed but raised a + non-fatal finding). +- **SKIPPED** means the scenario's `requires:` prerequisite did not PASS — include the reason + in the Findings section (e.g. "prerequisite `tier1/1.01-…` failed"). SKIPPED scenarios + have no duration (`—`). - **Never** soften a real regression to make the table green. A scenario that failed because of the PR's change is a FAIL — report it and recommend fixing the code, not the scenario. - If the user opted out of posting (or there's no PR), write only the artifact and print the diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index 8a1bace40bd..dff55ffcc54 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -23,6 +23,14 @@ before fanning out. For each selected scenario: +0. **Check `requires:`** — If `load_scenario` (or a prior YAML parse) reveals a `requires:` + field, look up the prerequisite scenario's result in the current run: + - Prerequisite **PASSED** → proceed normally (step 1+). + - Prerequisite **FAILED / not run / SKIPPED** → record this scenario as ⏭️ **SKIPPED** + with reason "prerequisite `` did not pass" and move to the next scenario. + The `requires:` value is a relative path from the scenarios root (e.g. + `tier1/1.01-init-template-python.yaml`). + 1. `load_scenario(path=, session_vars=)` — also tells you whether the scenario declares `pre`/`post` hooks. 2. If it has `pre` hooks: `run_pre_hooks(path=…, session_vars=…)`. Hooks run host-side, @@ -74,6 +82,12 @@ For each selected scenario: seconds) prevents collisions when multiple agent sessions drive the tester concurrently. No `instance_id` is needed — each scenario's `cwd` already isolates itself (defaults to the `-main` suffix). +- **Tier 1b** (`parallel-safe`, `verify-deploy`): runs **after all Tier 1 scenarios + complete**. Each Tier 1b scenario has a `requires:` field pointing to the Tier 1 scenario + whose scaffold it deploys. Only run a Tier 1b scenario if its prerequisite PASSED. Once + the prerequisites are confirmed, Tier 1b scenarios may be fanned out concurrently (each + has its own independent Azure environment). They also need the same cost acknowledgement + as Tier 2 since they provision real resources. - **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's parallel-readiness section for which scenarios support it. - **Tier 2** (`serial-only`): never parallelize. Run `2.00-setup-deploy-shared-agent` first, @@ -84,6 +98,7 @@ For each selected scenario: ## Capture per scenario -Record, for the report: the scenario stem, tier, PASS/FAIL, wall-clock **duration** -(`start_session` → `finish_session` incl. hooks, formatted `Hh Mm Ss`), and any -`report_finding` text (confusing UX, errors, doc mismatches). +Record, for the report: the scenario stem, tier, PASS/FAIL/SKIPPED, wall-clock **duration** +(`start_session` → `finish_session` incl. hooks, formatted `Hh Mm Ss`; `—` for SKIPPED), and +any `report_finding` text (confusing UX, errors, doc mismatches). SKIPPED scenarios include +the reason (e.g. "prerequisite `tier1/1.01-init-template-python.yaml` did not pass"). diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md index 758e153410e..9a659479681 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -63,9 +63,9 @@ and confirm via `ask_user` before running: - Always list the Tier 0 scenarios that will run (free). - If the set includes **Tier 1**, confirm `az login` is done. -- If the set includes **Tier 2**, require an **explicit cost acknowledgement** ("Tier 2 - provisions real Azure resources and incurs cost — proceed?"). If the user declines Tier 2, - drop it and run only Tier 0/1. +- If the set includes **Tier 1b** or **Tier 2**, require an **explicit cost acknowledgement** + ("Tier 1b/2 provisions real Azure resources and incurs cost — proceed?"). If the user + declines, drop cost-incurring tiers and run only Tier 0/1. Pick one `` of the form `YYYYMMDD-HHMMSS` for the whole run. All artifacts go under `/.reports//`. @@ -76,6 +76,11 @@ Drive each selected scenario per `running-scenarios.md`. Honor ordering: - **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each with its own `cwd` (no `instance_id` needed for distinct scenarios). +- **Tier 1b** (`verify-deploy`) is `parallel-safe` but **depends on Tier 1**: wait for all + Tier 1 scenarios to complete, then check each Tier 1b scenario's `requires:` field. Only + run it if the prerequisite PASSED; otherwise mark it ⏭️ SKIPPED. Once prerequisites are + confirmed, fan out Tier 1b scenarios concurrently. Tier 1b requires cost acknowledgement + (same as Tier 2) since it provisions Azure resources. - **Tier 2** is `serial-only` and order-dependent: `2.00-setup-deploy-shared-agent` **first**, then the targeted `2.01-`…`2.18-` scenarios **serially**, then `2.99-teardown-down` **last**. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index f43a1ed3310..d4a0683be2c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -50,10 +50,12 @@ run_post_hooks call — the scenarios reference {prefix}, {subscription}, {regio {tenant} (optional), and {shared_agent_name} placeholders. I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios -in tiers 0 and 1 are completely independent of each other and can be run in parallel. The scenarios -in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make -sure to take that into account when distributing the work. I want to run all of the tests regardless -of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. +in tiers 0 and 1 are completely independent of each other and can be run in parallel. Tier 1b +(verify-deploy) scenarios depend on their Tier 1 prerequisite passing first — run them after Tier 1 +completes, but they can be parallelized with each other. The scenarios in tier 2 however rely on a +setup scenario, and the teardown scenario should be run last, so make sure to take that into account +when distributing the work. I want to run all of the tests regardless of tier, and I acknowledge +that tier 1b and tier 2 have an azure cost implication, that's fine. After all of these scenarios are run, create a final result report. @@ -140,7 +142,7 @@ Requires the Go toolchain and sudo access in WSL. ## Authentication -Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in +Tier 1, Tier 1b, and Tier 2 scenarios read from / write to Azure, so a **human must log in manually before** starting a run. The scenarios do **not** perform login themselves, and the test-driving agent **cannot** complete it either: `az login` opens a **separate browser window** for account selection that requires @@ -238,6 +240,9 @@ fan-out primitive for the shape of the run: so parallel runs interfere), then `2.99-teardown` last. See the [Tier 2](#tier-2--cloud-end-to-end-tier2---%EF%B8%8F-incurs-azure-cost) section. +- **Tier 1b** (`verify-deploy`) is `parallel-safe` but must run **after Tier 1 + completes**. Check each scenario's `requires:` field — only proceed if the + prerequisite PASSED. Then fan out Tier 1b scenarios concurrently like Tier 0/1. ### Operational guardrails for the orchestrator @@ -331,7 +336,7 @@ its bugs: ## Tiers -Scenarios are organized into three tiers by cost and prerequisites. Each +Scenarios are organized into four tiers by cost and prerequisites. Each scenario also carries a `tags:` list that exposes the same axes plus the command(s) under test — see [Tags](#tags) for the full taxonomy and how to filter via `list_scenarios`. @@ -376,6 +381,27 @@ and verifies the generated files, then stops before `azd provision`. | `tier1/1.07-init-deploy-mode-container.yaml` | `init --deploy-mode container` (container build config) | | `tier1/1.08-init-validate-deploy-mode.yaml` | `init --deploy-mode` value validation (invalid value; code-mode required flags) — seeds from-code so the deploy-mode check is reached | +### Tier 1b — Deploy-verify (`tier1b/`) — ⚠️ incurs Azure cost +Verifies that Tier 1 scaffolds actually **deploy** and produce a working agent. +Each scenario reuses the on-disk scaffold from a Tier 1 init run (no init +duplication), provisions its own Azure resources, deploys, checks the agent is +accessible, then tears down with `azd down`. Independent Azure environments per +scenario — safe to parallelize once prerequisites pass. + +Each scenario declares a `requires:` field pointing to the Tier 1 scenario +whose scaffold it deploys. The orchestrator **must** check this: if the +prerequisite didn't PASS in the current run, the Tier 1b scenario is SKIPPED. + +| File | Verifies | Requires | +|------|----------|----------| +| `tier1b/1b.01-deploy-template-python.yaml` | Python scaffold deploys | `tier1/1.01-init-template-python.yaml` | +| `tier1b/1b.02-deploy-template-dotnet.yaml` | .NET scaffold deploys | `tier1/1.02-init-template-dotnet.yaml` | +| `tier1b/1b.03-deploy-from-azure-yaml-url.yaml` | URL-based scaffold deploys | `tier1/1.03-init-from-azure-yaml-url.yaml` | +| `tier1b/1b.04-deploy-from-code.yaml` | From-code scaffold deploys | `tier1/1.04-init-from-code.yaml` | +| `tier1b/1b.05-deploy-flags-agent-name-model.yaml` | Flags scaffold deploys | `tier1/1.05-init-flags-agent-name-model.yaml` | +| `tier1b/1b.06-deploy-deploy-mode-code.yaml` | Code-deploy scaffold deploys | `tier1/1.06-init-deploy-mode-code.yaml` | +| `tier1b/1b.07-deploy-deploy-mode-container.yaml` | Container-deploy scaffold deploys | `tier1/1.07-init-deploy-mode-container.yaml` | + ### Tier 2 — Cloud end-to-end (`tier2/`) — ⚠️ incurs Azure cost Provisions real resources. **Run order matters:** @@ -424,9 +450,9 @@ grouping — colons are treated as ordinary characters by the filter): | Namespace | Values | Meaning | |---|---|---| -| `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | +| `tier:N` | `tier:0`, `tier:1`, `tier:1b`, `tier:2` | The tier the scenario belongs to (same axis as the directory's four sections above). Use this to express cost / auth profile in one tag. | | `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:eval`, `cmd:optimize`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help`, `cmd:code`, `cmd:delete` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `2.12-run-local-and-invoke-local` runs both `run` and `invoke --local`; `2.00-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | -| traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | +| traits | `parallel-safe`, `serial-only`, `negative-path`, `picker`, `verify-deploy` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 / Tier 1b scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. `verify-deploy` flags Tier 1b scenarios that verify a Tier 1 scaffold deploys. | **Examples** (the tool's `tags:` parameter is OR across the list): @@ -434,6 +460,7 @@ grouping — colons are treated as ordinary characters by the filter): |---|---| | All `init` scenarios across every tier | `["cmd:init"]` | | Everything offline (no Azure auth, no cost) | `["tier:0"]` | +| All Tier 1b verify-deploy scenarios | `["verify-deploy"]` | | All Tier 2 cloud scenarios | `["tier:2"]` | | Invoke + sessions reuse scenarios | `["cmd:invoke", "cmd:sessions"]` | | CLI arg-validation scenarios only | `["negative-path"]` | @@ -495,6 +522,32 @@ regression to fix. > them — they intentionally carry no tags because they are configuration, > not scenarios. +## The `requires:` field + +A scenario may declare a `requires:` field with a path (relative to the scenarios +root) to another scenario that must PASS before this one can run: + +```yaml +requires: "tier1/1.01-init-template-python.yaml" +``` + +**Semantics:** + +- The orchestrator checks the prerequisite's result in the current run. +- If the prerequisite **PASSED** → proceed with this scenario normally. +- If the prerequisite **FAILED / was not run / was SKIPPED** → mark this + scenario as ⏭️ SKIPPED with reason "prerequisite `` did not pass". +- The `requires:` value is always a **relative path** from the scenarios root + (e.g. `tier1/1.01-init-template-python.yaml`, not an absolute path). +- The cli-interactive-tester MCP server ignores unknown top-level YAML keys, + so `requires:` is purely orchestrator-side logic — it doesn't affect + `load_scenario` or session behavior. + +**When to use:** Tier 1b verify-deploy scenarios use `requires:` to express +their dependency on the Tier 1 init scaffold they deploy. This ensures the +orchestrator doesn't waste time (and Azure cost) attempting to deploy a +scaffold that failed to initialize. + ## Profile / overrides Developer- and environment-specific values (subscription, region, model, diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml index 674d60e180a..393fb4ec141 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.01-init-template-python.yaml @@ -20,7 +20,8 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with '-{instance}' so concurrent instances don't collide. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-python-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml index 627738f8ff2..d9b6428da57 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.02-init-template-dotnet.yaml @@ -18,6 +18,7 @@ pre: goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-dotnet-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml index a07819750dd..8eac5e7d1d8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.03-init-from-azure-yaml-url.yaml @@ -29,6 +29,7 @@ pre: goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-from-url-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "Wait for the tool to fetch and parse the azure.yaml from the provided URL." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the '{subscription}' subscription." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml index 1b1290a1dae..a0c6b87c3c7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.04-init-from-code.yaml @@ -23,6 +23,7 @@ pre: goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-from-code-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml index 91987f4cbc4..045747eb5a8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.06-init-deploy-mode-code.yaml @@ -24,6 +24,7 @@ pre: goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-code-deploy-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml index f112788e53b..7286ed56649 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1/1.07-init-deploy-mode-container.yaml @@ -25,6 +25,7 @@ pre: goals: - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-container-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "When prompted for the AGENT NAME, set it to EXACTLY '{prefix}-t1-container-{instance}' (clear any pre-filled default first, then type it). This exact name is required so the scaffold subdirectory path is deterministic." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with container-deploy mode selected. Container mode may prompt for a startup command (the Dockerfile CMD, e.g. 'python main.py') — that is expected. However, it should NOT prompt for the separate entry-point or runtime prompts that code-deploy uses (those are distinct, code-deploy-only prompts)." - "If an existing agent manifest is detected, confirm reuse." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.01-deploy-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.01-deploy-template-python.yaml new file mode 100644 index 00000000000..da53c95dec6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.01-deploy-template-python.yaml @@ -0,0 +1,43 @@ +# Tier 1b (deploy verify) — provision + deploy the scaffold from tier1/1.01. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.01-init-template-python to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible, then +# cleans up with `azd down`. Each Tier 1 scaffold has its own independent azd +# environment (own Foundry project, own resource group), so `azd down` here +# only destroys THIS scenario's infra and cannot affect other parallel runs. +name: "verify-template-python" +command: "azd provision" +cwd: "~/working/azd-agents-t1-python-{instance}/{prefix}-t1-python-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.01-init-template-python.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-python-{instance}/{prefix}-t1-python-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.01-init-template-python first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 init scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.02-deploy-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.02-deploy-template-dotnet.yaml new file mode 100644 index 00000000000..e2b592ef236 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.02-deploy-template-dotnet.yaml @@ -0,0 +1,41 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.02. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.02-init-template-dotnet to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. +name: "deploy-template-dotnet" +command: "azd provision" +cwd: "~/working/azd-agents-t1-dotnet-{instance}/{prefix}-t1-dotnet-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.02-init-template-dotnet.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-dotnet-{instance}/{prefix}-t1-dotnet-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.02-init-template-dotnet first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 init scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.03-deploy-from-azure-yaml-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.03-deploy-from-azure-yaml-url.yaml new file mode 100644 index 00000000000..724c7f73fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.03-deploy-from-azure-yaml-url.yaml @@ -0,0 +1,41 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.03. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.03-init-from-azure-yaml-url to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. +name: "deploy-from-azure-yaml-url" +command: "azd provision" +cwd: "~/working/azd-agents-t1-from-url-{instance}/{prefix}-t1-from-url-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.03-init-from-azure-yaml-url.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-from-url-{instance}/{prefix}-t1-from-url-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.03-init-from-azure-yaml-url first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 init scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.04-deploy-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.04-deploy-from-code.yaml new file mode 100644 index 00000000000..9d6ba2889b8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.04-deploy-from-code.yaml @@ -0,0 +1,41 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.04. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.04-init-from-code to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. +name: "deploy-from-code" +command: "azd provision" +cwd: "~/working/azd-agents-t1-from-code-{instance}/{prefix}-t1-from-code-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.04-init-from-code.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-from-code-{instance}/{prefix}-t1-from-code-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.04-init-from-code first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 init scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.05-deploy-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.05-deploy-flags-agent-name-model.yaml new file mode 100644 index 00000000000..1045dfae86c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.05-deploy-flags-agent-name-model.yaml @@ -0,0 +1,41 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.05. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.05-init-flags-agent-name-model to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. +name: "deploy-flags-agent-name-model" +command: "azd provision" +cwd: "~/working/azd-agents-t1-flags-{instance}/{prefix}-qa-named-agent-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.05-init-flags-agent-name-model.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-flags-{instance}/{prefix}-qa-named-agent-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.05-init-flags-agent-name-model first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 init scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.06-deploy-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.06-deploy-deploy-mode-code.yaml new file mode 100644 index 00000000000..958897cae6a --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.06-deploy-deploy-mode-code.yaml @@ -0,0 +1,42 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.06. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.06-init-deploy-mode-code to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. This specifically validates the +# code-deploy (ZIP upload) path end-to-end. +name: "deploy-deploy-mode-code" +command: "azd provision" +cwd: "~/working/azd-agents-t1-code-deploy-{instance}/{prefix}-t1-code-deploy-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.06-init-deploy-mode-code.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-code-deploy-{instance}/{prefix}-t1-code-deploy-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.06-init-deploy-mode-code first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent via the code-deploy (ZIP upload) path to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the code-deploy agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 code-deploy scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.07-deploy-deploy-mode-container.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.07-deploy-deploy-mode-container.yaml new file mode 100644 index 00000000000..13418a19879 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier1b/1b.07-deploy-deploy-mode-container.yaml @@ -0,0 +1,42 @@ +# Tier 1b (verify-deploy) — provision + deploy the scaffold from tier1/1.07. +# +# ⚠️ Incurs Azure cost. Requires tier1/1.07-init-deploy-mode-container to have run +# first — it leaves a complete scaffold on disk that this scenario deploys. +# +# This scenario does NOT re-run init. It reuses the Tier 1 scaffold directly, +# provisions infrastructure, deploys the agent, verifies it's accessible and +# responsive, then cleans up with `azd down`. This specifically validates the +# container-deploy path end-to-end. +name: "deploy-deploy-mode-container" +command: "azd provision" +cwd: "~/working/azd-agents-t1-container-deploy-{instance}/{prefix}-t1-container-{instance}" +tags: ["tier:1b", "cmd:provision", "cmd:deploy", "verify-deploy", "parallel-safe"] +requires: "tier1/1.07-init-deploy-mode-container.yaml" + +pre: + - run: | + SCAFFOLD=~/working/azd-agents-t1-container-deploy-{instance}/{prefix}-t1-container-{instance} + if [ ! -f "$SCAFFOLD/azure.yaml" ]; then + echo "ERROR: Scaffold not found. Run tier1/1.07-init-deploy-mode-container first." + exit 1 + fi + if ! grep -q "azure.ai.agent" "$SCAFFOLD/azure.yaml"; then + echo "ERROR: Scaffold incomplete — azure.yaml missing agent service. Tier 1 likely failed." + exit 1 + fi + if [ ! -d "$SCAFFOLD/.azure" ]; then + echo "ERROR: No .azure/ environment. Tier 1 init did not save Azure selections." + exit 1 + fi + cwd: "~/working" + name: "assert tier 1 scaffold is complete" + +goals: + - "Wait for 'azd provision' to complete. This creates the Azure infrastructure (Foundry project, model deployment, resource group) defined by the Tier 1 scaffold." + - "After provision succeeds, run 'azd deploy' and wait for it to complete. This deploys the agent via the container-deploy path to the provisioned Foundry project." + - "Run 'azd ai agent show' and verify the agent is accessible — confirm it reports an agent name and endpoint URL." + - "Run 'azd ai agent invoke' and send a simple test message (e.g. 'Hello, are you there?'). Confirm the agent responds with a non-empty reply. This validates the container-deploy agent is functional, not just deployed." + - "Take a screenshot of the successful provision, deploy, show, and invoke output." + - "Run 'azd down --force --purge' to tear down all Azure resources created by this scenario." + - "Confirm the teardown succeeds (exit 0)." + - "Report a finding if provision, deploy, show, invoke, or teardown fails. A failure here means the Tier 1 container-deploy scaffold passed validation but cannot actually deploy or run — this is a real regression." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml index 7766378d8e7..993c187b73a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.01-show.yaml @@ -5,6 +5,7 @@ name: "show" command: "azd ai agent show" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:show", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml index 8b565759c65..1f87f909829 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.02-show-json.yaml @@ -5,6 +5,7 @@ name: "show-json" command: "azd ai agent show --output json" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:show", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml index 5ed85dd4c08..36fd7ca999a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.03-invoke-remote.yaml @@ -5,6 +5,7 @@ name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:invoke", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml index 8d38ef84524..4cccdc8ff0e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.04-invoke-new-session.yaml @@ -10,6 +10,7 @@ name: "invoke-session-vs-conversation" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:invoke", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml index cb4a19233fb..c715df52b23 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.05-invoke-input-file.yaml @@ -5,6 +5,7 @@ name: "invoke-input-file" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:invoke", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml index 9a6bfaee4f0..e625444ecf0 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.06-invoke-protocol-invocations.yaml @@ -12,7 +12,8 @@ name: "invoke-protocol-invocations" command: "azd ai agent invoke --protocol invocations \"Remember the word 'banana'.\"" cwd: "~/working/azd-agents-shared/{shared_agent_name}" -tags: ["tier:2", "cmd:invoke", "serial-only"] +tags: ["tier:2", "cmd:invoke", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml index 4c8068b1ff4..5dd95da5da3 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.07-sessions-lifecycle.yaml @@ -6,6 +6,7 @@ name: "sessions-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:sessions", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml index bed637575e2..c8da74e0acb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.08-files-lifecycle.yaml @@ -7,6 +7,7 @@ name: "files-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:files", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml index 4c7aabf36d1..4c9d598b6d9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.09-monitor-console.yaml @@ -6,6 +6,7 @@ name: "monitor-console" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:monitor", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml index 0aa08b4dd72..92b107b9857 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.10-monitor-system.yaml @@ -6,6 +6,7 @@ name: "monitor-system" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:monitor", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml index 4e631418aa1..56bd4aaa2a0 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.11-endpoint-update.yaml @@ -5,6 +5,7 @@ name: "endpoint-update" command: "azd ai agent endpoint update" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:endpoint", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml index 325fbe0c67d..8d1f2691778 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.12-run-local-and-invoke-local.yaml @@ -7,6 +7,7 @@ name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:run", "cmd:invoke", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml index ded03a75c77..579572a43bd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.13-eval-lifecycle.yaml @@ -16,7 +16,8 @@ name: "eval-lifecycle" command: "azd ai agent eval generate --max-samples 15 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" -tags: ["tier:2", "cmd:eval", "serial-only"] +tags: ["tier:2", "cmd:eval", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml index 46335b5bca2..f54d97e19bb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.14-optimize-submit-and-cancel.yaml @@ -17,7 +17,8 @@ name: "optimize-submit-and-cancel" command: "azd ai agent optimize --eval-model {model} --evaluator coherence --max-candidates 1 --no-wait" cwd: "~/working/azd-agents-shared/{shared_agent_name}" -tags: ["tier:2", "cmd:optimize", "serial-only"] +tags: ["tier:2", "cmd:optimize", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml index 2b8d289ba5f..f83b954ea4c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.15-doctor-provisioned-all-pass.yaml @@ -5,6 +5,7 @@ name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:doctor", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml index d5a4b88e24b..75c7c2b5cb4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.16-endpoint-show.yaml @@ -5,6 +5,7 @@ name: "endpoint-show" command: "azd ai agent endpoint show" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:endpoint", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" pre: - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml index 7ddb4b6684d..c8639179608 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.17-code-download.yaml @@ -9,6 +9,7 @@ name: "code-download" command: "azd ai agent code download --dest ~/working/azd-agents-shared/code-download-test" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:code", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" pre: - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml index 3bb1deaf2ed..f46c5ad33c1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier2/2.18-delete.yaml @@ -7,6 +7,7 @@ name: "delete" command: "azd ai agent delete --force" cwd: "~/working/azd-agents-shared/{shared_agent_name}" tags: ["tier:2", "cmd:delete", "serial-only"] +requires: "tier2/2.00-setup-deploy-shared-agent.yaml" pre: - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" From 359df4e66c9f610a3487aff1939adff59df2503b Mon Sep 17 00:00:00 2001 From: trangevi Date: Mon, 3 Aug 2026 10:10:20 -0700 Subject: [PATCH 40/42] Harden workflow steps and prohibit agent workarounds - Make Step 1b a mandatory gate with explicit verification - Windows/WSL: verify which azd returns /usr/local/bin/azd - Native Linux/macOS: verify azd version returns dev build - Hard gate: block all scenarios if verification fails - Add mandatory recipe validation in Step 5 (run one Tier 0 first) - Add Environment integrity section to running-scenarios.md - Add 'never work around infrastructure failures' driving convention - Add fleet sub-agent rules (no env modification, infra errors = FAIL) - Split prerequisites.md azd check into WSL and native Linux rows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../references/prerequisites.md | 3 +- .../references/running-scenarios.md | 34 ++++++++++ .../references/workflow.md | 64 ++++++++++++++----- 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/agent-scenario-tests/references/prerequisites.md index a59a15e027e..0b27d198945 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/agent-scenario-tests/references/prerequisites.md @@ -21,7 +21,8 @@ the user exactly what to fix — do **not** try to work around it. | `git` + `gh` CLIs | `gh auth status` | Ask the user to run `gh auth login`. | | cli-interactive-tester MCP server | The `list_scenarios` / `start_session` MCP tools are available to you | Stop. Tell the user to register the cli-interactive-tester MCP server (see its README) and re-run. | | `profile.local.yaml` | File exists in the scenarios dir | Stop. Tell the user to `cp profile.local.yaml.example profile.local.yaml` and set `prefix` + `subscription`. | -| Native Linux `azd` in WSL (Windows only) | `azd version` inside the tester returns a dev build, not a Windows `.exe` interop version | The skill automatically runs `setup-wsl.sh` (Step 1b) to rebuild from source. If you need to run it manually: `bash setup-wsl.sh` from the scenarios directory inside WSL. Symlinking to `azd.exe` does not work (causes git safe.directory, TTY, and file-locking errors). | +| Native Linux `azd` in WSL (Windows only) | Inside WSL, run `which azd` — must return `/usr/local/bin/azd` (not `/mnt/c/…` or a path ending in `azd.exe`). Then run `azd version` — must contain the expected dev version string. | **Hard stop.** If `which azd` returns a Windows interop path, the file-locking on UNC paths will fail all init/provision scenarios. Run Step 1b (`setup-wsl.sh`) to build and install native Linux azd. Do not proceed until both checks pass. | +| Native Linux `azd` (native Linux/macOS) | `azd version` returns the expected dev build | Ask the user to build and install `azd` from source. No special path check is needed — any valid `azd` path works on native Linux. | ### Auth (tier-dependent — only enforce for tiers actually selected) diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md index dff55ffcc54..5d5541549b2 100644 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ b/.github/skills/agent-scenario-tests/references/running-scenarios.md @@ -19,6 +19,20 @@ This applies to `path:` on `load_scenario` / `run_pre_hooks` / `run_post_hooks` the path style is almost certainly the cause — translate `C:\…` → `/mnt/c/…` and retry once before fanning out. +## Environment integrity + +The driving agent must **never install, replace, or modify** the `azd` binary, any `azd` +extension, or any system tool during a scenario run — regardless of operating system. The +environment was set up before scenarios start (Step 1b on Windows/WSL, manual build on native +Linux/macOS); if it is broken, the run stops — the agent does not fix it. + +- If a scenario fails due to an environment issue (wrong binary, missing tool, file-locking + error on WSL, path resolution failure, or similar), report it as **FAIL** with an + infrastructure finding. Do NOT attempt to work around it by installing packages, switching + binaries, downloading alternative builds, or modifying system state. +- This rule applies to both the orchestrating agent and any fleet sub-agents. No agent + participating in the run may alter the test environment. + ## Per-scenario loop For each selected scenario: @@ -68,6 +82,11 @@ For each selected scenario: that does not exist, or expect output that does not appear, fail the scenario. Do not substitute an alternative command, skip the broken step, or invent a workaround. The scenario must be updated by a human — the driving agent must not silently patch over it. +- **Never work around infrastructure failures.** If `azd` fails due to file-locking (common + on WSL when using the wrong binary), path resolution, missing tools, or similar environment + issues, that is a FAIL — not an invitation to install a different binary or modify the + environment. Report the infrastructure failure and move on. Workarounds mask setup gaps and + make results unreliable. This applies on all platforms (Windows/WSL, native Linux, macOS). - **Prefer `choice_text` over `choice_index`** (indices shift between releases). - **Clear a pre-filled text field before typing** (e.g. the agent-name prompt); otherwise your value *appends* to the default (`defaultyourvalue`). @@ -96,6 +115,21 @@ For each selected scenario: - **Validate the recipe with one scenario before fanning out** — confirm `load_scenario` → `start_session` → one `send_action` round-trips for a single Tier 0 scenario first. +### Fleet sub-agent rules + +When fanning out scenarios to sub-agents (e.g. Tier 0/1 parallel waves), every sub-agent +must obey these rules: + +- **Do not modify the environment.** Each sub-agent inherits the verified environment set up + by the orchestrator (Step 1b on Windows/WSL, or the user's build on native Linux/macOS). + Sub-agents must NOT install software, modify PATH, download binaries, or change the `azd` + binary or any extension — on any platform. +- **Infrastructure errors → FAIL and return.** If a sub-agent encounters an environment error + (wrong binary, file-locking, missing tool), it must FAIL the scenario with an infrastructure + finding and return the error to the orchestrator. It must NOT attempt to fix the environment. +- **Each sub-agent runs exactly one scenario.** It loads the scenario, drives the goals, and + reports PASS/FAIL. It does not make decisions about other scenarios or the overall run. + ## Capture per scenario Record, for the report: the scenario stem, tier, PASS/FAIL/SKIPPED, wall-clock **duration** diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/agent-scenario-tests/references/workflow.md index 9a659479681..e87fe08ccd7 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/agent-scenario-tests/references/workflow.md @@ -5,11 +5,17 @@ Run the checks in `prerequisites.md`. If a hard prerequisite is missing, stop with a clear message. Don't auto-install or work around a missing MCP server or profile. -### Step 1b — Rebuild WSL binaries (Windows only) +### Step 1b — Build and verify the `azd` binary (mandatory gate) -Before running any scenarios, rebuild the native Linux `azd` and extension binaries from the -current repo source so the tester always exercises the latest local code. Execute -`setup-wsl.sh` inside WSL via the tester: +Before running **any** scenarios, the orchestrator must ensure a working native Linux `azd` +dev build is available. The exact steps depend on the host OS: + +#### Windows (WSL) + +On Windows, scenarios run inside WSL where the default `azd` may resolve to Windows +`azd.exe` via interop — which causes file-locking failures on UNC paths (`\\wsl.localhost\…`). +**This step is mandatory and must not be skipped.** Execute `setup-wsl.sh` inside WSL via +the tester: ```text start_session(command="bash /mnt/c//setup-wsl.sh", @@ -21,6 +27,27 @@ start_session(command="bash /mnt/c//setup-wsl.sh", Wait for it to print "Done. WSL is ready for scenario testing." and then `finish_session`. If the build fails, stop and report the build error — do not proceed with stale binaries. +After `setup-wsl.sh` succeeds, **verify** the installation by starting a quick tester session +and running `which azd && azd version`. Confirm that: +1. `which azd` returns `/usr/local/bin/azd` (not `/mnt/c/…` or a path ending in `azd.exe`) +2. `azd version` output contains the expected dev version string (e.g. `0.0.0-dev.0`) + +Record the verified version string for the report. If either check fails, stop — do NOT +proceed to Step 5. + +#### Native Linux / macOS + +On native Linux or macOS, `setup-wsl.sh` does not apply. The user builds and installs `azd` +from source using their normal workflow (e.g. `go install`, `make`, or equivalent). Before +proceeding, verify that `azd version` returns the expected dev build version. If it does not, +stop and ask the user to build and install the correct version. + +#### Hard gate + +**If Step 1b is skipped or verification fails, do NOT proceed to Step 5.** No scenarios may +run until the `azd` binary is verified. This is not optional — running scenarios against +the wrong binary produces unreliable results and wastes time and cost. + ### Step 2 — Resolve the PR ```bash @@ -72,17 +99,24 @@ under `/.reports//`. ### Step 5 — Run the scenarios -Drive each selected scenario per `running-scenarios.md`. Honor ordering: - -- **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each - with its own `cwd` (no `instance_id` needed for distinct scenarios). -- **Tier 1b** (`verify-deploy`) is `parallel-safe` but **depends on Tier 1**: wait for all - Tier 1 scenarios to complete, then check each Tier 1b scenario's `requires:` field. Only - run it if the prerequisite PASSED; otherwise mark it ⏭️ SKIPPED. Once prerequisites are - confirmed, fan out Tier 1b scenarios concurrently. Tier 1b requires cost acknowledgement - (same as Tier 2) since it provisions Azure resources. -- **Tier 2** is `serial-only` and order-dependent: `2.00-setup-deploy-shared-agent` **first**, - then the targeted `2.01-`…`2.18-` scenarios **serially**, then `2.99-teardown-down` **last**. +Drive each selected scenario per `running-scenarios.md`. Start with a mandatory validation +step, then honor ordering: + +1. **Recipe validation (mandatory).** Run one Tier 0 scenario synchronously before fanning + out. Pick a fast, non-interactive scenario (e.g. `0.01-version`). If it fails with an + infrastructure error (file-locking, wrong binary, missing tool), **stop the entire run** — + do not fan out into a fleet of failures. Fix the environment issue (re-run Step 1b on + Windows, rebuild on native Linux) and start over. + +2. **Tier 0 / Tier 1** are `parallel-safe` — they may be run concurrently (small waves), each + with its own `cwd` (no `instance_id` needed for distinct scenarios). +3. **Tier 1b** (`verify-deploy`) is `parallel-safe` but **depends on Tier 1**: wait for all + Tier 1 scenarios to complete, then check each Tier 1b scenario's `requires:` field. Only + run it if the prerequisite PASSED; otherwise mark it ⏭️ SKIPPED. Once prerequisites are + confirmed, fan out Tier 1b scenarios concurrently. Tier 1b requires cost acknowledgement + (same as Tier 2) since it provisions Azure resources. +4. **Tier 2** is `serial-only` and order-dependent: `2.00-setup-deploy-shared-agent` **first**, + then the targeted `2.01-`…`2.18-` scenarios **serially**, then `2.99-teardown-down` **last**. Record per scenario: PASS/FAIL, wall-clock duration (`Hh Mm Ss`), and any `report_finding` entries. From 409774c475ad471a957502558ae266dd8872ad64 Mon Sep 17 00:00:00 2001 From: Travis Angevine Date: Tue, 4 Aug 2026 09:21:45 -0700 Subject: [PATCH 41/42] Restructure scenario driving into namespaced agents and skills (#9429) Split the overloaded scenario README and single agent-scenario-tests skill into an asymmetric agent/skill model so scenario runs execute deterministically: - Agents (.github/agents/): foundry-extension-scenario-orchestrator (dispatcher and cost/prereq gate), -worker (single-scenario driver), -author (authoring front door). All are deliberately human-selected (disable-model-invocation). - Skills (.github/skills/): foundry-extension-scenario-pr-regression (renamed from agent-scenario-tests; PR/diff-scoped), -suite-run (full/tag/tier sweep), -authoring (scenario authoring plus no-execution validation). - driving-mechanics.md is the single source of truth for execution mechanics; agents and skills link to it instead of restating the rules. It consolidates the former references/running-scenarios.md (nothing dropped). - README.md now carries only the human authoring contract; AGENTS.md routes coding agents to the new agents and skills. Every discoverable item uses the foundry-extension- namespace prefix to signal Foundry-extension ownership in this shared repository. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...foundry-extension-scenario-author.agent.md | 63 ++++ ...y-extension-scenario-orchestrator.agent.md | 112 +++++++ ...foundry-extension-scenario-worker.agent.md | 111 +++++++ .../references/running-scenarios.md | 138 --------- .../SKILL.md | 201 +++++++++++++ .../SKILL.md | 49 ++- .../references/impact-mapping.md | 0 .../references/prerequisites.md | 4 +- .../references/reporting.md | 2 +- .../references/workflow.md | 6 +- .../SKILL.md | 195 ++++++++++++ cli/azd/extensions/azure.ai.agents/AGENTS.md | 20 +- .../README.md | 280 +++++------------- .../driving-mechanics.md | 180 +++++++++++ 14 files changed, 990 insertions(+), 371 deletions(-) create mode 100644 .github/agents/foundry-extension-scenario-author.agent.md create mode 100644 .github/agents/foundry-extension-scenario-orchestrator.agent.md create mode 100644 .github/agents/foundry-extension-scenario-worker.agent.md delete mode 100644 .github/skills/agent-scenario-tests/references/running-scenarios.md create mode 100644 .github/skills/foundry-extension-scenario-authoring/SKILL.md rename .github/skills/{agent-scenario-tests => foundry-extension-scenario-pr-regression}/SKILL.md (52%) rename .github/skills/{agent-scenario-tests => foundry-extension-scenario-pr-regression}/references/impact-mapping.md (100%) rename .github/skills/{agent-scenario-tests => foundry-extension-scenario-pr-regression}/references/prerequisites.md (94%) rename .github/skills/{agent-scenario-tests => foundry-extension-scenario-pr-regression}/references/reporting.md (92%) rename .github/skills/{agent-scenario-tests => foundry-extension-scenario-pr-regression}/references/workflow.md (93%) create mode 100644 .github/skills/foundry-extension-scenario-suite-run/SKILL.md create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md diff --git a/.github/agents/foundry-extension-scenario-author.agent.md b/.github/agents/foundry-extension-scenario-author.agent.md new file mode 100644 index 00000000000..6c90a8ce12f --- /dev/null +++ b/.github/agents/foundry-extension-scenario-author.agent.md @@ -0,0 +1,63 @@ +--- +name: foundry-extension-scenario-author +description: >- + Front door for AUTHORING the azure.ai.agents cli-interactive-tester scenarios. Writes a new + goal-based scenario YAML (or fixes an existing one) so it follows the framework's tier / tag / + hook / fixtures / requires conventions and the goals-are-the-contract judging rules, then + lint-validates it statically. Generative and repo-writing, but it never RUNS a scenario and + never provisions Azure resources — running is the foundry-extension-scenario-orchestrator / foundry-extension-scenario-suite-run / + foundry-extension-scenario-worker path. Deliberately human-selected (never auto-run). +# Deliberate selection only — the model must not auto-start authoring. A human picks this agent; +# the foundry-extension-scenario-authoring skill provides the model-triggered entry when appropriate. +disable-model-invocation: true +--- + +# Scenario Author + +You are the **front door for authoring** the `azure.ai.agents` cli-interactive-tester scenarios. +You help a test author create a new scenario (or bring an existing one up to standard) that will +run correctly under the framework's mechanics, and you **validate it without running it**. + +You are generative and you **write repo files** (scenario YAML under +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`, and fixtures under +its `fixtures/` tree). You do **not** run scenarios, drive the tester, or provision anything — +authoring never incurs Azure cost. + +## How you work — load the skill + +Authoring is a defined procedure. Load and follow the **`foundry-extension-scenario-authoring`** skill +(`.github/skills/foundry-extension-scenario-authoring/SKILL.md`); it is the single source for: + +- the **authoring contract** (goals are a literal, checkable spec — single-sourced in the + scenarios README's "How scenarios are judged" section), +- the **scenario anatomy** (annotated YAML skeleton) and the field references (tiers, tags, + `requires:`, profile placeholders, hooks, fixtures, idempotency, conventions), +- the **authoring procedure** (pick tier / placement, write `command` + `cwd`, add idempotency + hooks, seed fixtures, set `tags` / `requires`, write `goals`), +- the **static validation loop** (tag lint via `list_scenarios`, YAML shape, `requires:` / + fixture / placeholder resolution) — **no execution**. + +Don't restate the taxonomy from memory — read the skill and the sections it links to so the +scenario matches the current conventions. + +## Hard boundaries + +- **Never run a scenario.** You do not call `start_session` / `send_action` / `finish_session`, + and you do not run `azd provision` / `deploy` / `down`. The only tester tool authoring uses is + `list_scenarios`, purely to lint tags and confirm the file parses. +- **Never provision or incur Azure cost.** If the author wants to *confirm* a new scenario drives + cleanly, that is a separate, cost-gated run — hand off to the **`foundry-extension-scenario-orchestrator`** agent + (or the `foundry-extension-scenario-suite-run` skill). Say so explicitly; don't try to run it yourself. +- **Keep scope to authoring.** One scenario targets one command / flow. Editing product code, + running the suite, or reviewing a PR are other agents' jobs. + +## Exit criteria + +- A single-command scenario YAML was authored (or corrected) in the correct `tierN/` directory, + following the `foundry-extension-scenario-authoring` skill: compliant `tags:`, appropriate `cwd` + idempotency + hooks, any needed fixture / `requires:`, and `goals:` written as a literal, checkable contract. +- The skill's static validation loop passed (`list_scenarios` lists the file under all its tags — + not `tags: []`, the YAML parses, and every `requires:` / fixture / placeholder reference + resolves). +- **No scenario was executed** and no Azure resources were created; any run-to-confirm was handed + off to the run path with a cost note. diff --git a/.github/agents/foundry-extension-scenario-orchestrator.agent.md b/.github/agents/foundry-extension-scenario-orchestrator.agent.md new file mode 100644 index 00000000000..5505db266f6 --- /dev/null +++ b/.github/agents/foundry-extension-scenario-orchestrator.agent.md @@ -0,0 +1,112 @@ +--- +name: foundry-extension-scenario-orchestrator +description: >- + Front door for RUNNING the azure.ai.agents cli-interactive-tester scenarios. Coordinates a + whole run — verifies prerequisites and the azd binary, gates Azure cost, validates the recipe, + fans scenarios out to foundry-extension-scenario-worker agents in the right order, and hands off reporting. Routes + the selection strategy to a run skill: foundry-extension-scenario-pr-regression for a PR/diff regression check, or + foundry-extension-scenario-suite-run for a full or tag/tier sweep. Deliberately human-selected (never auto-run), + honoring the extension AGENTS.md rule that scenarios are never run automatically. +# Deliberate selection only — the model must not auto-start a scenario run. A human picks this +# agent (matching agentic-workflows.agent.md and the extension AGENTS.md "never run +# automatically" rule). +disable-model-invocation: true +--- + +# Scenario Orchestrator + +You are the **front door for running** the `azure.ai.agents` cli-interactive-tester scenarios. +You own the parts of a run that are the same no matter *which* scenarios run — prerequisites, +the `azd` binary gate, cost consent, recipe validation, ordering, and reporting — and you +**delegate** the two things that differ: *which* scenarios to run (a run skill) and *driving +each* scenario (the `foundry-extension-scenario-worker` agent). + +You do **not** drive scenarios one action at a time yourself, and you do **not** edit scenario +YAML or product code. Your writes are limited to run artifacts under `.reports/` and (for PR +runs) a PR comment. + +## Route the request + +Identify the intent and load the matching **skill** for the selection strategy, then run the +shared flow below: + +- **PR / diff regression** ("test my change", "run the impacted scenarios for this PR", "check + the agents extension for regressions before merge") → load the **`foundry-extension-scenario-pr-regression`** + skill. It maps the PR diff to impacted tags and owns the PR comment. +- **Full or filtered sweep** ("run all scenarios", "run every `init` scenario", "run all of + Tier 2", "nightly sweep", "run the `parallel-safe` set") → load the **`foundry-extension-scenario-suite-run`** + skill. It discovers scenarios by tag/tier via `list_scenarios`. +- **Authoring / editing a scenario** ("write a new scenario", "add coverage for `cmd:foo`") → + this is **not** a run. Hand off to the **`foundry-extension-scenario-author`** agent (or the + `foundry-extension-scenario-authoring` skill). Do not start a run to author. + +If the intent is ambiguous (e.g. "test init"), ask whether they mean a PR-scoped regression or +a broad sweep before proceeding. + +## Shared run flow (you own these gates) + +Run these in order regardless of which skill selected the scenarios. Each references a single +source — read it, don't restate it. + +1. **Prerequisites.** Verify MCP server availability, `profile.local.yaml`, and derive the + merged `session_vars` (profile merge + `shared_agent_name` + `fixtures_dir`) per + `.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md`. Thread `session_vars` + unchanged through every worker. +2. **`azd` binary gate (mandatory).** Ensure a verified native-Linux `azd` dev build is + installed before any scenario runs, per + `.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md` § Step 1b (Windows/WSL: + `setup-wsl.sh` then confirm `which azd` = `/usr/local/bin/azd` and `azd version` shows the + dev string; native Linux/macOS: confirm the user's dev build). **If verification fails, + stop** — do not run scenarios against the wrong binary. +3. **Cost / consent gate.** List the plan grouped by tier. Tier 0 is free; Tier 1 needs + `az login`; **Tier 1b and Tier 2 provision real Azure resources** and require an *explicit* + cost acknowledgement before running. If the user declines, drop the cost-incurring tiers. +4. **Recipe validation (mandatory).** Before fanning out, drive one fast Tier 0 scenario + (e.g. `0.01-version`) end-to-end — spawn a single `foundry-extension-scenario-worker` and wait. If it fails + with an infrastructure error, **stop the whole run** and fix the environment (re-run the + binary gate); do not fan out into a fleet of failures. + +## Execution — fan out to workers + +The rules for *how* a scenario is driven live once in the executor spec +**`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`**. +You don't drive scenarios yourself; you spawn one **`foundry-extension-scenario-worker`** per scenario (via the +`agent` tool) and honor the ordering and parallelism in that spec: + +- **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), each worker + with a distinct `session_id` suffixed with a Unix-epoch timestamp. No `instance_id` for + distinct scenarios. +- **`requires:` gating is yours.** You hold the run's results, so before dispatching any + scenario with a `requires:` field, look up the prerequisite's verdict **in this run** and + tell the worker whether it passed. If it did not pass (or wasn't run), mark the scenario + ⏭️ SKIPPED and don't spawn a worker. +- **Tier 1b** (`verify-deploy`, ⚠️ cost): only after all Tier 1 workers finish and only for + scenarios whose `requires:` prerequisite PASSED; then fan out concurrently. +- **Tier 2** (`serial-only`, ⚠️ cost): never parallelize — `2.00-setup` first, then + `2.01…2.18` serially, `2.18-delete` before teardown, `2.99-teardown-down` last. Launch + cost-incurring workers conservatively (background workers are typically not cancellable + mid-run; a stop can't recall an in-flight `azd provision`). + +Give each worker its inputs (scenario path in the correct style, `session_vars`, `run_name`, +`output_dir` under a single ``, `session_id`, `instance_id` if fanning the same +scenario out N times, and its prerequisite status). Collect each worker's returned verdict +block. + +## Reporting handoff + +Aggregate every worker's verdict into `.reports//FINAL-REPORT.md` and, for a PR +run, post the PR comment — per +`.github/skills/foundry-extension-scenario-pr-regression/references/reporting.md`. Never soften a real regression +to make the table green. If a Tier 2 run started but was interrupted before `2.99-teardown`, +run teardown (or `2.00-setup`'s down hook) so no Azure resources are left provisioned, then +report that status explicitly. + +## Exit criteria + +- The request was routed to the correct run skill (or handed off to authoring). +- Prerequisites, the `azd` binary gate, cost consent, and recipe validation all passed before + any fan-out; cost-incurring tiers ran only with explicit acknowledgement. +- Every selected scenario has a recorded PASS / FAIL / ⏭️ SKIPPED (with duration and findings), + `requires:`-gated scenarios that didn't qualify are SKIPPED (not FAIL), a `FINAL-REPORT.md` + was written, any PR comment was posted (unless opted out), and any Azure resources were torn + down. diff --git a/.github/agents/foundry-extension-scenario-worker.agent.md b/.github/agents/foundry-extension-scenario-worker.agent.md new file mode 100644 index 00000000000..cee13e12791 --- /dev/null +++ b/.github/agents/foundry-extension-scenario-worker.agent.md @@ -0,0 +1,111 @@ +--- +name: foundry-extension-scenario-worker +description: >- + Drives exactly ONE azure.ai.agents cli-interactive-tester scenario to a PASS / FAIL / + SKIPPED verdict and returns a structured report. Spawned by the foundry-extension-scenario-orchestrator agent + or a scenario run skill (foundry-extension-scenario-pr-regression / foundry-extension-scenario-suite-run) — one worker per + scenario, in parallel waves. Restricted to the cli-interactive-tester MCP tools: it cannot + edit files, run host shell commands, install or modify anything, or spawn other agents, which + keeps it fail-loud and unable to work around a broken environment. +# Spawned only: the model must not auto-select this agent. Invoke it explicitly via the +# `agent` tool from the orchestrator / run skill. +disable-model-invocation: true +tools: + # The cli-interactive-tester MCP server (drives the CLI through tmux/WSL). The server is + # registered per-user (see the scenarios README), and its tools appear as + # `cli-interactive-tester-*`, so the server name is `cli-interactive-tester`. If a checkout + # registers it under a different name, update this line — unrecognized tool names are + # silently ignored, which would leave this worker with no driving tools. + - cli-interactive-tester/* + - read # read the driving-mechanics spec (below); no other file access is needed + - todo # track the per-scenario steps + # Deliberately NOT granted: edit (no repo writes — never patch a scenario to make it pass), + # execute/shell (no host commands — never modify the environment), agent (a worker never + # spawns sub-agents), web. +--- + +# Scenario Worker + +You drive **exactly one** cli-interactive-tester scenario for the `azure.ai.agents` extension, +decide a single verdict, and return a compact report to your caller. You are **spawned** by the +`foundry-extension-scenario-orchestrator` agent or by a run skill — you never choose scenarios, plan a suite, or +decide anything about the overall run. + +## Authoritative spec — follow it exactly + +The one source of truth for *how* to drive a scenario is the executor spec: + +**`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`** + +`read` it at the start of your run and follow it exactly: path style (Windows → WSL), +environment integrity, the per-scenario loop, the fail-loud execution rules, and capture. Do +**not** restate, reinterpret, or override it here. If anything in this prompt seems to conflict +with the spec, the spec wins. + +## Inputs (provided in your spawn prompt) + +Your caller gives you everything you need — do not go looking for it yourself: + +- **`scenario_path`** — the scenario YAML, already in the correct path style (WSL `/mnt/c/…` + on Windows, native absolute path otherwise). +- **`session_vars`** — the merged profile map (`prefix`, `subscription`, `region`, `model`, + optional `tenant`, `shared_agent_name`, `fixtures_dir`). Pass it **unchanged** on every + `load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. +- **`run_name`** — the scenario stem (e.g. `1.04-init-from-code`); role-suffixed for + two-session scenarios. +- **`output_dir`** — the WSL/native path of `.reports//tester-reports`. +- **`session_id`** — a unique id (already timestamp-suffixed for fleet safety), plus an + optional **`instance_id`** when the caller is fanning the *same* scenario out N times. +- **Prerequisite status** — if the scenario declares `requires:`, the caller tells you whether + that prerequisite **PASSED** in the current run. Requires-gating is a run-level decision the + caller owns; you only act on what you are told (see below). + +## Procedure + +1. If the caller told you the scenario's `requires:` prerequisite **did not pass**, return + immediately with verdict **⏭️ SKIPPED** and reason `prerequisite did not pass`. Do + not start a session. +2. Otherwise drive the scenario through the tester following the per-scenario loop in the spec: + `load_scenario` → (if present) `run_pre_hooks` → `start_session` (with `run_name`, + `output_dir`, `session_id`, and `instance_id` if given) → drive the `goals:` with + `send_action` / `select` / screenshots → `finish_session` → (if present) `run_post_hooks`. + Always `finish_session` for every session you start. Screenshot key steps and + `report_finding` for any confusing UX, error, or doc mismatch. + +## Verdict rules (fail-loud — do not soften) + +Apply the spec's execution rules; the essentials: + +- **The goals are the contract.** PASS **only** when the product's actual behavior matches the + goals. A different-but-reasonable error, or a referenced flag/subcommand that no longer + exists, is a **FAIL** — not a PASS-with-observation. +- **Never adapt around broken goals.** If a goal says to run a command/flag that doesn't exist + or expects output that never appears, **FAIL** — do not substitute, skip, or invent a + workaround. A human must fix the scenario. +- **A `select` miss is a hard failure.** Report it and stop the scenario — do not retry with a + different `choice_text`/`choice_index`, and do not verify/"correct" a pick after sending it. +- **Never retry a failed scenario** unless its `goals:` explicitly say to. +- **Never work around a broken environment.** Wrong binary, file-locking, missing tool, path + failure → **FAIL** with an infrastructure finding and return. You have no `edit`/`shell` + tools by design: do not attempt to install, replace, or modify anything. + +## What you return + +Return a single compact block your caller can drop straight into the aggregate report — no +prose preamble: + +```text +scenario: e.g. 1.04-init-from-code +tier: <0 | 1 | 1b | 2> +verdict: <✅ PASS | ❌ FAIL | ⏭️ SKIPPED | ⚠️ PASS-with-finding> +duration: (— for SKIPPED; start_session → finish_session incl. hooks) +findings: +report_dir: // (tester HTML + screenshots) +``` + +## Exit criteria + +- Exactly one scenario was driven to a single verdict (or SKIPPED before starting), every + session you started was `finish_session`- d, and the structured block above was returned. +- You made **no** decisions about other scenarios or the overall run, and you did **not** + modify the environment, edit any file, or run any host command. diff --git a/.github/skills/agent-scenario-tests/references/running-scenarios.md b/.github/skills/agent-scenario-tests/references/running-scenarios.md deleted file mode 100644 index 5d5541549b2..00000000000 --- a/.github/skills/agent-scenario-tests/references/running-scenarios.md +++ /dev/null @@ -1,138 +0,0 @@ - -# Running scenarios through the tester - -This mirrors the scenarios `README.md` ("Driving conventions" and "Orchestrating a fleet -run"). Follow it so the run actually *tests* the CLI instead of papering over its bugs. - -## Path style (Windows → WSL) - -On Windows the tester drives CLIs through tmux **inside WSL**, and it resolves every -path-shaped MCP argument on the WSL side. Pass POSIX paths: - -| Orchestrator OS | Pass to MCP tools | -| --- | --- | -| Windows | `/mnt/c/Repos/azure-dev/.../scenarios/tier0/0.01-version.yaml` | -| macOS / Linux | native absolute path | - -This applies to `path:` on `load_scenario` / `run_pre_hooks` / `run_post_hooks` and to -`scenario_path:` on `start_session`. If `load_scenario` returns `Scenario file not found`, -the path style is almost certainly the cause — translate `C:\…` → `/mnt/c/…` and retry once -before fanning out. - -## Environment integrity - -The driving agent must **never install, replace, or modify** the `azd` binary, any `azd` -extension, or any system tool during a scenario run — regardless of operating system. The -environment was set up before scenarios start (Step 1b on Windows/WSL, manual build on native -Linux/macOS); if it is broken, the run stops — the agent does not fix it. - -- If a scenario fails due to an environment issue (wrong binary, missing tool, file-locking - error on WSL, path resolution failure, or similar), report it as **FAIL** with an - infrastructure finding. Do NOT attempt to work around it by installing packages, switching - binaries, downloading alternative builds, or modifying system state. -- This rule applies to both the orchestrating agent and any fleet sub-agents. No agent - participating in the run may alter the test environment. - -## Per-scenario loop - -For each selected scenario: - -0. **Check `requires:`** — If `load_scenario` (or a prior YAML parse) reveals a `requires:` - field, look up the prerequisite scenario's result in the current run: - - Prerequisite **PASSED** → proceed normally (step 1+). - - Prerequisite **FAILED / not run / SKIPPED** → record this scenario as ⏭️ **SKIPPED** - with reason "prerequisite `` did not pass" and move to the next scenario. - The `requires:` value is a relative path from the scenarios root (e.g. - `tier1/1.01-init-template-python.yaml`). - -1. `load_scenario(path=, session_vars=)` — also tells you whether - the scenario declares `pre`/`post` hooks. -2. If it has `pre` hooks: `run_pre_hooks(path=…, session_vars=…)`. Hooks run host-side, - sequentially, fail-fast (unless `continue_on_error: true`). -3. `start_session(scenario_path=…, session_vars=…, run_name=, output_dir=)`. - - `run_name` = the YAML filename without `.yaml` (e.g. `0.01-version`, `2.02-show-json`). - - For scenarios that start two sessions (`2.12-run-local-and-invoke-local`), suffix the - `run_name` with a role tag (`…-run`, `…-invoke`). - - `output_dir` = WSL path of `/.reports//tester-reports`. - Reuse the **same** `` across every scenario in the run. -4. Drive the session's `goals:` with `send_action` / `select_by_text` / screenshots, then - `finish_session`. -5. If it has `post` hooks: `run_post_hooks(path=…, session_vars=…)`. - -## Driving conventions (fail-loud) - -- **The scenario goals are the contract.** A scenario PASSES only when the product's actual - behavior matches what the goals describe. If the goals say "expect error X" and the - product prints a different error (even a reasonable one), that is a FAIL. If the goals - reference a flag or subcommand that no longer exists, that is a FAIL. The driving agent's - job is to **verify** goals were met, not to **rationalize** why they weren't. Do not mark - a scenario as PASSED with an "observation" when the goals were not achieved — observations - are for incidental notes on scenarios that genuinely passed all their goals. -- **Don't verify/retry after a `select`.** Reading back the echo and "correcting" a pick - hides the very picker defect the test exists to catch. Send the action and let downstream - prompts surface any failure. -- **Treat a select miss as a hard failure.** `select_by_text` is fail-loud - (`ERROR during 'select': …`). Report a finding and stop that scenario — do **not** retry - with a different `choice_text`/`choice_index`. -- **Never retry a failed scenario.** If a scenario fails (command errors, unexpected output, - non-zero exit), report the finding and move on. Do **not** re-run the scenario hoping for - a different result — unless the scenario's `goals:` explicitly instruct a retry. Retrying - masks flaky behavior and makes the test suite unreliable as a regression signal. -- **Never adapt around broken goals.** If the goals instruct you to run a command or flag - that does not exist, or expect output that does not appear, fail the scenario. Do not - substitute an alternative command, skip the broken step, or invent a workaround. The - scenario must be updated by a human — the driving agent must not silently patch over it. -- **Never work around infrastructure failures.** If `azd` fails due to file-locking (common - on WSL when using the wrong binary), path resolution, missing tools, or similar environment - issues, that is a FAIL — not an invitation to install a different binary or modify the - environment. Report the infrastructure failure and move on. Workarounds mask setup gaps and - make results unreliable. This applies on all platforms (Windows/WSL, native Linux, macOS). -- **Prefer `choice_text` over `choice_index`** (indices shift between releases). -- **Clear a pre-filled text field before typing** (e.g. the agent-name prompt); otherwise - your value *appends* to the default (`defaultyourvalue`). -- **Pause before the first cloud-creating action.** The Step 4 cost confirmation covers - this; never enter a Tier 2 provision flow without it. - -## Parallelism & ordering - -- **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), one - sub-agent per scenario, each with a distinct descriptive `session_id` **postfixed with a - timestamp** (e.g. `fleet-1.04-init-from-code-1752434100`). The timestamp (Unix epoch - seconds) prevents collisions when multiple agent sessions drive the tester concurrently. - No `instance_id` is needed — each scenario's `cwd` already isolates itself (defaults to - the `-main` suffix). -- **Tier 1b** (`parallel-safe`, `verify-deploy`): runs **after all Tier 1 scenarios - complete**. Each Tier 1b scenario has a `requires:` field pointing to the Tier 1 scenario - whose scaffold it deploys. Only run a Tier 1b scenario if its prerequisite PASSED. Once - the prerequisites are confirmed, Tier 1b scenarios may be fanned out concurrently (each - has its own independent Azure environment). They also need the same cost acknowledgement - as Tier 2 since they provision real resources. -- **Same scenario N times** in parallel: pass `instance_id="1"`, `"2"`, … See the README's - parallel-readiness section for which scenarios support it. -- **Tier 2** (`serial-only`): never parallelize. Run `2.00-setup-deploy-shared-agent` first, - then `2.01-`…`2.18-` serially (they share one deployed agent and mutate shared session/file/ - endpoint state), then `2.99-teardown-down` last. -- **Validate the recipe with one scenario before fanning out** — confirm `load_scenario` → - `start_session` → one `send_action` round-trips for a single Tier 0 scenario first. - -### Fleet sub-agent rules - -When fanning out scenarios to sub-agents (e.g. Tier 0/1 parallel waves), every sub-agent -must obey these rules: - -- **Do not modify the environment.** Each sub-agent inherits the verified environment set up - by the orchestrator (Step 1b on Windows/WSL, or the user's build on native Linux/macOS). - Sub-agents must NOT install software, modify PATH, download binaries, or change the `azd` - binary or any extension — on any platform. -- **Infrastructure errors → FAIL and return.** If a sub-agent encounters an environment error - (wrong binary, file-locking, missing tool), it must FAIL the scenario with an infrastructure - finding and return the error to the orchestrator. It must NOT attempt to fix the environment. -- **Each sub-agent runs exactly one scenario.** It loads the scenario, drives the goals, and - reports PASS/FAIL. It does not make decisions about other scenarios or the overall run. - -## Capture per scenario - -Record, for the report: the scenario stem, tier, PASS/FAIL/SKIPPED, wall-clock **duration** -(`start_session` → `finish_session` incl. hooks, formatted `Hh Mm Ss`; `—` for SKIPPED), and -any `report_finding` text (confusing UX, errors, doc mismatches). SKIPPED scenarios include -the reason (e.g. "prerequisite `tier1/1.01-init-template-python.yaml` did not pass"). diff --git a/.github/skills/foundry-extension-scenario-authoring/SKILL.md b/.github/skills/foundry-extension-scenario-authoring/SKILL.md new file mode 100644 index 00000000000..91809a0737b --- /dev/null +++ b/.github/skills/foundry-extension-scenario-authoring/SKILL.md @@ -0,0 +1,201 @@ +--- +name: foundry-extension-scenario-authoring +license: MIT +metadata: + version: "1.0" + # Bump major on breaking prompt/trigger changes; bump minor on new references or authoring rules. + # 1.0: initial authoring + validation skill for the azure.ai.agents cli-interactive-tester + # scenarios. Taxonomy (tiers/tags/profile/hooks/fixtures/requires) is single-sourced in the + # scenarios README; this skill adds the authoring procedure and a no-execution validation loop. +description: >- + **WORKFLOW SKILL** — Authors and validates cli-interactive-tester **scenarios** for the + azure.ai.agents extension: writes a new goal-based scenario YAML (or edits an existing one) so + it follows the framework's tier / tag / hook / fixtures / requires conventions and the goals- + are-the-contract judging rules, then lint-validates it **without running it** (no Azure cost). + Typically driven through the foundry-extension-scenario-author agent. + + INVOKES: cli-interactive-tester MCP tool list_scenarios (for tag/lint validation only — never + start_session), read/edit of scenario YAML files, ask_user. + + USE FOR: write a new scenario, add scenario coverage for a command or flag, author a + cli-interactive-tester scenario, fix a scenario's tags / hooks / requires, add a fixture for a + scenario, bring a scenario up to the authoring contract, close a coverage gap flagged by a PR + regression run. + + DO NOT USE FOR: RUNNING scenarios or a suite (use foundry-extension-scenario-suite-run, or foundry-extension-scenario-pr-regression + for a PR — this skill never drives a scenario or incurs Azure cost), driving a single scenario + (that is the foundry-extension-scenario-worker agent), azd core preflight (use azd-preflight), changelog (use + changelog-generation), creating PRs (use pull-request), scenarios for any extension other than + azure.ai.agents. +--- + +# foundry-extension-scenario-authoring + +Authors and validates goal-based **scenarios** for the `azure.ai.agents` +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) suite under +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. This skill +**writes and lints** scenarios; it never **runs** them (running is `foundry-extension-scenario-suite-run` / +`foundry-extension-scenario-pr-regression`, driven by the `foundry-extension-scenario-worker` agent, and incurs Azure cost for +Tier 1b / Tier 2). + +## The authoring contract (read this first) + +A scenario's `goals:` are its **contract**: a run PASSES only when the product's real behavior +matches what the goals literally describe. So the whole point of authoring is to write goals +that are a *verifiable spec of correct behavior* — not a loose description the driver has to +interpret charitably. The judging rules are single-sourced in the scenarios README: +[**How scenarios are judged (authoring contract)**](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#how-scenarios-are-judged-authoring-contract). +The consequences for how you write goals: + +- **Write goals as literal, checkable assertions.** "Confirm the process exits cleanly and + prints a version string" is checkable; "make sure version works" is not. A driver will FAIL a + scenario whose goals aren't met — it will not rationalize a near-miss into a pass. +- **Never encode a workaround in a goal.** If a goal names a flag/command that doesn't exist or + expects output that never appears, the scenario FAILS by design. Keep goals current with the + product; don't paper over a bug in the goal text. +- **Key interactive pickers off stable text labels, not positions.** The driver prefers + `choice_text` over `choice_index`, so phrase picker goals around the label (e.g. *select "Use + the code in the current directory"*) — indices shift between releases. +- **Guard pre-filled prompts.** When a prompt comes pre-populated (e.g. the agent name), the + goal must tell the driver to **clear the field first** before typing, or the typed value + appends to the default. See the RESOURCE NAMING / AGENT NAME goals in + `tier1/1.04-init-from-code.yaml` for the canonical phrasing. +- **Gate cost explicitly.** Any goal that reaches `azd provision` / real resource creation + belongs in Tier 1b or Tier 2 and must be tagged and tiered accordingly (see below). + +> The deeper *why* behind the picker / clear-field / no-retry rules is the executor spec +> [`driving-mechanics.md`](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md). +> You don't need to restate it — just author goals that hold under it. + +## Scenario anatomy + +One file targets **one** command or flow. The taxonomy behind each field is single-sourced in +the README — this is the annotated skeleton; follow the links for the rules: + +```yaml +# One-line comment: tier, what it targets, and any cost/prereq note. +name: "short-kebab-name" # tester run label; keep it short and unique-ish +command: "azd ai agent …" # the installed extension entry point +cwd: "~/working/azd-agents--{instance}" # see cwd + idempotency below; /tmp for read-only +tags: ["tier:N", "cmd:", "parallel-safe"] # REQUIRED — see the Tags taxonomy +requires: "tier1/1.0X-….yaml" # optional; only when this scenario needs another to PASS first +env: # optional; init scenarios disable agent auto-detect + AZD_DISABLE_AGENT_DETECT: "1" +allocate_ports: [agent] # optional; only for scenarios that bind a port (e.g. 2.12) +pre: # optional host-side setup (reset dir, seed fixture, auth guard) + - run: "rm -rf {cwd-or-path}" + cwd: "~/working" + name: "reset working dir" +post: # optional host-side cleanup + - run: "…" + name: "cleanup" +goals: # the CONTRACT — literal, checkable steps + - "Wait for … and confirm …." + - "Take a screenshot of the final output." + - "Report a finding if …." +``` + +Field references (do not restate these — link to them): + +- **Tiers & placement** — which directory (`tier0/`…`tier2/`) and cost profile: + [README § Tiers](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#tiers). +- **Tags** — the `tier:N` / `cmd:*` / trait namespaces and rules: + [README § Tags](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#tags). +- **`requires:`** — cross-scenario prerequisites (Tier 1b → Tier 1): + [README § The `requires:` field](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#the-requires-field). +- **Profile placeholders** (`{prefix}`, `{subscription}`, `{region}`, `{model}`, `{tenant}`, + `{shared_agent_name}`, `{fixtures_dir}`, `{instance}`): + [README § Profile / overrides](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#profile--overrides). +- **Pre/post hooks** — semantics (host-side, sequential, fail-fast), fields, and the reset / + fixture-seed / auth-guard patterns: + [README § Pre/post hooks](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#prepost-hooks). +- **Fixtures** — the `{fixtures_dir}` seed pattern: + [README § Fixtures](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#fixtures). +- **Idempotency** — how a stateful scenario resets itself so re-runs start clean: + [README § Re-running scenarios](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#re-running-scenarios-idempotency). +- **Conventions & parallel-readiness** — resource naming, `{instance}` suffixing, port allocation: + [README § Conventions](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#conventions) + and [§ Parallel-readiness](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md#parallel-readiness--port-allocation). + +## Authoring procedure + +1. **Confirm the target and scope.** Identify the single command / flow under test and whether + it is a happy path or a `negative-path` (error / non-zero-exit) assertion. One command per + file — if you're tempted to test two, that's two scenarios (or a documented lifecycle + scenario, like `2.07-sessions-lifecycle`). If the request is broad or ambiguous, ask via + `ask_user` which command and tier are intended. + +2. **Pick the tier and file placement.** Choose the lowest tier that can actually exercise the + behavior (offline Tier 0 > auth-only Tier 1 > deploy-verify Tier 1b > cloud Tier 2), because + cost and flakiness rise with tier. Place the file in the matching `tierN/` directory and name + it `.-.yaml`, continuing the existing numbering in that directory. + +3. **Write `command` and `cwd`.** + - `command:` invokes the installed extension (`azd ai agent …`) or `azd …` for + provision/deploy/down steps. + - Read-only, stateless scenarios (`version`, `--help`, `sample list`) use `cwd: "/tmp"` and + declare no hooks. + - Stateful scenarios use a dedicated working dir suffixed with `-{instance}` (e.g. + `~/working/azd-agents--{instance}`) so parallel instances stay isolated. + +4. **Make it idempotent with hooks.** For a stateful scenario add a `pre` hook that `rm -rf`s + its own `cwd` (and, if it needs source code, a second hook that seeds a fixture from + `{fixtures_dir}` — see `tier1/1.04-init-from-code.yaml`). Tier 2 setup additionally downs any + leftover deployed project first. Prefer **pre-wipe only** (no `post` delete) so the scaffold + stays on disk for inspection. Use `continue_on_error` / a longer `timeout` only where an + existing scenario shows it's needed (e.g. the Tier 2 down hook). + +5. **Add a fixture only if required.** If the flow needs pre-existing source (the "use the code + in the current directory" path), add the minimal tree under `fixtures//` and seed it via + the hook. Keep fixtures minimal — just enough to satisfy detection. + +6. **Set `requires:` if it depends on another scenario.** Tier 1b deploy-verify scenarios point + `requires:` at the Tier 1 init scenario whose scaffold they deploy (a **relative path from + the scenarios root**). Don't use `requires:` to sequence independent scenarios — it's a hard + prerequisite, not an ordering hint. + +7. **Write the `tags:` list.** At minimum: one `tier:N`, at least one `cmd:*`, and exactly one of + `parallel-safe` / `serial-only` (Tier 0/1/1b → `parallel-safe`; Tier 2 → `serial-only`). Add + `negative-path`, `picker`, or `verify-deploy` when they apply. Multi-command flows carry + multiple `cmd:*` tags. + +8. **Write the `goals:` as the contract.** Follow the authoring-contract rules above: literal, + checkable, stable-label picks, clear-field-before-typing, screenshot key steps, and a final + "report a finding if …" goal. For resource-creating flows, include the RESOURCE NAMING and + AGENT NAME goals (prefix `{prefix}-`, suffix `-{instance}`) so parallel runs don't collide. + +## Validation loop (no execution) + +Validate **statically** — never `start_session`, never drive the scenario, never provision. + +1. **Tag lint via `list_scenarios`.** Run + `list_scenarios(root="", tags=[])` and confirm the new + file appears under each of its tags. If `list_scenarios` reports `tags: []` for the file, the + `tags:` field is missing or malformed — fix it. (`list_scenarios` uses a real YAML parser, so + this also catches YAML syntax errors in the file.) +2. **YAML shape.** Re-read the file and confirm required fields (`name`, `command`, `cwd`, + `tags`, `goals`) are present and well-formed, and that `goals` is a non-empty list of strings. +3. **`requires:` resolves.** If `requires:` is set, confirm the referenced path exists relative + to the scenarios root and points at the intended prerequisite. +4. **Fixture / hook paths resolve.** If a `pre` hook seeds a fixture, confirm the referenced + `fixtures//` tree exists and the hook uses `{fixtures_dir}` (not a hardcoded path). +5. **Placeholders only reference known variables.** Every `{name}` in `command` / `cwd` / hooks / + goals must be one of the profile placeholders or `{instance}` — a typo'd placeholder would run + against a literal string. +6. **Coverage cross-check.** If this scenario closes a coverage gap flagged by a PR regression + run, confirm its `cmd:*` tag matches the changed command so future impact-mapping picks it up. + +Report the validation result to the user. If they then want to *run* the new scenario to confirm +it drives cleanly, that's a separate, cost-gated step — hand off to the `foundry-extension-scenario-suite-run` +skill / `foundry-extension-scenario-orchestrator` agent (Tier 1b / Tier 2 incur Azure cost and need explicit +consent). This skill stops at a validated, unexecuted scenario. + +## Exit criteria + +- A single-command scenario YAML was authored (or corrected) in the correct `tierN/` directory + with a compliant `tags:` list, appropriate `cwd` + idempotency hooks, any needed fixture / + `requires:`, and `goals:` written as a literal, checkable contract. +- The static validation loop passed: `list_scenarios` lists the file under all its tags (not + `tags: []`), the YAML parses, and every `requires:` / fixture / placeholder reference resolves. +- **No scenario was executed** and no Azure resources were created; any run-to-confirm was handed + off to the run skills with a cost note. diff --git a/.github/skills/agent-scenario-tests/SKILL.md b/.github/skills/foundry-extension-scenario-pr-regression/SKILL.md similarity index 52% rename from .github/skills/agent-scenario-tests/SKILL.md rename to .github/skills/foundry-extension-scenario-pr-regression/SKILL.md index c5bb65760e6..8485fa61b02 100644 --- a/.github/skills/agent-scenario-tests/SKILL.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/SKILL.md @@ -1,29 +1,34 @@ --- -name: agent-scenario-tests +name: foundry-extension-scenario-pr-regression license: MIT metadata: - version: "1.0" + version: "2.0" # Bump major on breaking prompt/trigger changes; bump minor on new references or mapping rules. + # 2.0: renamed from agent-scenario-tests; execution mechanics moved to the scenarios' + # driving-mechanics.md; per-scenario driving delegated to the foundry-extension-scenario-worker agent. description: >- **WORKFLOW SKILL** — Runs the azure.ai.agents extension's cli-interactive-tester - scenarios locally as a PR regression check. Resolves the current branch's PR, - maps changed files to impacted scenario tags, drives the matching scenarios - through the cli-interactive-tester MCP server, and posts a results comment on the PR. + scenarios locally as a **PR-scoped** regression check. Resolves the current branch's PR, + maps changed files to impacted scenario tags, drives the matching scenarios through the + cli-interactive-tester MCP server (fanning out to foundry-extension-scenario-worker agents), and posts a + results comment on the PR. Typically dispatched by the foundry-extension-scenario-orchestrator agent, but + can trigger directly. INVOKES: git CLI, gh CLI, cli-interactive-tester MCP tools (list_scenarios, load_scenario, run_pre_hooks, start_session, send_action, finish_session, - run_post_hooks), ask_user. + run_post_hooks), the foundry-extension-scenario-worker agent, ask_user. - USE FOR: run agent scenarios, scenario regression check, cli-interactive-tester, - test agents extension PR, run impacted scenarios, check agents extension for regressions, - agent scenario tests, validate azure.ai.agents change. + USE FOR: run agent scenarios for a PR, scenario regression check, test agents extension PR, + run impacted scenarios, check agents extension for regressions before merge, validate an + azure.ai.agents change / diff. - DO NOT USE FOR: azd core preflight (use azd-preflight), changelog (use changelog-generation), - creating PRs (use pull-request), authoring brand-new scenarios from scratch without a code - change, running scenarios for any extension other than azure.ai.agents. + DO NOT USE FOR: running the whole suite or an arbitrary tag/tier sweep not tied to a diff + (use foundry-extension-scenario-suite-run), authoring or editing scenarios (use foundry-extension-scenario-authoring), azd core + preflight (use azd-preflight), changelog (use changelog-generation), creating PRs (use + pull-request), scenarios for any extension other than azure.ai.agents. --- -# agent-scenario-tests +# foundry-extension-scenario-pr-regression Runs the `azure.ai.agents` extension's interactive CLI scenarios as a **local** PR regression gate and reports the results back on the pull request. @@ -49,6 +54,13 @@ provisions nothing, **Tier 1b** (`verify-deploy`) provisions per-scenario Azure verify Tier 1 scaffolds actually deploy, and **Tier 2** incurs Azure cost for cloud-feature testing — both Tier 1b and Tier 2 are only run after explicit user confirmation. +This skill selects **what** to run (PR impact → tags) and owns the PR reporting. It does not +re-document **how** a scenario is driven — that is the executor spec at +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`, +which the spawned **foundry-extension-scenario-worker** agent follows for each scenario. This skill is the +selection-strategy half of the **foundry-extension-scenario-orchestrator** front door; the orchestrator may +dispatch it, or it may trigger directly. + > This skill drives scenarios **deliberately, with user consent**. That is different from > the extension's `AGENTS.md` rule that coding agents must not invoke scenarios on their > own during ordinary work — here the user has explicitly asked for a scenario run. @@ -59,7 +71,16 @@ testing — both Tier 1b and Tier 2 are only run after explicit user confirmatio {{ references/impact-mapping.md }} -{{ references/running-scenarios.md }} +## Execution mechanics (single source) + +Per-scenario driving — path style, the `requires:` gate, fail-loud rules, parallelism and +ordering, and capture — is **not** restated here. It lives once in the executor spec: +[`driving-mechanics.md`](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md) +(repo path +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`). +Fan each selected scenario out to a **foundry-extension-scenario-worker** agent, which loads, drives, and +reports one scenario per that spec. This skill's job is selection (impact → tags), gating, +and reporting — not driving. {{ references/reporting.md }} diff --git a/.github/skills/agent-scenario-tests/references/impact-mapping.md b/.github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md similarity index 100% rename from .github/skills/agent-scenario-tests/references/impact-mapping.md rename to .github/skills/foundry-extension-scenario-pr-regression/references/impact-mapping.md diff --git a/.github/skills/agent-scenario-tests/references/prerequisites.md b/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md similarity index 94% rename from .github/skills/agent-scenario-tests/references/prerequisites.md rename to .github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md index 0b27d198945..c2c914512e6 100644 --- a/.github/skills/agent-scenario-tests/references/prerequisites.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md @@ -12,7 +12,7 @@ the user exactly what to fix — do **not** try to work around it. tester runs inside WSL, so a Windows path like `C:\Repos\azure-dev\...\scenarios\tier0\0.01-version.yaml` must be passed as `/mnt/c/Repos/azure-dev/.../scenarios/tier0/0.01-version.yaml`. On macOS/Linux use the native - absolute path. See `running-scenarios.md` § Path style. + absolute path. See `driving-mechanics.md` § Path style (Windows → WSL). ### Tooling @@ -49,7 +49,7 @@ The scenarios reference `{prefix}`, `{subscription}`, `{region}`, `{model}`, `{t inside the scenarios directory. On Windows (where the tester runs inside WSL) this is the WSL-translated path (e.g. `/mnt/c/Repos/azure-dev/.../fixtures`); on native Linux/macOS it is the regular absolute path. Apply the same path-style logic used for - scenario paths (see `running-scenarios.md` § Path style). + scenario paths (see `driving-mechanics.md` § Path style (Windows → WSL)). Scenario pre-hooks use `{fixtures_dir}` to locate test fixture files. 4. Pass the merged map (including `shared_agent_name` and `fixtures_dir`) as `session_vars` on **every** `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` diff --git a/.github/skills/agent-scenario-tests/references/reporting.md b/.github/skills/foundry-extension-scenario-pr-regression/references/reporting.md similarity index 92% rename from .github/skills/agent-scenario-tests/references/reporting.md rename to .github/skills/foundry-extension-scenario-pr-regression/references/reporting.md index 1341c0689b4..068b2ab437a 100644 --- a/.github/skills/agent-scenario-tests/references/reporting.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/reporting.md @@ -41,7 +41,7 @@ formatting). Keep it scannable — full detail lives in the artifact. Suggested **Coverage gaps:** this PR also touches `mcp.go`, which has no scenario — consider adding one. -Run locally via the `agent-scenario-tests` skill. Not run in CI. +Run locally via the `foundry-extension-scenario-pr-regression` skill (or the `foundry-extension-scenario-orchestrator` agent). Not run in CI. ``` Rules: diff --git a/.github/skills/agent-scenario-tests/references/workflow.md b/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md similarity index 93% rename from .github/skills/agent-scenario-tests/references/workflow.md rename to .github/skills/foundry-extension-scenario-pr-regression/references/workflow.md index e87fe08ccd7..deb6d91c24b 100644 --- a/.github/skills/agent-scenario-tests/references/workflow.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md @@ -99,8 +99,10 @@ under `/.reports//`. ### Step 5 — Run the scenarios -Drive each selected scenario per `running-scenarios.md`. Start with a mandatory validation -step, then honor ordering: +Drive each selected scenario per the executor spec +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md` +— fan each scenario out to a **foundry-extension-scenario-worker** agent (one scenario per worker). Start with a +mandatory validation step, then honor ordering: 1. **Recipe validation (mandatory).** Run one Tier 0 scenario synchronously before fanning out. Pick a fast, non-interactive scenario (e.g. `0.01-version`). If it fails with an diff --git a/.github/skills/foundry-extension-scenario-suite-run/SKILL.md b/.github/skills/foundry-extension-scenario-suite-run/SKILL.md new file mode 100644 index 00000000000..7950e6f582d --- /dev/null +++ b/.github/skills/foundry-extension-scenario-suite-run/SKILL.md @@ -0,0 +1,195 @@ +--- +name: foundry-extension-scenario-suite-run +license: MIT +metadata: + version: "1.0" + # Bump major on breaking prompt/trigger changes; bump minor on new references or selection rules. + # 1.0: initial split from the scenarios README's fleet prompt; full / tag / tier sweep that is + # NOT tied to a PR diff. Per-scenario driving is delegated to the foundry-extension-scenario-worker agent and + # specified once in the scenarios' driving-mechanics.md. +description: >- + **WORKFLOW SKILL** — Runs the azure.ai.agents extension's cli-interactive-tester scenarios as + a **full or tag/tier-filtered sweep** that is *not* tied to a PR diff. Discovers scenarios via + list_scenarios, gates Azure cost, and fans them out to foundry-extension-scenario-worker agents in tier order, + then writes an aggregate report. Typically dispatched by the foundry-extension-scenario-orchestrator agent, but + can trigger directly. + + INVOKES: cli-interactive-tester MCP tools (list_scenarios, load_scenario, run_pre_hooks, + start_session, send_action, finish_session, run_post_hooks), the foundry-extension-scenario-worker agent, + ask_user. + + USE FOR: run the whole scenario suite, run all scenarios, smoke-test the azure.ai.agents + extension end to end, a tag-filtered sweep (e.g. "run every `init` scenario", "run everything + tagged `parallel-safe`"), a tier sweep (e.g. "run all of Tier 0", "run Tier 2"), a scheduled / + periodic full regression not scoped to a diff. + + DO NOT USE FOR: PR- or diff-scoped selection (use foundry-extension-scenario-pr-regression — it maps changed + files to impacted tags and comments on the PR), authoring or editing scenarios (use + foundry-extension-scenario-authoring), azd core preflight (use azd-preflight), changelog (use + changelog-generation), creating PRs (use pull-request), scenarios for any extension other than + azure.ai.agents. +--- + +# foundry-extension-scenario-suite-run + +Runs the `azure.ai.agents` extension's interactive CLI scenarios as a **full or filtered +sweep** and writes an aggregate report. Selection here is by **tag / tier**, not by a PR diff — +that is the one thing that distinguishes this skill from `foundry-extension-scenario-pr-regression`. + +## Overview + +The `azure.ai.agents` extension ships goal-based scenarios for the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server under `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. +They are **never run in CI** — they need the tester MCP server, a populated +`profile.local.yaml`, manual `az`/`gh` login, and (for Tier 1b / Tier 2) real Azure resources. + +This skill is the **opt-in, run-locally** flow for running many scenarios at once: the whole +suite, or a subset selected by `tags:` (a command like `cmd:init`, a trait like `parallel-safe`, +or a tier like `tier:0`). It: + +1. Resolves the sweep's **tag/tier filter** from the user's request (or runs everything). +2. Enumerates the matching scenarios via `list_scenarios`. +3. Gates Azure cost (Tier 1b / Tier 2), then drives the scenarios in the correct order. +4. Writes an aggregate `FINAL-REPORT.md` (no PR comment unless the user asks). + +It is cost- and side-effect-aware: Tier 0 is free/offline, Tier 1 needs Azure auth but +provisions nothing, **Tier 1b** (`verify-deploy`) provisions per-scenario Azure resources, and +**Tier 2** incurs Azure cost — Tier 1b and Tier 2 run only after explicit user confirmation. + +This skill selects **what** to run (tag/tier filter) and owns the aggregate report. It does not +re-document **how** a scenario is driven — that is the executor spec at +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`, +which the spawned **foundry-extension-scenario-worker** agent follows for each scenario. This skill is one of the +two selection strategies behind the **foundry-extension-scenario-orchestrator** front door; the orchestrator may +dispatch it, or it may trigger directly. + +> This skill drives scenarios **deliberately, with user consent**. That is different from the +> extension's `AGENTS.md` rule that coding agents must not invoke scenarios on their own during +> ordinary work — here the user has explicitly asked for a scenario run. + +## Prerequisites + +The prerequisites, profile merge, and `session_vars` derivation are **identical** to the +PR-regression flow and are single-sourced there. Follow +[`../foundry-extension-scenario-pr-regression/references/prerequisites.md`](../foundry-extension-scenario-pr-regression/references/prerequisites.md) +(repo path `.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md`) before doing +anything else. In particular: verify the tester MCP server and `profile.local.yaml`, and derive +the merged `session_vars` (profile merge + `shared_agent_name` + `fixtures_dir`) that must be +threaded, unchanged, through every scenario. + +The mandatory **`azd` binary build/verify gate** and **recipe validation** are shared gates the +**foundry-extension-scenario-orchestrator** owns (see the `foundry-extension-scenario-orchestrator` agent and +[`../foundry-extension-scenario-pr-regression/references/workflow.md`](../foundry-extension-scenario-pr-regression/references/workflow.md) +§ Step 1b). When the orchestrator dispatches this skill it has already run them; when this skill +runs standalone, perform those same gates first — do not run any scenario against an unverified +`azd` binary. + +## Workflow + +### Step 1 — Resolve the tag/tier filter + +Translate the user's request into a `list_scenarios` filter: + +- **Whole suite** ("run everything", "run all scenarios") → no tag filter (enumerate all), but + still honor tier ordering and the cost gate below. +- **By command** ("all `init` scenarios") → `tags=["cmd:init"]`. +- **By trait** ("the `parallel-safe` set", "smoke test") → the matching trait tag(s), e.g. + `tags=["parallel-safe"]`. +- **By tier** ("all of Tier 0", "run Tier 2") → the tier tag(s), e.g. `tags=["tier:0"]`. + +`list_scenarios` filtering is **OR across tags, case-sensitive, exact match**. See the scenarios +[README](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md) +§ Tags for the taxonomy. If the request is ambiguous about scope (one command vs. a whole tier), +ask via `ask_user` before enumerating. + +### Step 2 — Enumerate + +```text +list_scenarios(root="", tags=[, ...]) # omit tags for the whole suite +``` + +Group the returned scenarios by tier (0 / 1 / 1b / 2). This grouping drives both the cost gate +and the run order. + +### Step 3 — Confirm the plan (cost gate) + +Show the user the concrete scenario list grouped by tier and confirm via `ask_user` before +running: + +- Always list the Tier 0 scenarios that will run (free). +- If the set includes **Tier 1**, confirm `az login` is done. +- If the set includes **Tier 1b** or **Tier 2**, require an **explicit cost acknowledgement** + ("Tier 1b/2 provisions real Azure resources and incurs cost — proceed?"). If the user + declines, drop the cost-incurring tiers and run only what remains. + +Pick one `` of the form `YYYYMMDD-HHMMSS` for the whole sweep. All artifacts go +under `/.reports//`. + +### Step 4 — Run the scenarios + +Drive each selected scenario per the executor spec (see **Execution mechanics** below) — fan +each scenario out to a **foundry-extension-scenario-worker** agent, one scenario per worker, honoring ordering: + +1. **Recipe validation (mandatory).** Run one fast Tier 0 scenario (e.g. `0.01-version`) + synchronously before fanning out. If it fails with an infrastructure error, **stop the whole + run** and fix the environment — do not fan out into a fleet of failures. +2. **Tier 0 / Tier 1** (`parallel-safe`): fan out in small waves (4–6 at a time), each worker + with its own `cwd` (no `instance_id` for distinct scenarios). +3. **Tier 1b** (`verify-deploy`, ⚠️ cost): only after all Tier 1 workers finish, and only for + scenarios whose `requires:` prerequisite **PASSED** this run (otherwise ⏭️ SKIPPED); then fan + out concurrently. +4. **Tier 2** (`serial-only`, ⚠️ cost): never parallelize — `2.00-setup-deploy-shared-agent` + **first**, then `2.01-`…`2.18-` **serially**, `2.18-delete` before teardown, then + `2.99-teardown-down` **last**. + +`requires:` gating is a run-level decision: before dispatching a scenario that declares +`requires:`, look up the prerequisite's verdict **in this run** and tell the worker whether it +passed. Collect each worker's returned verdict block. + +### Step 5 — Report + +Aggregate every worker's verdict into `.reports//FINAL-REPORT.md` (see +**Reporting** below). Post a PR comment **only** if the user explicitly asked to tie this sweep +to a PR; a suite run is not PR-scoped by default. If a Tier 2 run started but was interrupted +before `2.99-teardown`, run `2.99-teardown-down` (or `2.00-setup`'s down hook) so no Azure +resources are orphaned, then report that status. + +## Execution mechanics (single source) + +Per-scenario driving — path style, the `requires:` gate, the fail-loud execution rules, +parallelism / ordering, and capture — is **not** restated here. It lives once in the executor +spec: +[`driving-mechanics.md`](../../../cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md) +(repo path +`cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md`). +Fan each selected scenario out to a **foundry-extension-scenario-worker** agent, which loads, drives, and reports +one scenario per that spec. This skill's job is selection (tag/tier filter), gating, ordering, +and reporting — not driving. + +## Reporting + +The `FINAL-REPORT.md` format and the PASS / FAIL / ⏭️ SKIPPED rules are single-sourced in +[`../foundry-extension-scenario-pr-regression/references/reporting.md`](../foundry-extension-scenario-pr-regression/references/reporting.md) +(repo path `.github/skills/foundry-extension-scenario-pr-regression/references/reporting.md`). Follow it, with two +sweep-specific differences: + +- **Header framing.** A sweep is not PR-scoped, so the run header records the **tag/tier filter** + that selected the set (e.g. "Filter: `cmd:init` across all tiers", or "Full suite") instead of + a PR number / impacted-tag set. +- **PR comment is opt-in.** By default write only the local artifact and print the summary to the + user. Only post a PR comment if the user explicitly tied the sweep to a PR. + +Never soften a real regression to make the table green. + +## Exit criteria + +- The tag/tier filter was resolved from the user's request (or confirmed as a whole-suite run), + and the concrete scenario set was **confirmed by the user** — including an explicit cost + acknowledgement before any Tier 1b or Tier 2 run. +- The shared gates (prerequisites, `azd` binary verify, recipe validation) passed before any + fan-out. +- Every selected scenario was driven to a recorded PASS / FAIL / ⏭️ SKIPPED with duration and + findings; scenarios with a `requires:` prerequisite that did not PASS are ⏭️ SKIPPED (not FAIL). +- A `FINAL-REPORT.md` was written under `.reports//`, and any Tier 1b / Tier 2 run + was followed by appropriate teardown so no Azure resources are left running. diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index e0d8712aed4..5bfeefdb8d0 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -48,11 +48,17 @@ files, monitor, endpoint, doctor, down) and are organized by tier: - **Tier 0** — offline, no Azure auth, no cost (help, version, validation, picker UX) - **Tier 1** — local-only with Azure auth (init flows) +- **Tier 1b** — deploy-verify: provisions a Tier 1 scaffold to confirm it deploys (incurs Azure cost) - **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) Each scenario carries a set of tags based on what is being tested and how. See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, -profile setup, and orchestration rules. +profile setup, and the human authoring contract, and +`tests/cli-interactive-tester-scenarios/driving-mechanics.md` for the executor +mechanics. Runs are driven through the `foundry-extension-scenario-orchestrator` agent — it routes +to the `foundry-extension-scenario-pr-regression` and `foundry-extension-scenario-suite-run` skills and fans work out +to `foundry-extension-scenario-worker` agents; new scenarios are authored through the +`foundry-extension-scenario-author` agent / `foundry-extension-scenario-authoring` skill. ### Guidance for coding agents @@ -64,14 +70,16 @@ cli-interactive-tester MCP server, a populated `profile.local.yaml`, and user-facing command path covered by an existing scenario (anything under `internal/cmd/` that maps to a `cmd:*` tag, or shared helpers used by those commands). In your summary, point the user at the relevant scenario(s) - and suggest they run the tester against the matching tag set to validate - the change. + and suggest they validate the change by selecting the `foundry-extension-scenario-orchestrator` + agent (for a PR, the `foundry-extension-scenario-pr-regression` skill maps the diff to the + matching tag set automatically). 2. **Add or update a scenario** when your change introduces a new command, flag, prompt, or user-visible flow — or meaningfully alters an existing - one. Place the new YAML alongside the others, follow the tagging taxonomy - documented in the scenarios README, and mention the new/changed scenario - in the PR description so reviewers know to exercise it. + one. Use the `foundry-extension-scenario-author` agent (or the `foundry-extension-scenario-authoring` skill) to + place the new YAML alongside the others following the tagging taxonomy and + authoring contract, and mention the new/changed scenario in the PR + description so reviewers know to exercise it. 3. **Do not modify scenarios to match buggy behavior.** Scenarios are user-facing specifications of how the command should behave; if a scenario diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index d4a0683be2c..566edecf820 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -17,14 +17,24 @@ cp profile.local.yaml.example profile.local.yaml # edit profile.local.yaml — set `prefix` and `subscription` at minimum ``` -Then ask Copilot CLI to load a scenario and accomplish its goals. The -orchestrator must **load both profile files, merge them (local overrides -shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}-{ts}` -(where `{ts}` is a compact timestamp like `MMDDHHmm`, e.g. `07141038`), and pass -the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, -`start_session`, and `run_post_hooks` call** — the scenario YAMLs reference -those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, -`{tenant}` (optional), and `{shared_agent_name}` placeholders. +**You don't drive these scenarios by hand.** Runs are executed by agents so they stay +deterministic and fail-loud. Pick the **`foundry-extension-scenario-orchestrator`** agent as your front door +and tell it what you want; it routes to a run skill and fans the work out to +`foundry-extension-scenario-worker` agents: + +- **Testing a PR / your change** → it uses the **`foundry-extension-scenario-pr-regression`** skill (maps the + diff to impacted scenarios and posts a results comment). +- **A full or tag/tier sweep** ("run everything", "all `init` scenarios", "all of Tier 2") → + it uses the **`foundry-extension-scenario-suite-run`** skill. + +The orchestrator (or the run skill) **loads both profile files, merges them (local overrides +shared), derives `shared_agent_name = {prefix}-{shared_agent_suffix}-{ts}` (where `{ts}` is a +compact timestamp like `MMDDHHmm`, e.g. `07141038`), and passes the merged map as +`session_vars` on every `load_scenario`, `run_pre_hooks`, `start_session`, and +`run_post_hooks` call** — the scenario YAMLs reference those values via `{prefix}`, +`{subscription}`, `{region}`, `{model}`, `{tenant}` (optional), and `{shared_agent_name}` +placeholders. The step-by-step driving rules those agents follow live in +[`driving-mechanics.md`](./driving-mechanics.md). Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting the working dir or seeding a fixture), and a few declare **`post:` hooks** @@ -32,39 +42,15 @@ the working dir or seeding a fixture), and a few declare **`post:` hooks** `run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. See [Pre/post hooks](#prepost-hooks) below. -Here's also a sample prompt to run all of the scenarios, utilizing fleet mode: +To run **everything**, select the `foundry-extension-scenario-orchestrator` agent and state your intent and +cost consent, e.g. *"Run the full scenario suite across all tiers; I accept the Tier 1b / +Tier 2 Azure cost."* It discovers the scenarios, enforces the prerequisite and `azd`-binary +gates, validates the recipe on one scenario, fans Tier 0/1 out in parallel waves, runs Tier 1b +after its Tier 1 prerequisites pass, runs Tier 2 serially, and writes a final report. -``` -Within the agents extension, there is a tests/cli-interactive-tester-scenarios directory, containing -a set of test scenarios for the cli-interactive-tester. I want you to use the cli-interactive-tester to; - load the scenarios, - start the session and accomplish the goals, - if the scenario declares pre or post hooks, run them before/after the session, - and take screenshots at each step. - -First, read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge -them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}-{ts}" -where {ts} is a compact timestamp (MMDDHHmm, e.g. 07141038). -Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / -run_post_hooks call — the scenarios reference {prefix}, {subscription}, {region}, {model}, -{tenant} (optional), and {shared_agent_name} placeholders. - -I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios -in tiers 0 and 1 are completely independent of each other and can be run in parallel. Tier 1b -(verify-deploy) scenarios depend on their Tier 1 prerequisite passing first — run them after Tier 1 -completes, but they can be parallelized with each other. The scenarios in tier 2 however rely on a -setup scenario, and the teardown scenario should be run last, so make sure to take that into account -when distributing the work. I want to run all of the tests regardless of tier, and I acknowledge -that tier 1b and tier 2 have an azure cost implication, that's fine. - -After all of these scenarios are run, create a final result report. - -Create a plan to accomplish this -``` - -For more selective fan-outs (e.g. "just the `init` scenarios" or "everything -in Tier 0") the tester's `list_scenarios` MCP tool filters by `tags:`. See -[Tags](#tags) below for the taxonomy and an example tag-filtered prompt. +For a **subset** (e.g. "just the `init` scenarios" or "everything in Tier 0") name the subset +instead — the `foundry-extension-scenario-suite-run` skill filters by `tags:` via `list_scenarios`. See +[Tags](#tags) for the taxonomy. ## Paths run inside WSL (on Windows) @@ -98,20 +84,11 @@ On macOS/Linux these are simply native paths (no WSL involved). ### This applies to MCP tool arguments too -The same path-resolution rule applies to **every path-shaped argument an -orchestrator passes to the tester's MCP tools** — most importantly the `path:` -argument on `load_scenario`, `run_pre_hooks`, and `run_post_hooks`, and the -`scenario_path:` argument on `start_session`. The server resolves them on the -WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: - -| Orchestrator OS | Pass to MCP tools | Don't pass | -| --- | --- | --- | -| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/tier0/0.01-version.yaml` | `C:\Repos\azure-dev\...\tier0\0.01-version.yaml` | -| macOS / Linux | native absolute path | — | - -**Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the -path style is almost certainly the cause — translate `C:\…` to `/mnt/c/…` and -retry one call before fanning out. +Driving agents must pass **WSL-style paths** to every path-shaped argument on the tester's MCP +tools (`path:` on `load_scenario` / `run_pre_hooks` / `run_post_hooks`, and `scenario_path:` / +`output_dir:` on `start_session`). The full rule, the Windows/macOS/Linux table, and the +`Scenario file not found` failure-mode hint live in the executor spec: +[`driving-mechanics.md`](./driving-mechanics.md) § Path style (Windows → WSL). ### Installing azd in WSL (Windows developers) @@ -216,122 +193,44 @@ advantage of both where it's safe. **not** parameterized with `{instance}` (doing so would break the shared-agent assumption) and should be run serially. -To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the -same `instance_id` for paired `run`/`invoke` sessions of one scenario). +How the executor actually fans these out (per-instance `session_id`s, wave sizes, tier +ordering) is specified in [`driving-mechanics.md`](./driving-mechanics.md) § Parallelism & +ordering — this section only documents which scenarios are *authored* to support concurrency. ## Orchestrating a fleet run -When a driving agent wants to run **many scenarios concurrently** (e.g. via -parallel background sub-agents, one scenario per sub-agent), pick the right -fan-out primitive for the shape of the run: - -- **Different scenarios in parallel** (the common case for a full Tier 0/1 - sweep): give each sub-agent a distinct, descriptive `session_id` — e.g. - `fleet-0.01-version`, `fleet-1.04-init-from-code` — and call `start_session` with - the scenario's own `cwd`. **No `instance_id` is needed**: each scenario's `cwd` - already isolates itself via the `{instance}` substitution, which defaults to - `"main"` when `instance_id` is omitted. -- **Same scenario N times in parallel:** use `instance_id="1"`, `"2"`, … per - call. See [Parallel-readiness](#parallel-readiness--port-allocation) for which - scenarios are authored to support this. -- **Tier 2 ordering is fixed**, not parallel-friendly. Run `2.00-setup` first, - then the targeted `2.01-`…`2.18-` scenarios **serially** (they share one deployed - agent and mutate shared state — sessions, files, endpoint configuration — - so parallel runs interfere), then `2.99-teardown` last. See the - [Tier 2](#tier-2--cloud-end-to-end-tier2---%EF%B8%8F-incurs-azure-cost) - section. -- **Tier 1b** (`verify-deploy`) is `parallel-safe` but must run **after Tier 1 - completes**. Check each scenario's `requires:` field — only proceed if the - prerequisite PASSED. Then fan out Tier 1b scenarios concurrently like Tier 0/1. - -### Operational guardrails for the orchestrator - -A few hard-won lessons that apply regardless of fleet size: - -- **Validate the recipe with one sub-agent before fanning out.** Spend 30 - seconds confirming that `load_scenario`, `start_session`, and one - `send_action` round-trip work end-to-end for *one* scenario before launching - a wave. This is the cheapest way to catch wiring issues (wrong path style, - wrong tool surface, auth not set up) before they multiply across many agents. -- **Background sub-agents are typically not cancellable mid-run.** Once launched, - they will run to completion (or until the runtime times them out). For Tier 1 - and especially Tier 2 scenarios with Azure side effects, launch - conservatively — a stop request can't recall an in-flight `azd provision`. -- **Keep waves small.** The wall-clock bottleneck on a fleet run is per-agent - LLM time and per-account model concurrency, not the MCP server (which is - per-`session_id`-parallel by design). Launching 4–6 sub-agents at a time and - rolling forward typically finishes a sweep faster than launching everything - at once. - -## Driving conventions - -These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent -should follow them so the runs actually *test* the CLI instead of papering over -its bugs: - -- **The scenario goals are the contract.** A scenario PASSES only when the - product's actual behavior matches what the goals describe. If the goals say - "expect error X" and the product prints a different error (even a reasonable - one), that is a FAIL. If the goals reference a flag or subcommand that no - longer exists, that is a FAIL. The driving agent's job is to **verify** goals - were met, not to **rationalize** why they weren't. Do not mark a scenario as - PASSED with an "observation" when the goals were not achieved — observations - are for incidental notes on scenarios that genuinely passed all their goals. -- **Don't verify/retry after a `select`.** These runs exist to catch picker - bugs; reading back the echo and "correcting" a pick hides the very defect the - test is for. Send the action and let downstream prompts surface any failure. -- **Treat a select miss as a hard failure.** The tester's `select_by_text` is - fail-loud: a missing target raises `LookupError`, surfaced as - `ERROR during 'select': …`. **Report a finding and stop** — do not retry with a - different `choice_text`/`choice_index` to work around it. -- **Never retry a failed scenario.** If a scenario fails (command errors, - unexpected output, non-zero exit), report the finding and move on. Do **not** - re-run the scenario hoping for a different result — unless the scenario's - `goals:` explicitly instruct a retry. Retrying masks flaky behavior and makes - the test suite unreliable as a regression signal. -- **Never adapt around broken goals.** If the goals instruct you to run a - command or flag that does not exist, or expect output that does not appear, - fail the scenario. Do not substitute an alternative command, skip the broken - step, or invent a workaround. The scenario must be updated by a human — the - driving agent must not silently patch over it. -- **Prefer `choice_text` over `choice_index`** when the label is stable (indices - shift between releases). -- **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent - name) come pre-populated with an editable default; typing without clearing - *appends* to it. Select-all then delete (or backspace) first so your value - replaces the default instead of producing `defaultyourvalue`. +Fleet orchestration — fan-out primitives, per-`session_id` timestamps, wave sizes, Tier 1b / +Tier 2 ordering, and the operational guardrails (validate the recipe first, launch +cost-incurring waves conservatively, keep waves small) — is the executor's job and is +specified once in [`driving-mechanics.md`](./driving-mechanics.md) § Parallelism & ordering. In +practice you don't orchestrate by hand: the `foundry-extension-scenario-orchestrator` agent runs that flow and +spawns one `foundry-extension-scenario-worker` per scenario. + +## How scenarios are judged (authoring contract) + +These are the rules that decide whether a scenario **passes**, so they matter most when you +*author* goals — write goals that hold under them. This is the human-facing half of the +contract; the operational rules the executor follows (select handling, retries, +`run_name` / `output_dir` / duration capture, environment integrity, path style) live in +[`driving-mechanics.md`](./driving-mechanics.md). A driving agent applies both. + +- **The scenario goals are the contract.** A scenario PASSES only when the product's actual + behavior matches what the goals describe. If the goals say "expect error X" and the product + prints a different error (even a reasonable one), that is a FAIL. If the goals reference a + flag or subcommand that no longer exists, that is a FAIL. So write goals as the *literal, + verifiable* spec of correct behavior — the driver's job is to **verify** them, not to + **rationalize** why they weren't met, and it will not mark a scenario PASSED with an + "observation" when the goals were not achieved. +- **Never adapt around broken goals.** If a goal instructs a command or flag that does not + exist, or expects output that does not appear, the driver **fails** the scenario rather than + substituting an alternative, skipping the step, or inventing a workaround. Keep goals current + so this doesn't happen — a broken goal must be fixed by a human, not patched over at run time. +- **Prefer stable labels.** When a goal drives an interactive picker, key it off a stable text + label rather than a positional index — the driver prefers `choice_text` over `choice_index` + because indices shift between releases. - **Pause before the first cloud-creating action.** Provisioning is expensive and - irreversible-ish; confirm with the user before entering an `init`/`provision` - flow that creates real resources (especially when running in parallel). -- **Pass `run_name=` to every `start_session` call.** The - scenario stem is the YAML filename without `.yaml` (e.g. `0.01-version`, - `2.02-show-json`, `2.12-run-local-and-invoke-local`). Without `run_name` the - tester auto-names the run folder `agent_YYYYMMDD_HHMMSS`, which makes - archived runs in `.reports//tester-reports/` hard to cross-reference - with the scenario list. For scenarios that start two sessions - (e.g. `2.12-run-local-and-invoke-local`), suffix the run_name with a role tag - (`2.12-run-local-and-invoke-local-run`, `2.12-run-local-and-invoke-local-invoke`) - so each session gets its own clearly named folder. -- **Pass `output_dir` to every `start_session` call** so the tester writes - screenshots and HTML reports directly into this repo's archive layout - instead of its own working directory. Use the WSL path of the - `.reports//tester-reports/` folder under this scenarios - directory, with `` of the form `YYYYMMDD-HHMMSS`. Pick **one** - `` per suite run and reuse it across every session — this - groups all scenarios from one run under a single folder. The driving agent - should also write the final cross-scenario summary to - `.reports//FINAL-REPORT.md` at the end. Example - `output_dir` (the WSL view of this scenarios directory in this repo): - `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.reports/20260603-171132/tester-reports`. - If your clone lives elsewhere, substitute the WSL path of *your* - `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. -- **Record a time-to-complete per scenario.** Capture wall-clock duration for - every scenario (from `start_session` to `finish_session`, including pre/post - hooks) and include it as a `Duration` column in the per-scenario tables of - `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). - This makes regression slowdowns easy to spot across runs — Tier 2 in - particular has scenarios that legitimately take many minutes (provision, - deploy) and others that should complete in seconds. + irreversible-ish; a run must have explicit cost consent before entering any `init` / + `provision` flow that creates real resources (especially in parallel). ## Tiers @@ -466,49 +365,14 @@ grouping — colons are treated as ordinary characters by the filter): | CLI arg-validation scenarios only | `["negative-path"]` | | Everything safe to run in parallel | `["parallel-safe"]` | -Sample prompt that uses tag filtering: - -``` -Use the cli-interactive-tester to run every `init` scenario across all tiers. - -First, call list_scenarios with root="tests/cli-interactive-tester-scenarios" -and tags=["cmd:init"] to enumerate the matching scenarios. - -Then read tests/cli-interactive-tester-scenarios/profile.yaml and -profile.local.yaml and merge them (local overrides shared); also derive -shared_agent_name = "{prefix}-{shared_agent_suffix}-{ts}" where {ts} is a compact -timestamp (MMDDHHmm, e.g. 07141038). Pass the merged map as -session_vars on every load_scenario / run_pre_hooks / start_session / -run_post_hooks call. - -For each scenario returned by list_scenarios: load it, run any pre hooks, -start the session and accomplish the goals (take screenshots at each step), -finish the session, run any post hooks. The Tier 0/1 `init` scenarios are -parallel-safe (also tagged `parallel-safe`); fan them out via fleet mode. -The Tier 2 `init` scenario (`2.00-setup-deploy-shared-agent`) is `serial-only` -— run it on its own and only if I confirm I want to spend on Azure resources. -``` - -You can also get copilot to generate the tags list instead of manually specifying -it. For example, if you want to run all of the scenarios to test the changes -in a PR, modify the above prompt to start with something like: - -``` -Here's a PR: https://github.com/Azure/azure-dev/pull/8532. In the -tests\cli-interactive-tester-scenarios directory, there are a set of test scenarios, -with tags to categorize what they're testing. I want you to come up with a set of -tags which, when used to select these test scenarios, would properly test the -changes made by the PR provided. - -Next, call list_scenarios with those tags, to enumerate matching scenarios. - -Then read tests/cli-interactive-tester-scenarios/profile.yaml and .... - -``` +To run a **tag or tier subset**, use the `foundry-extension-scenario-suite-run` skill (or the +`foundry-extension-scenario-orchestrator` agent) and name the subset — e.g. *"run every `init` scenario across +all tiers"* or *"run everything tagged `parallel-safe`"*. It calls `list_scenarios` with the +right tags, applies the cost gate for any Tier 1b / Tier 2 members, and drives them. -And, if you're running these scenarios as a part of creating or reviewing a PR, -you can ask copilot to generate a summary report and add it as a comment directly -on the pull request. +To **test a PR**, use the `foundry-extension-scenario-pr-regression` skill (or the `foundry-extension-scenario-orchestrator` +agent): it maps the PR's changed files to the impacted tags automatically, enumerates the +matching scenarios, runs them, and posts a summary report as a PR comment. When adding a new scenario, give it a `tags:` list that follows this taxonomy: at minimum a `tier:N`, at least one `cmd:*`, and either diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md new file mode 100644 index 00000000000..6596e6f6d83 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/driving-mechanics.md @@ -0,0 +1,180 @@ + +# Driving mechanics — the executor spec + +**This is the single source of truth for *how* the cli-interactive-tester scenarios are +driven.** It is written for the **executor** — the `foundry-extension-scenario-worker` agent that drives one +scenario, and the run skills (`foundry-extension-scenario-pr-regression`, `foundry-extension-scenario-suite-run`) that select and +fan out scenarios. Agents and skills **link to this file** rather than restating it, so the +rules live in exactly one place. + +- **Humans** author scenarios and read results — they don't perform these steps by hand. The + human-facing **authoring contract** (what a passing scenario *means*, and what you owe a + scenario as its author) lives in [`README.md`](./README.md). This doc is the operational + counterpart: the runtime rules a driver must obey. +- If a rule here and the README's authoring contract ever seem to disagree, the README defines + *intent* (how scenarios are judged) and this file defines *mechanics* (how to execute); they + are designed to agree. + +--- + +## Path style (Windows → WSL) + +On Windows the tester drives CLIs through **tmux inside WSL**, and it resolves every +path-shaped MCP argument on the **WSL** side — not the orchestrator side. Pass POSIX paths: + +| Orchestrator OS | Pass to MCP tools | Don't pass | +| --- | --- | --- | +| Windows | `/mnt/c/Repos/azure-dev/.../scenarios/tier0/0.01-version.yaml` | `C:\Repos\azure-dev\...\tier0\0.01-version.yaml` | +| macOS / Linux | native absolute path | — | + +This applies to **every** path-shaped argument: `path:` on `load_scenario` / `run_pre_hooks` / +`run_post_hooks`, `scenario_path:` on `start_session`, and `output_dir:`. + +**Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the path style is +almost certainly the cause — translate `C:\…` → `/mnt/c/…` and retry **one** call before +fanning out. + +--- + +## Environment integrity (never work around a broken environment) + +The driving agent must **never install, replace, or modify** the `azd` binary, any `azd` +extension, or any system tool during a run — on any OS. The environment is prepared **before** +scenarios start (the `azd` build/verify gate; see the orchestrator's prerequisites). If it is +broken, the run **stops** — the agent does not fix it. + +- If a scenario fails due to an environment issue (wrong binary, missing tool, file-locking on + WSL, path resolution failure, or similar), report it as **FAIL** with an infrastructure + finding. Do **not** work around it by installing packages, switching binaries, downloading + builds, or modifying system state. +- This applies to the orchestrator **and every fleet sub-agent**. No participant may alter the + test environment. + +--- + +## Per-scenario loop + +For each selected scenario: + +0. **Check `requires:`** — if the scenario declares a `requires:` field (a relative path from + the scenarios root, e.g. `tier1/1.01-init-template-python.yaml`), look up the + prerequisite's result **in the current run**: + - Prerequisite **PASSED** → proceed (step 1+). + - Prerequisite **FAILED / not run / SKIPPED** → record this scenario as ⏭️ **SKIPPED** with + reason `prerequisite did not pass` and move on. +1. `load_scenario(path=, session_vars=)` — also reports whether the + scenario declares `pre` / `post` hooks. +2. If it has `pre` hooks: `run_pre_hooks(path=…, session_vars=…)`. Hooks run host-side, + sequentially, fail-fast (unless a hook sets `continue_on_error: true`). +3. `start_session(scenario_path=…, session_vars=…, run_name=, output_dir=)`. +4. Drive the scenario's `goals:` with `send_action` / `select` / screenshots, then + `finish_session` (this releases ports and generates the HTML report). +5. If it has `post` hooks: `run_post_hooks(path=…, session_vars=…)`. + +Always `finish_session` for every session you start. + +--- + +## Execution rules (fail-loud) + +These make the run actually *test* the CLI instead of papering over its bugs. They are the +operational form of the README's **authoring contract** — see +[`README.md`](./README.md) for what each means for goal-writing. + +- **The scenario goals are the contract.** A scenario PASSES **only** when the product's actual + behavior matches what the goals describe. If the goals say "expect error X" and the product + prints a different (even reasonable) error, that is a **FAIL**. If the goals reference a flag + or subcommand that no longer exists, that is a **FAIL**. Your job is to **verify** goals were + met, not to **rationalize** why they weren't. Never mark a scenario PASSED with an + "observation" when the goals were not achieved — observations are for incidental notes on + scenarios that genuinely passed all goals. +- **Never adapt around broken goals.** If the goals instruct you to run a command/flag that + doesn't exist, or expect output that doesn't appear, **FAIL** the scenario. Do not substitute + an alternative command, skip the broken step, or invent a workaround — a human must update + the scenario. +- **Never retry a failed scenario.** On failure (command error, unexpected output, non-zero + exit) report the finding and move on. Do **not** re-run hoping for a different result unless + the scenario's `goals:` explicitly instruct a retry — retrying masks flakiness. +- **Don't verify/retry after a `select`.** Reading back the echo and "correcting" a pick hides + the very picker defect these runs exist to catch. Send the action; let downstream prompts + surface any failure. +- **Treat a `select` miss as a hard failure.** The tester's `select` is fail-loud (a missing + target surfaces as `ERROR during 'select': …`). Report a finding and **stop that scenario** — + do not retry with a different `choice_text` / `choice_index`. +- **Prefer `choice_text` over `choice_index`** when the label is stable (indices shift between + releases). +- **Clear a pre-filled text field before typing** (e.g. the agent-name prompt): select-all then + delete/backspace first, otherwise your value *appends* to the default (`defaultyourvalue`). +- **Pause before the first cloud-creating action.** Provisioning is expensive and + irreversible-ish; the orchestrator's cost/consent gate must be satisfied before entering any + `init` / `provision` flow that creates real resources (especially in parallel). + +--- + +## Parallelism & ordering + +Concurrency primitive: **parallel background sub-agents, one scenario per sub-agent.** + +- **Validate the recipe with one scenario before fanning out.** Confirm `load_scenario` → + `start_session` → one `send_action` round-trips for a single fast Tier 0 scenario (e.g. + `0.01-version`). If it fails with an infrastructure error, **stop the whole run** and fix the + environment — do not fan out into a fleet of failures. +- **Tier 0 / Tier 1** (`parallel-safe`): fan out in **small waves (4–6 at a time)**, rolling + forward. Give each sub-agent a distinct, descriptive `session_id` **suffixed with a Unix-epoch + timestamp** (e.g. `fleet-1.04-init-from-code-1752434100`) to avoid collisions when multiple + agent sessions drive the tester concurrently. **No `instance_id` needed** — each scenario's + `cwd` already isolates itself via `{instance}`, which defaults to `"main"`. +- **Tier 1b** (`parallel-safe`, `verify-deploy`, ⚠️ Azure cost): runs **after all Tier 1 + scenarios complete**. Each declares a `requires:` field pointing at the Tier 1 scaffold it + deploys — only run it if that prerequisite **PASSED**; otherwise ⏭️ SKIP. Once prerequisites + are confirmed, fan out Tier 1b concurrently (independent Azure environments). Needs the same + cost acknowledgement as Tier 2. +- **Tier 2** (`serial-only`, ⚠️ Azure cost): **never parallelize.** Run + `2.00-setup-deploy-shared-agent` **first**, then `2.01-`…`2.18-` **serially** (they share one + deployed agent and mutate shared session/file/endpoint state), `2.18-delete` before teardown, + then `2.99-teardown-down` **last**. Tier 2 uses **no `instance_id`** (it would break the + shared-agent assumption). +- **Same scenario N times in parallel:** pass `instance_id="1"`, `"2"`, … per `start_session` + call; reuse the same `instance_id` for paired `run`/`invoke` sessions of one scenario. Only + scenarios authored for it support this (Tier 0 work-dir scenarios, all Tier 1 `init` + scenarios, and `2.12` for its allocated `{agent}` port). + +### Keep waves small + +The wall-clock bottleneck is per-agent LLM time and per-account model concurrency, not the MCP +server (which is per-`session_id`-parallel by design). Launching 4–6 sub-agents at a time and +rolling forward typically finishes a sweep faster than launching everything at once. Background +sub-agents are typically **not cancellable mid-run** — for Tier 1b/Tier 2 Azure side effects, +launch conservatively (a stop request can't recall an in-flight `azd provision`). + +### Fleet sub-agent rules + +Every sub-agent spawned for a wave must obey: + +- **Do not modify the environment** (see [Environment integrity](#environment-integrity-never-work-around-a-broken-environment)). +- **Infrastructure errors → FAIL and return.** Fail the scenario with an infrastructure finding + and return control to the orchestrator; do not attempt a fix. +- **Each sub-agent runs exactly one scenario** — loads it, drives the goals, reports + PASS/FAIL/SKIP. It makes no decisions about other scenarios or the overall run. + +--- + +## Artifacts & capture + +- **`run_name=`** on every `start_session` (the YAML filename without `.yaml`, + e.g. `0.01-version`, `2.02-show-json`). Without it the tester auto-names the run folder + `agent_YYYYMMDD_HHMMSS`, which is hard to cross-reference. For scenarios that start two + sessions (`2.12-run-local-and-invoke-local`), suffix a role tag (`…-run`, `…-invoke`). +- **`output_dir`** on every `start_session`, pointing at the WSL path of + `/.reports//tester-reports`. Pick **one** `` + (form `YYYYMMDD-HHMMSS`) per suite run and **reuse it across every session**, so all scenarios + from one run group under a single folder. +- **Screenshot key steps** and file `report_finding` for any confusing UX, error, or doc + mismatch. +- **Record per scenario** for the report: scenario stem, tier, **PASS / FAIL / ⏭️ SKIPPED**, + wall-clock **duration** (`start_session` → `finish_session`, including hooks; formatted + `Hh Mm Ss`, e.g. `3m 21s`, `1h 04m 12s`; `—` for SKIPPED), and any `report_finding` text. + SKIPPED scenarios include the reason (e.g. `prerequisite tier1/1.01-init-template-python.yaml + did not pass`). +- The driving agent writes the final cross-scenario summary to + `.reports//FINAL-REPORT.md` (the `.reports/` tree is git-ignored). From d1de5ccdd814568bd1c9b2e0ab9f0f2e66390f25 Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 4 Aug 2026 11:25:14 -0700 Subject: [PATCH 42/42] Bootstrap pinned Go for WSL scenarios Install and verify the Go version from cli/azd/go.mod during the mandatory WSL setup gate, then align scenario documentation and orchestrator guidance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 89c4d3b8-af6f-471d-a01f-299d4c5fd1a5 --- ...y-extension-scenario-orchestrator.agent.md | 8 +- .../references/prerequisites.md | 3 +- .../references/workflow.md | 9 +- .../README.md | 19 +- .../setup-wsl.sh | 173 +++++++++++++++--- 5 files changed, 176 insertions(+), 36 deletions(-) diff --git a/.github/agents/foundry-extension-scenario-orchestrator.agent.md b/.github/agents/foundry-extension-scenario-orchestrator.agent.md index 5505db266f6..5ff405dcbd7 100644 --- a/.github/agents/foundry-extension-scenario-orchestrator.agent.md +++ b/.github/agents/foundry-extension-scenario-orchestrator.agent.md @@ -55,9 +55,11 @@ source — read it, don't restate it. 2. **`azd` binary gate (mandatory).** Ensure a verified native-Linux `azd` dev build is installed before any scenario runs, per `.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md` § Step 1b (Windows/WSL: - `setup-wsl.sh` then confirm `which azd` = `/usr/local/bin/azd` and `azd version` shows the - dev string; native Linux/macOS: confirm the user's dev build). **If verification fails, - stop** — do not run scenarios against the wrong binary. + `setup-wsl.sh` bootstraps the repository-pinned Go version when needed, then confirm + `which azd` = `/usr/local/bin/azd` and `azd version` shows the dev string; native + Linux/macOS: confirm the user's dev build without running WSL setup). **If bootstrap or + verification fails, stop** — do not run scenarios against the wrong binary and do not + delegate environment repair to a worker. 3. **Cost / consent gate.** List the plan grouped by tier. Tier 0 is free; Tier 1 needs `az login`; **Tier 1b and Tier 2 provision real Azure resources** and require an *explicit* cost acknowledgement before running. If the user declines, drop the cost-incurring tiers. diff --git a/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md b/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md index c2c914512e6..5dcd39de932 100644 --- a/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/prerequisites.md @@ -21,7 +21,8 @@ the user exactly what to fix — do **not** try to work around it. | `git` + `gh` CLIs | `gh auth status` | Ask the user to run `gh auth login`. | | cli-interactive-tester MCP server | The `list_scenarios` / `start_session` MCP tools are available to you | Stop. Tell the user to register the cli-interactive-tester MCP server (see its README) and re-run. | | `profile.local.yaml` | File exists in the scenarios dir | Stop. Tell the user to `cp profile.local.yaml.example profile.local.yaml` and set `prefix` + `subscription`. | -| Native Linux `azd` in WSL (Windows only) | Inside WSL, run `which azd` — must return `/usr/local/bin/azd` (not `/mnt/c/…` or a path ending in `azd.exe`). Then run `azd version` — must contain the expected dev version string. | **Hard stop.** If `which azd` returns a Windows interop path, the file-locking on UNC paths will fail all init/provision scenarios. Run Step 1b (`setup-wsl.sh`) to build and install native Linux azd. Do not proceed until both checks pass. | +| WSL setup tools (Windows only) | Inside WSL, verify `git`, `curl` or `wget`, `awk`, `grep`, `tar`, `sha256sum`, `uname`, and `sudo` are available | **Hard stop.** `setup-wsl.sh` bootstraps the exact Go version from `cli/azd/go.mod`, but it does not install general OS packages. Tell the user which setup tool is missing. | +| Native Linux `azd` in WSL (Windows only) | After Step 1b, run `which azd` — must return `/usr/local/bin/azd` (not `/mnt/c/…` or a path ending in `azd.exe`). Then run `azd version` — must contain the expected dev version string. | **Hard stop.** If `which azd` returns a Windows interop path, file-locking on UNC paths will fail all init/provision scenarios. Re-run Step 1b (`setup-wsl.sh`), which also installs the pinned Go toolchain when needed. Do not proceed until both checks pass. | | Native Linux `azd` (native Linux/macOS) | `azd version` returns the expected dev build | Ask the user to build and install `azd` from source. No special path check is needed — any valid `azd` path works on native Linux. | ### Auth (tier-dependent — only enforce for tiers actually selected) diff --git a/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md b/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md index deb6d91c24b..661f283fdca 100644 --- a/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md +++ b/.github/skills/foundry-extension-scenario-pr-regression/references/workflow.md @@ -24,8 +24,15 @@ start_session(command="bash /mnt/c//setup-wsl.sh", run_name="setup-wsl") ``` +`setup-wsl.sh` owns the WSL build toolchain: it reads the exact Go version from +`cli/azd/go.mod` and installs that official version under `/usr/local/go` when Go is absent or +mismatched. The download is SHA-256 verified. This authorized bootstrap happens before scenario +execution and is not a scenario workaround. + Wait for it to print "Done. WSL is ready for scenario testing." and then `finish_session`. -If the build fails, stop and report the build error — do not proceed with stale binaries. +If Go cannot be downloaded, verified, or installed, or if the build fails, stop and report the +gate error — do not proceed with stale binaries and do not ask a scenario worker to repair the +environment. After `setup-wsl.sh` succeeds, **verify** the installation by starting a quick tester session and running `which azd && azd version`. Confirm that: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 566edecf820..299c7b83082 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -105,17 +105,26 @@ bash setup-wsl.sh ``` This script: -1. Cross-compiles `azd` core (`linux/amd64`) → `/usr/local/bin/azd` -2. Ensures the extensions dev kit (`microsoft.azd.extensions`) is installed -3. Builds, packages, and installs the `azure.ai.agents` extension from source +1. Reads the required Go version from `cli/azd/go.mod` and, when necessary, downloads the + matching official WSL architecture build from `go.dev`, verifies its SHA-256 checksum, and + installs it under `/usr/local/go` +2. Builds `azd` core for the native WSL architecture → `/usr/local/bin/azd` +3. Ensures the extensions dev kit (`microsoft.azd.extensions`) is installed +4. Builds, packages, and installs the `azure.ai.agents` extension from source using `azd x build` → `azd x pack --bundle` → `azd extension install` -4. Verifies both azd and the extension report expected dev versions +5. Verifies both azd and the extension report expected dev versions The script properly registers the extension in azd's config, so it will always use your dev build — never the published registry version. **Re-run `setup-wsl.sh` after every local code change** you want to test. -Requires the Go toolchain and sudo access in WSL. +It supports x86-64 and ARM64 WSL environments. Go does not need to be installed first; the +script bootstraps the exact repository-pinned version. It requires network access to `go.dev`, +Git, `curl` or `wget`, `awk`, `grep`, `tar`, `sha256sum`, `uname`, and sudo access in WSL. It +does not install general OS packages or modify shell startup files. + +On native Linux or macOS, do not run `setup-wsl.sh`. Build and install the repository's +development `azd` and extension through your normal local workflow. ## Authentication diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh index 61a59b59576..9d919895afa 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/setup-wsl.sh @@ -2,23 +2,24 @@ # setup-wsl.sh — Build and install native Linux azd + extension for WSL testing. # # Run this from inside WSL (or via `wsl bash setup-wsl.sh` from Windows) after -# making local code changes. It compiles native Linux/amd64 binaries from the +# making local code changes. It compiles native Linux binaries from the # repo source so the cli-interactive-tester drives your dev build directly. # # Prerequisites: -# - Go toolchain installed in WSL (or accessible via PATH) # - Git installed in WSL -# - sudo access (for installing azd to /usr/local/bin) +# - curl or wget, plus awk, grep, tar, sha256sum, and uname +# - sudo access (for installing Go and azd under /usr/local) # # Usage: # cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios # bash setup-wsl.sh # # What it does: -# 1. Builds azd core (linux/amd64) → /usr/local/bin/azd -# 2. Ensures the azd extensions dev kit (microsoft.azd.extensions) is installed -# 3. Builds + packages + installs the azure.ai.agents extension from source -# 4. Verifies the dev version is running +# 1. Installs the Go version pinned by cli/azd/go.mod when needed +# 2. Builds azd core for the native Linux architecture -> /usr/local/bin/azd +# 3. Ensures the azd extensions dev kit (microsoft.azd.extensions) is installed +# 4. Builds + packages + installs the azure.ai.agents extension from source +# 5. Verifies the dev version is running set -euo pipefail @@ -28,6 +29,16 @@ EXTENSION_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" AZD_DIR="$(cd "$EXTENSION_DIR/../.." && pwd)" REPO_ROOT="$(cd "$AZD_DIR/../.." && pwd)" +TEMP_DIR="$(mktemp -d)" +BUNDLE_ZIP="" +cleanup() { + rm -rf "$TEMP_DIR" + if [ -n "$BUNDLE_ZIP" ]; then + rm -f "$BUNDLE_ZIP" + fi +} +trap cleanup EXIT + echo "=== setup-wsl.sh ===" echo " Repo root: $REPO_ROOT" echo " azd source: $AZD_DIR" @@ -35,53 +46,166 @@ echo " Extension src: $EXTENSION_DIR" echo "" # --- Prerequisites --- -if ! command -v go &>/dev/null; then - echo "ERROR: Go toolchain not found. Install Go in WSL first." >&2 +for cmd in git awk grep tar sha256sum sudo uname; do + if ! command -v "$cmd" &>/dev/null; then + echo "ERROR: Required command '$cmd' was not found in WSL." >&2 + exit 1 + fi +done + +if ! grep -qi microsoft /proc/sys/kernel/osrelease 2>/dev/null; then + echo "ERROR: setup-wsl.sh must only be run inside WSL." >&2 + exit 1 +fi + +if command -v curl &>/dev/null; then + download() { + curl --fail --location --silent --show-error --output "$2" "$1" + } +elif command -v wget &>/dev/null; then + download() { + wget --quiet --output-document="$2" "$1" + } +else + echo "ERROR: curl or wget is required to download the pinned Go toolchain." >&2 exit 1 fi if ! sudo -n true 2>/dev/null; then - echo "NOTE: sudo access is needed to install azd to /usr/local/bin." + echo "NOTE: sudo access is needed to install Go and azd under /usr/local." echo " You may be prompted for your password." fi -# --- Step 1: Build azd core --- -echo "▸ Building azd core (linux/amd64)..." +# --- Step 1: Ensure the repository-pinned Go toolchain --- +GO_VERSION=$(awk '$1 == "go" { sub(/\r$/, "", $2); print $2; exit }' "$AZD_DIR/go.mod") +if [[ ! "$GO_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([a-z0-9.-]+)?$ ]]; then + echo "ERROR: Could not read a valid Go version from $AZD_DIR/go.mod." >&2 + exit 1 +fi + +case "$(uname -m)" in + x86_64|amd64) + GO_ARCH="amd64" + ;; + aarch64|arm64) + GO_ARCH="arm64" + ;; + *) + echo "ERROR: Unsupported WSL architecture '$(uname -m)'." >&2 + exit 1 + ;; +esac + +export PATH="/usr/local/go/bin:$PATH" +export GOTOOLCHAIN=local +EXPECTED_GO_VERSION="go${GO_VERSION}" +EXPECTED_GO_PLATFORM="linux/${GO_ARCH}" +CURRENT_GO_VERSION="" +CURRENT_GO_PLATFORM="" + +if command -v go &>/dev/null; then + read -r _ _ CURRENT_GO_VERSION CURRENT_GO_PLATFORM < <(go version 2>/dev/null || true) || true +fi + +if [ "$CURRENT_GO_VERSION" = "$EXPECTED_GO_VERSION" ] && + [ "$CURRENT_GO_PLATFORM" = "$EXPECTED_GO_PLATFORM" ]; then + echo "▸ Using repository-pinned Go toolchain: $CURRENT_GO_VERSION $CURRENT_GO_PLATFORM" +else + GO_ARCHIVE="go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" + GO_METADATA="$TEMP_DIR/go-releases.json" + GO_ARCHIVE_PATH="$TEMP_DIR/$GO_ARCHIVE" + + echo "▸ Installing repository-pinned Go toolchain: $EXPECTED_GO_VERSION $EXPECTED_GO_PLATFORM" + download "https://go.dev/dl/?mode=json&include=all" "$GO_METADATA" + + GO_SHA256=$(awk -v target="$GO_ARCHIVE" ' + index($0, "\"filename\"") && index($0, "\"" target "\"") { + found = 1 + next + } + found && index($0, "\"sha256\"") { + value = $0 + sub(/^.*"sha256"[[:space:]]*:[[:space:]]*"/, "", value) + sub(/".*$/, "", value) + print value + exit + } + ' "$GO_METADATA") + + if [[ ! "$GO_SHA256" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: Could not resolve the official SHA-256 for $GO_ARCHIVE." >&2 + exit 1 + fi + + download "https://go.dev/dl/$GO_ARCHIVE" "$GO_ARCHIVE_PATH" + if ! printf '%s %s\n' "$GO_SHA256" "$GO_ARCHIVE_PATH" | sha256sum --check --status; then + echo "ERROR: SHA-256 verification failed for $GO_ARCHIVE." >&2 + exit 1 + fi + + tar -C "$TEMP_DIR" -xzf "$GO_ARCHIVE_PATH" + read -r _ _ CANDIDATE_GO_VERSION CANDIDATE_GO_PLATFORM < <("$TEMP_DIR/go/bin/go" version) + if [ "$CANDIDATE_GO_VERSION" != "$EXPECTED_GO_VERSION" ] || + [ "$CANDIDATE_GO_PLATFORM" != "$EXPECTED_GO_PLATFORM" ]; then + echo "ERROR: Downloaded Go archive verification failed." >&2 + echo " Expected: $EXPECTED_GO_VERSION $EXPECTED_GO_PLATFORM" >&2 + echo " Got: $CANDIDATE_GO_VERSION $CANDIDATE_GO_PLATFORM" >&2 + exit 1 + fi + + sudo rm -rf /usr/local/go + sudo mv "$TEMP_DIR/go" /usr/local/go + hash -r + + read -r _ _ CURRENT_GO_VERSION CURRENT_GO_PLATFORM < <(go version) + if [ "$CURRENT_GO_VERSION" != "$EXPECTED_GO_VERSION" ] || + [ "$CURRENT_GO_PLATFORM" != "$EXPECTED_GO_PLATFORM" ]; then + echo "ERROR: Go installation verification failed." >&2 + echo " Expected: $EXPECTED_GO_VERSION $EXPECTED_GO_PLATFORM" >&2 + echo " Got: $CURRENT_GO_VERSION $CURRENT_GO_PLATFORM" >&2 + exit 1 + fi + + echo " Installed and verified $CURRENT_GO_VERSION $CURRENT_GO_PLATFORM" +fi +echo "" + +# --- Step 2: Build azd core --- +echo "▸ Building azd core ($EXPECTED_GO_PLATFORM)..." COMMIT=$(cd "$REPO_ROOT" && git rev-parse HEAD 2>/dev/null || echo "0000000000000000000000000000000000000000") VERSION="0.0.0-dev.0" LDFLAGS="-X 'github.com/azure/azure-dev/cli/azd/internal.Version=${VERSION} (commit ${COMMIT})'" -(cd "$AZD_DIR" && GOOS=linux GOARCH=amd64 go build \ +(cd "$AZD_DIR" && GOOS=linux GOARCH="$GO_ARCH" go build \ -ldflags="$LDFLAGS" \ - -o /tmp/azd-dev-build \ + -o "$TEMP_DIR/azd-dev-build" \ .) -sudo install -m 755 /tmp/azd-dev-build /usr/local/bin/azd -rm -f /tmp/azd-dev-build +sudo install -m 755 "$TEMP_DIR/azd-dev-build" /usr/local/bin/azd echo " ✓ Installed /usr/local/bin/azd" echo "" -# --- Step 2: Ensure microsoft.azd.extensions is available --- +# --- Step 3: Ensure microsoft.azd.extensions is available --- echo "▸ Checking for azd extensions dev kit (microsoft.azd.extensions)..." if azd x version &>/dev/null; then echo " ✓ microsoft.azd.extensions is already installed" else echo " → Installing microsoft.azd.extensions from registry..." - azd extension install microsoft.azd.extensions --no-prompt + azd extension install microsoft.azd.extensions --force --no-prompt echo " ✓ Installed microsoft.azd.extensions" fi echo "" -# --- Step 3: Build extension from source --- -echo "▸ Building azure.ai.agents extension (linux/amd64)..." +# --- Step 4: Build extension from source --- +echo "▸ Building azure.ai.agents extension ($EXPECTED_GO_PLATFORM)..." azd x build -C "$EXTENSION_DIR" echo " ✓ Extension built" echo "" -# --- Step 4: Package as bundle --- +# --- Step 5: Package as bundle --- echo "▸ Packaging extension bundle..." azd x pack --bundle -C "$EXTENSION_DIR" @@ -98,16 +222,13 @@ fi echo " ✓ Bundle created: $BUNDLE_ZIP" echo "" -# --- Step 5: Install from bundle --- +# --- Step 6: Install from bundle --- echo "▸ Installing extension from bundle..." azd extension install "$BUNDLE_ZIP" --force --no-prompt echo " ✓ Extension installed and registered" echo "" -# Clean up the bundle zip -rm -f "$BUNDLE_ZIP" - -# --- Step 6: Verify --- +# --- Step 7: Verify --- echo "▸ Verifying installation..." AZD_VER=$(azd version 2>&1 | head -1)