From 16db9dd88e552258392eee4c3e1204de8edcc81c Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 2 Jun 2026 11:15:29 -0700 Subject: [PATCH 01/13] Add test scenarios for the cli-interactive-tester tool Signed-off-by: trangevi --- .../00-doctor-empty-dir.yaml | 18 ++ .../00-doctor-local-only.yaml | 17 ++ .../00-help-root.yaml | 11 + .../00-init-picker-navigation.yaml | 30 +++ .../00-init-validate-mutually-exclusive.yaml | 20 ++ .../00-init-validate-no-prompt-missing.yaml | 20 ++ .../00-sample-list-json-filters.yaml | 12 ++ .../00-sample-list-text.yaml | 10 + .../00-version.yaml | 10 + .../10-init-deploy-mode-code.yaml | 35 ++++ .../10-init-flags-agent-name-model.yaml | 29 +++ .../10-init-from-code.yaml | 33 +++ .../10-init-from-manifest-url.yaml | 29 +++ .../10-init-template-dotnet.yaml | 31 +++ .../10-init-template-python.yaml | 32 +++ .../20-setup-deploy-shared-agent.yaml | 43 ++++ .../21-show-json.yaml | 20 ++ .../21-show.yaml | 20 ++ .../22-invoke-input-file.yaml | 20 ++ .../22-invoke-new-session.yaml | 20 ++ .../22-invoke-remote.yaml | 20 ++ .../23-sessions-lifecycle.yaml | 23 ++ .../24-files-lifecycle.yaml | 28 +++ .../25-monitor-console.yaml | 21 ++ .../25-monitor-system.yaml | 21 ++ .../26-endpoint-update.yaml | 20 ++ .../27-run-local-and-invoke-local.yaml | 30 +++ .../28-eval-init-run-show.yaml | 23 ++ .../28-eval-update.yaml | 20 ++ .../29-optimize-submit-status.yaml | 21 ++ .../2A-doctor-provisioned-all-pass.yaml | 21 ++ .../2Z-teardown-down.yaml | 22 ++ .../README.md | 196 ++++++++++++++++++ .../fixtures/from-code/app.py | 17 ++ .../fixtures/from-code/requirements.txt | 4 + 35 files changed, 947 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml new file mode 100644 index 00000000000..8c64eec4fe7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -0,0 +1,18 @@ +# Tier 0 (offline) — `doctor` in an empty directory degrades gracefully. +name: "doctor-empty-dir" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-doctor-empty" + +# Guarantee an empty working dir so the "no azd project" path is exercised. +# start_session recreates the dir, so removing it is enough. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-empty" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor in a directory that has no azd project. Wait for the check report to render." + - "Confirm the command reports checks as skipped (or failed) with readable, non-crashing output — it should NOT panic or print a Go stack trace." + - "Note the exit code behavior described in help: 2 when all checks are skipped (preconditions unmet), 1 on failure, 0 when at least one passes." + - "Take a screenshot of the doctor report." + - "Report a finding if doctor crashes, prints a stack trace, or gives an unhelpful/confusing message for a missing project." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml new file mode 100644 index 00000000000..2f8ebd1a374 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -0,0 +1,17 @@ +# Tier 0 (offline) — `doctor --local-only` skips remote checks. +name: "doctor-local-only" +command: "azd ai agent doctor --local-only" +cwd: "~/working/azd-agents-doctor-empty" + +# Guarantee an empty working dir for a deterministic local-only run. +pre: + - run: "rm -rf ~/working/azd-agents-doctor-empty" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run doctor with --local-only. Wait for the report to render." + - "Confirm only local checks are attempted and remote/network-dependent checks are skipped (the report should indicate skipped remote checks)." + - "Confirm the run completes without attempting any network calls and without crashing." + - "Take a screenshot of the report." + - "Report a finding if remote checks still run, or if the output is confusing about what was skipped." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml new file mode 100644 index 00000000000..bd9db3b63bd --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml @@ -0,0 +1,11 @@ +# Tier 0 (offline) — verify root help lists every expected subcommand. +name: "help-root" +command: "azd ai agent --help" +cwd: "/tmp" + +goals: + - "Wait for the help output to render (it includes an ASCII-art banner and a 'Usage:' section)." + - "Confirm the 'Available Commands' list includes: doctor, endpoint, eval, files, init, invoke, monitor, optimize, run, sample, sessions, show, version." + - "Confirm the global flags are listed: --cwd/-C, --debug, --environment/-e, --no-prompt, --output/-o." + - "Take a screenshot of the help output." + - "Report a finding if any listed command is missing, if the usage text is malformed, or if the command exits non-zero." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml new file mode 100644 index 00000000000..1d4958cb13d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -0,0 +1,30 @@ +# Tier 0 (offline) — exercise the `init` interactive picker UX, then abort. +# +# This scenario probes the early interactive prompts only (init mode, language, +# template list). It intentionally ABORTS with Ctrl-C before reaching any Azure / +# Foundry project prompts, so it needs no Azure auth and creates no resources. +name: "init-picker-navigation" +command: "azd ai agent init" +cwd: "~/working/azd-agents-picker" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Clean dir so the abort-before-Azure check can't be confused by prior state. +pre: + - run: "rm -rf ~/working/azd-agents-picker" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the first prompt asking how to initialize (e.g. start from a template / use existing code / use a manifest)." + - "Select 'Start new from a template', then wait for the language prompt." + - "Select a language (Python), then wait for the template list prompt." + - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." + - "Type a string that matches nothing; confirm the list shows an empty/no-match state and does not crash." + - "Clear the filter (backspace) and confirm the full list returns." + - "Press Down many times past the end of the list; confirm selection stays bounded and does not error." + - "If a prompt mentions a '?' hint, press '?' and confirm a helpful hint is shown." + - "Take a screenshot at each interesting state." + - "Press Ctrl-C (or Escape at the top-level prompt) to abort. Confirm it exits cleanly without a stack trace and without leaving a half-written azure.yaml." + - "Report a finding for any picker glitch: filter not recovering, crash on no-match, unbounded scrolling, unhelpful hints, or a messy/abrupt abort." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml new file mode 100644 index 00000000000..1ce21ce14d8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -0,0 +1,20 @@ +# Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. +name: "init-validate-mutually-exclusive" +command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" +cwd: "~/working/azd-agents-validate" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so leftover files can't affect the validation result. +pre: + - run: "rm -rf ~/working/azd-agents-validate" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the command to fail fast (it should not start the interactive wizard)." + - "Confirm the error message clearly states that --from-code and --manifest/-m are mutually exclusive." + - "Confirm the process exits non-zero and does NOT create or modify any project files in the directory." + - "Take a screenshot of the error output." + - "Report a finding if the flags are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml new file mode 100644 index 00000000000..7843e58c31d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -0,0 +1,20 @@ +# Tier 0 (offline) — `init --no-prompt` with no resolvable inputs fails helpfully. +name: "init-validate-no-prompt-missing" +command: "azd ai agent init --no-prompt" +cwd: "~/working/azd-agents-validate-noprompt" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Empty dir is a precondition: no existing code/manifest to resolve from. +pre: + - run: "rm -rf ~/working/azd-agents-validate-noprompt" + cwd: "~/working" + name: "reset to an empty working dir" + +goals: + - "Run init in --no-prompt mode in an empty directory with no flags and no existing code/manifest." + - "Confirm the command does NOT hang waiting for input (no-prompt must never block on a prompt)." + - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --from-code, --manifest, or --project-id)." + - "Take a screenshot of the error output." + - "Report a finding if it hangs, prompts interactively despite --no-prompt, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml new file mode 100644 index 00000000000..a3679f6ec8b --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — `sample list` JSON output and filter flags. +name: "sample-list-json-filters" +command: "bash" +cwd: "/tmp" + +goals: + - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." + - "Run: azd ai agent sample list --language python --output json. Confirm results are filtered to python samples only." + - "Run: azd ai agent sample list --type agent --output json. Confirm results only include agent-type templates." + - "Run: azd ai agent sample list --featured-only --output json. Confirm only featured samples are returned (a subset of the full list)." + - "Take a screenshot after each command." + - "Report a finding if any command produces invalid JSON, ignores its filter, or errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml new file mode 100644 index 00000000000..51b05c29be6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -0,0 +1,10 @@ +# Tier 0 (offline) — `sample list` renders the human-readable catalog. +name: "sample-list-text" +command: "azd ai agent sample list" +cwd: "/tmp" + +goals: + - "Wait for the curated sample catalog to render as human-readable text." + - "Confirm at least one sample entry is shown with a name/title and a manifest or repo reference usable with 'azd ai agent init -m' or 'azd init -t'." + - "Take a screenshot of the catalog output." + - "Report a finding if the list is empty, truncated, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml new file mode 100644 index 00000000000..a58a88b0076 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml @@ -0,0 +1,10 @@ +# Tier 0 (offline) — verify `azd ai agent version` prints a version string. +name: "version" +command: "azd ai agent version" +cwd: "/tmp" + +goals: + - "Wait for the command to print a version string (e.g. a value like '0.1.x-preview' or a commit-based 'vdev' build identifier)." + - "Confirm the process exits cleanly without an error or stack trace." + - "Take a screenshot of the final output." + - "Report a finding if no version is printed, if the output is empty, or if the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml new file mode 100644 index 00000000000..647a7b34675 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -0,0 +1,35 @@ +# Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Targets the --deploy-mode code path which prompts for entry-point and runtime +# (instead of building a container image). +name: "init-deploy-mode-code" +command: "azd ai agent init --from-code --deploy-mode code" +cwd: "~/working/azd-agents-t1-code-deploy" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Seed a committed Python fixture so code-deploy has real source to package. +# Override the fixture location with AZD_AGENTS_FIXTURES if needed. +pre: + - run: "rm -rf ~/working/azd-agents-t1-code-deploy" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." + - "If an existing agent manifest is detected, confirm reuse." + - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." + - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the entry-point or runtime prompts are missing, confusing, or not persisted." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml new file mode 100644 index 00000000000..36f15bdaf00 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -0,0 +1,29 @@ +# Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Verifies that the override flags are honored in the generated files. +name: "init-flags-agent-name-model" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name qa-named-agent --model gpt-4.1-mini" +cwd: "~/working/azd-agents-t1-flags" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-flags" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the manifest to download and parse." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "Accept any remaining model defaults (version, SKU, capacity, deployment name)." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name 'qa-named-agent' and the model 'gpt-4.1-mini' (the values passed via flags, not the manifest defaults)." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml new file mode 100644 index 00000000000..cebe8f71fc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -0,0 +1,33 @@ +# Tier 1 (auth, scaffold only) — init from existing code in the current directory. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Precondition: the cwd should already contain agent source code (and ideally an +# agent manifest). The pre hooks seed a committed Python fixture so this is +# guaranteed and the run is idempotent. Override the fixture location with +# AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. +name: "init-from-code" +command: "azd ai agent init --from-code" +cwd: "~/working/azd-agents-t1-from-code" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +pre: + - run: "rm -rf ~/working/azd-agents-t1-from-code" + cwd: "~/working" + name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-t1-from-code && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code/" + cwd: "~/working" + name: "seed from-code agent fixture (app.py + requirements.txt)" + +goals: + - "Wait for the tool to inspect the current directory and treat its code as the agent source." + - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "Wait for initialization to complete — look for 'Next:' in the output." + - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if code-detection misbehaves or if the manifest-reuse prompt is confusing." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml new file mode 100644 index 00000000000..7f790eb49a7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -0,0 +1,29 @@ +# Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +name: "init-from-manifest-url" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" +cwd: "~/working/azd-agents-t1-manifest" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-manifest" + cwd: "~/working" + name: "reset working dir" + +goals: + - "Wait for the tool to fetch and parse the manifest from the provided URL." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity, deployment name)." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the manifest fails to download/parse, or if any field is dropped during scaffold." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml new file mode 100644 index 00000000000..a1f2c0487b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -0,0 +1,31 @@ +# Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. +# +# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +name: "init-template-dotnet" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-dotnet" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-dotnet" + cwd: "~/working" + name: "reset working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select C# / .NET as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding if the .NET language path behaves differently from Python in any confusing way." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml new file mode 100644 index 00000000000..d3614385d6f --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -0,0 +1,32 @@ +# Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. +# +# Requires `azd auth login`. Reads subscriptions/Foundry projects but does NOT +# run `azd provision`, so no resources are created and no cost is incurred. +name: "init-template-python" +command: "azd ai agent init" +cwd: "~/working/azd-agents-t1-python" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +pre: + - run: "rm -rf ~/working/azd-agents-t1-python" + cwd: "~/working" + name: "reset working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Pick the first starter template in the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." + - "Take a screenshot of the completed init output." + - "STOP here — do NOT run 'azd provision'. Report a finding for any confusing prompt, wrong default, or scaffold problem." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml new file mode 100644 index 00000000000..4ceb20913d4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -0,0 +1,43 @@ +# Tier 2 (cloud E2E) — SETUP: deploy the shared agent used by all 21-..2A scenarios. +# +# ⚠️ Incurs Azure cost. Run this FIRST. The deployed agent lives in the shared +# working directory ~/working/azd-agents-shared and is reused by the targeted +# scenarios. Run 2Z-teardown-down.yaml LAST to clean up. +name: "setup-deploy-shared-agent" +command: "azd ai agent init" +cwd: "~/working/azd-agents-shared" + +env: + AZD_DISABLE_AGENT_DETECT: "1" + +# Idempotent setup: if a previous run left a project here, tear down its Azure +# resources FIRST (so we never orphan them), then clear the dir for a fresh +# deploy. The down step gets a long timeout and continues on error (there may be +# nothing to tear down). Re-using a clean path also avoids the resource-name hash +# collision in issue #8360. +pre: + - run: "if [ -f ~/working/azd-agents-shared/azure.yaml ]; then (cd ~/working/azd-agents-shared && azd down --force --purge); fi" + cwd: "~/working/azd-agents-shared" + name: "tear down any leftover deployed agent" + continue_on_error: true + timeout: 900 + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + +goals: + - "When asked how to initialize, select 'Start new from a template'." + - "Select Python as the language." + - "Select the 'Basic Responses' template from the list." + - "When asked how to deploy, select 'Container' (hosted agent)." + - "If asked to select an Azure AI Foundry project, create a new one and follow the prompts." + - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." + - "If asked for a location/region, select 'East US 2'." + - "When asked to select a model, choose 'gpt-4.1-mini'." + - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "If asked for container/resource size, select 'Small'." + - "Wait for initialization to complete — look for 'Next:' or a success message." + - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." + - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." + - "Take a screenshot of the successful provision and 'show' output." + - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml new file mode 100644 index 00000000000..0f9dc491c3a --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `show --output json` returns well-formed JSON. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show-json" +command: "azd ai agent show --output json" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the command to print JSON." + - "Confirm the output is valid JSON and includes the agent name, version, status, and endpoint fields." + - "Confirm the JSON values match what the table form of 'show' reports." + - "Take a screenshot of the JSON output." + - "Report a finding if the JSON is malformed, missing expected fields, or inconsistent with the table output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml new file mode 100644 index 00000000000..fb06d2ff454 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `show` reports the deployed agent (table output). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "show" +command: "azd ai agent show" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the status table to render." + - "Confirm the agent name, version, and a status field (e.g. 'active') are shown, auto-resolved from azure.yaml and the current azd environment." + - "Confirm the endpoint URL for the deployed agent is displayed." + - "Take a screenshot of the status table." + - "Report a finding if the agent cannot be resolved, the status looks wrong, or fields are missing/blank." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml new file mode 100644 index 00000000000..79ab211dbcb --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke -f ` sends a file as the request body. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-input-file" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Create a small request file, e.g.: printf '%s' 'Summarize the benefits of unit testing in one sentence.' > request.txt" + - "Run: azd ai agent invoke -f request.txt and wait for the response." + - "Confirm the agent responds to the contents of the file (not to an empty or literal-path message)." + - "Take a screenshot of the response." + - "Report a finding if the file body is ignored, mis-read, or if -f errors on a valid file." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml new file mode 100644 index 00000000000..ce93cf77620 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke --new-session` resets conversation state. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-new-session" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent invoke \"My name is Quinn. Remember it.\" and wait for a response." + - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session reused the same conversation)." + - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent does NOT recall 'Quinn' — the new session discarded prior history." + - "Take a screenshot after each invoke." + - "Report a finding if --new-session still carries over prior conversation context, or if session persistence between the first two invokes does not work." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml new file mode 100644 index 00000000000..9fedf175a0c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `invoke` a deployed agent remotely on Foundry. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "invoke-remote" +command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the remote invocation to complete (the agent is auto-detected from azure.yaml)." + - "Confirm a non-empty model response is returned (not an error)." + - "Confirm the command does NOT 404 and the response references no auth/endpoint errors." + - "Take a screenshot of the response." + - "Report a finding if invoke returns a 404, an auth error, an empty response, or hangs past a reasonable timeout." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml new file mode 100644 index 00000000000..f559df8cc08 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — `sessions` lifecycle: create, list, show, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `sessions` command group end-to-end in one run. +name: "sessions-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent sessions create. Confirm a new session is created and note its session ID." + - "Run: azd ai agent sessions list --output table. Confirm the newly created session appears in the list." + - "Run: azd ai agent sessions show using the ID from create. Confirm session details (status, version) are shown." + - "Run: azd ai agent sessions delete . Confirm the session is deleted (synchronously)." + - "Run: azd ai agent sessions list again and confirm the deleted session no longer appears." + - "Take a screenshot after each step." + - "Report a finding if any subcommand errors, if the created session is missing from list, or if delete does not remove it." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml new file mode 100644 index 00000000000..4356ab7a54d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -0,0 +1,28 @@ +# Tier 2 (cloud E2E) — `files` lifecycle: upload, list, stat, mkdir, download, delete. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# Targets the `files` command group end-to-end against the last invoke session. +# Note: file operations target a session — run an invoke first if no session exists. +name: "files-lifecycle" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure a session exists: run 'azd ai agent invoke \"hi\"' first so files commands have a session to target (or pass --session-id consistently)." + - "Create a local test file: printf 'col1,col2\\n1,2\\n' > sample.csv" + - "Run: azd ai agent files upload ./sample.csv --target-path /data/sample.csv. Confirm success." + - "Run: azd ai agent files list /data --output table. Confirm sample.csv is listed." + - "Run: azd ai agent files stat /data/sample.csv. Confirm metadata (size/type) is returned." + - "Run: azd ai agent files mkdir /data/output. Confirm the directory is created." + - "Run: azd ai agent files download /data/sample.csv --target-path ./downloaded.csv. Confirm the file downloads and its contents match the original." + - "Run: azd ai agent files delete /data/output --recursive and azd ai agent files delete /data/sample.csv. Confirm both are removed." + - "Run: azd ai agent files list /data and confirm the deleted entries are gone." + - "Take a screenshot after each step." + - "Report a finding if any subcommand errors, if upload/download corrupts the file, or if a session cannot be resolved." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml new file mode 100644 index 00000000000..d3288f76781 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `monitor` console logs for the last session. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists to stream logs from. +name: "monitor-console" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Ensure recent activity: run 'azd ai agent invoke \"generate some output\"' so the session has console logs." + - "Run: azd ai agent monitor --tail 50. Confirm recent console (stdout/stderr) log lines are fetched and printed, then the command exits (no --follow)." + - "Confirm timestamps are shown in local time by default and the output is readable." + - "Take a screenshot of the log output." + - "Report a finding if no session can be auto-resolved, if logs are empty despite recent activity, or if the output is garbled." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml new file mode 100644 index 00000000000..08c182ab1d3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `monitor --type system` container/system events. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully, +# and at least one invoke has happened so a session exists. +name: "monitor-system" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent monitor --type system --tail 50. Confirm system/container event logs are fetched and printed." + - "Confirm the output is distinct from console logs (system events such as container lifecycle, not stdout/stderr)." + - "Optionally run: azd ai agent monitor --type system --follow for a few seconds to confirm streaming works, then stop it with Ctrl-C and confirm it exits cleanly." + - "Take a screenshot of the event output." + - "Report a finding if system logs are empty, indistinguishable from console logs, or if --follow does not stream / does not stop cleanly." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml new file mode 100644 index 00000000000..8cc712f582c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `endpoint update` patches endpoint/card without a new version. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "endpoint-update" +command: "azd ai agent endpoint update" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run endpoint update for the default (auto-detected) agent service." + - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." + - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." + - "Take a screenshot of the update result and the post-update 'show' output." + - "Report a finding if endpoint update creates a new version, errors against an already-deployed agent, or gives a confusing result." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml new file mode 100644 index 00000000000..f27e7b4267c --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -0,0 +1,30 @@ +# Tier 2 (cloud E2E) — run the agent locally and invoke it via --local. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run (project scaffolded). +# `azd ai agent run` blocks the terminal, so this needs TWO sessions: one runs the +# agent, a second invokes it with --local. +name: "run-local-and-invoke-local" +command: "azd ai agent run --no-inspector" +cwd: "~/working/azd-agents-shared" + +notes: | + Use two tester sessions: + - session_id "run": runs 'azd ai agent run --no-inspector' (this 'command'). + - session_id "invoke": a bash session for 'azd ai agent invoke --local'. + The agent listens on port 8088 by default. Stop the run session with Ctrl-C at the end. + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "In the run session, wait until the local agent reports it is listening (look for a port 8088 / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." + - "Start a SECOND session (session_id 'invoke') running bash in the same cwd." + - "In the invoke session, run: azd ai agent invoke --local \"Hello from local!\" and wait for a response from the locally running agent." + - "Confirm the response comes from the local process (not Foundry) and is non-empty." + - "Take a screenshot of both the run session (showing it listening) and the invoke response." + - "Stop the run session with Ctrl-C and confirm the local agent shuts down cleanly." + - "Report a finding if the local server fails to start, if --local invoke cannot reach it, or if shutdown is messy." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml new file mode 100644 index 00000000000..df9dfcd33ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml @@ -0,0 +1,23 @@ +# Tier 2 (cloud E2E) — `eval` flow: init a suite, run it, then list/show results. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully +# (a deployed agent is required to generate and run evals). +name: "eval-init-run-show" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --eval-model gpt-4.1-mini --max-samples 15 and wait for it to generate an eval config and dataset." + - "Confirm an eval.yaml (and dataset) is written to the project root." + - "Run: azd ai agent eval run and wait for the evaluation run to complete." + - "Run: azd ai agent eval list and confirm the run appears." + - "Run: azd ai agent eval show and confirm run details / scores are displayed." + - "Take a screenshot after init, run, and show." + - "Report a finding if generation fails, eval.yaml is malformed, the run errors, or results are missing/unreadable." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml new file mode 100644 index 00000000000..7101c392f82 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml @@ -0,0 +1,20 @@ +# Tier 2 (cloud E2E) — `eval update` refreshes dataset and evaluators. +# +# Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. +name: "eval-update" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Confirm an eval.yaml from a prior 'eval init' exists in the project root (run 28-eval-init-run-show.yaml first if not)." + - "Run: azd ai agent eval update --dataset-only and confirm only the dataset is updated (evaluators left unchanged)." + - "Run: azd ai agent eval update --evaluator-only and confirm only the evaluators are updated (dataset left unchanged)." + - "Take a screenshot after each update." + - "Report a finding if --dataset-only or --evaluator-only updates the wrong thing, errors, or prompts despite the scoping flag." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml new file mode 100644 index 00000000000..1c53ded1649 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `optimize` submit (no-wait) then check status/list. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "optimize-submit-status" +command: "bash" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run: azd ai agent optimize --eval-model gpt-4.1-mini --max-iterations 1 --no-wait (agent auto-detected from the azd project). Provide any other values it prompts for using sensible defaults." + - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." + - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." + - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." + - "Take a screenshot after submit, status, and list." + - "Report a finding if submission fails, the job ID is not surfaced, or status/list cannot find the job. (Optional: cancel with 'azd ai agent optimize cancel ' to avoid leaving it running.)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml new file mode 100644 index 00000000000..cffe0288a63 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `doctor` against a fully provisioned project (checks pass). +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "doctor-provisioned-all-pass" +command: "azd ai agent doctor" +cwd: "~/working/azd-agents-shared" + +# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +pre: + - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run doctor against the provisioned project. Wait for the full check suite (local + remote) to render." + - "Confirm the local checks pass (project, azure.yaml, agent service config)." + - "Confirm the remote checks pass (deployed agent reachable / active)." + - "Confirm doctor suggests a sensible next command (e.g. invoke or run) and exits 0 (at least one check passed, none failed)." + - "Take a screenshot of the doctor report." + - "Report a finding if any check fails or is skipped unexpectedly for a healthy provisioned project, or if the suggested next step is wrong." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml new file mode 100644 index 00000000000..bcad6af5fc7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — TEARDOWN: destroy all resources created by the setup scenario. +# +# ⚠️ Run this LAST, after all 21-..2A targeted scenarios are done. Cleans up the +# shared agent and its Azure resources to stop incurring cost. +name: "teardown-down" +command: "azd down --force --purge" +cwd: "~/working/azd-agents-shared" + +goals: + - "Run 'azd down --force --purge' in the shared project directory." + - "Wait for the full teardown to complete — all resources (Foundry account/project, model deployment, container resources) should be deleted and purged." + - "Confirm the command reports success and no resources are left behind." + - "Take a screenshot of the completed teardown." + - "Report a finding if teardown fails, leaves orphaned resources, or hangs." + +# After the in-session teardown, clear the shared working dir so the next full +# Tier 2 pass starts clean. continue_on_error so a missing dir is not fatal. +post: + - run: "rm -rf ~/working/azd-agents-shared" + cwd: "~/working" + name: "clear the shared working dir" + continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md new file mode 100644 index 00000000000..895a892d7db --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -0,0 +1,196 @@ + +# `azd ai agent` — cli-interactive-tester scenarios + +Goal-based scenarios for driving the `azure.ai.agents` extension through the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server. Each file targets **one** command or flow at a time and uses the +strict `goals:` list format so the run is repeatable and reviewable. + +## How to run + +Register the cli-interactive-tester MCP server (see its README), then ask +Copilot CLI to load a scenario and accomplish its goals, e.g.: + +``` +Use the cli-interactive-tester to load the scenario at +tests/cli-interactive-tester-scenarios/00-version.yaml. If it declares pre hooks, +run them first; then start the session, accomplish the goals, take screenshots at +each step, and run any post hooks after finishing. +``` + +Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting +the working dir or seeding a fixture), and a few declare **`post:` hooks** +(cleanup). The agent must invoke them via the tester's `run_pre_hooks` / +`run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. +See [Pre/post hooks](#prepost-hooks) below. + +## Paths run inside WSL (on Windows) + +The cli-interactive-tester drives CLIs through **tmux**, which on Windows runs +inside **WSL**. The scenario YAML files live on the Windows filesystem (in this +repo), but every `cwd` value is resolved against the **WSL filesystem** where the +command actually executes: + +- `~/working/azd-agents-shared` → `/home//working/azd-agents-shared` +- `/tmp` → WSL's `/tmp` + +Implications: + +- `azd` and the `azure.ai.agents` extension must be installed **inside WSL**, + since that is where the scenario commands run. +- `cwd` directories do not need to pre-exist — the tester creates them if missing. +- The `cwd` convention is three-way by design: ephemeral `/tmp` for read-only + scenarios that touch no project (`version`, `--help`, `sample list`); a unique + `~/working/azd-agents-*` dir per `init`/`doctor` scenario for isolation; and a + single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they + operate on the same deployed agent. + +On macOS/Linux these are simply native paths (no WSL involved). + +## Tiers + +Scenarios are organized into three tiers by cost and prerequisites. + +### Tier 0 — Offline (prefix `00-`) +No Azure auth, no network resource creation. Fast and deterministic. Safe to run +in any order, any time. + +| File | Targets | +|------|---------| +| `00-version.yaml` | `version` | +| `00-help-root.yaml` | root help / command discovery | +| `00-sample-list-text.yaml` | `sample list` (text) | +| `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | +| `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | +| `00-doctor-local-only.yaml` | `doctor --local-only` | +| `00-init-validate-mutually-exclusive.yaml` | `init` flag validation (`--from-code` + `-m`) | +| `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | +| `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | + +### Tier 1 — Auth, scaffold only (prefix `10-`) +Requires `azd auth login` (reads subscriptions/Foundry projects) but **does not +provision** any resources and incurs no cost. Each completes a project scaffold +and verifies the generated files, then stops before `azd provision`. + +| File | Targets | +|------|---------| +| `10-init-template-python.yaml` | `init` new-from-template, Python | +| `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | +| `10-init-from-manifest-url.yaml` | `init -m ` | +| `10-init-from-code.yaml` | `init --from-code` | +| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` | +| `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | + +### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost +Provisions real resources. **Run order matters:** + +1. `20-setup-deploy-shared-agent.yaml` **first** — deploys the shared agent. +2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). +3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. + +All Tier 2 scenarios share one working directory (`~/working/azd-agents-shared`) +so they operate on the same deployed agent. + +| File | Targets | +|------|---------| +| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` (SETUP) | +| `21-show.yaml` | `show` (table) | +| `21-show-json.yaml` | `show --output json` | +| `22-invoke-remote.yaml` | `invoke` (remote) | +| `22-invoke-new-session.yaml` | `invoke --new-session` | +| `22-invoke-input-file.yaml` | `invoke -f ` | +| `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | +| `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | +| `25-monitor-console.yaml` | `monitor` (console) | +| `25-monitor-system.yaml` | `monitor --type system` | +| `26-endpoint-update.yaml` | `endpoint update` | +| `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | +| `28-eval-init-run-show.yaml` | `eval init/run/list/show` | +| `28-eval-update.yaml` | `eval update` | +| `29-optimize-submit-status.yaml` | `optimize` + `optimize status/list` | +| `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | +| `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | + +## Conventions + +- **Subscription**: `azd ai agent development` +- **Region**: `East US 2` +- **Model**: `gpt-4.1-mini` (cheap/fast for testing) +- `command:` invokes the installed extension as `azd ai agent …`. +- Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent + auto-detection prompts. +- Every scenario asks the driver to screenshot key steps and file a finding + (`report_finding`) for any confusing UX, error, or doc mismatch. + +## Pre/post hooks + +Scenarios use the tester's **`pre:`** and **`post:`** hook lists for host-side +setup and cleanup. Hooks run on the host (inside WSL on Windows), outside the +tmux session, **sequentially and fail-fast** unless a hook sets +`continue_on_error: true`. Each entry is a string or a mapping with `run` +(required), `cwd` (defaults to the scenario `cwd`, created if missing), `env`, +`continue_on_error` (default `false`), `timeout` (default **120s**), and `name`. + +How they're used here: + +- **`pre` reset** — stateful Tier 0/1 scenarios `rm -rf` their own working dir so + re-runs start clean. (`start_session` recreates the dir, so removing it is + enough; the doctor/init scenarios just need an empty dir.) +- **`pre` fixture seed** — the `--from-code` scenarios + (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python + fixture into the dir so the source exists before `init --from-code` inspects it + (see [Fixtures](#fixtures)). +- **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs + `azd down --force --purge` if a leftover project exists in the shared dir (so it + never orphans live Azure resources), then clears the dir. The down hook uses + `timeout: 900` and `continue_on_error: true`. +- **`pre` precondition guard (Tier 2 reuse)** — `21-…2A` print a clear "run + 20-setup first" warning if the shared agent isn't deployed (non-fatal). +- **`post` cleanup** — `2Z-teardown-down` clears the shared working dir after the + in-session `azd down` completes. + +## Fixtures + +`fixtures/from-code/` holds a minimal Python agent source tree (`app.py` + +`requirements.txt`) that satisfies the extension's `--from-code` detection +(it looks for `requirements.txt` or any `.py`, and defaults the entry point to +`app.py`). The from-code scenarios copy it into the working dir via a `pre` hook. + +The hook references the fixture by absolute path with an overridable env var: + +```sh +cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/." "$cwd" +``` + +If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view +of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this +`fixtures/` directory before running the from-code scenarios. + +## Re-running scenarios (idempotency) + +Idempotency is handled **per scenario** via `pre`/`post` hooks rather than a +separate reset step — every scenario that holds state resets itself, so they can +be run back to back in any order within a tier: + +- Tier 0/1 stateful scenarios **pre-wipe** their own `cwd`. Cleanup is pre-wipe + **only** (no `post` delete), so the generated scaffold stays on disk for + inspection after a run while the next run still starts clean. +- The shared Tier 2 dir is reset by `20-setup`'s `pre` hook, which **downs any + leftover deployed project first** to avoid orphaning live Azure resources (this + also sidesteps the resource-name hash collision behind + [#8360](https://github.com/Azure/azure-dev/issues/8360)). `2Z-teardown-down` + additionally clears the dir in a `post` hook. +- Read-only scenarios (`version`, `--help`, `sample list`) run in `/tmp`, hold no + state, and declare no hooks. + +> If a Tier 2 run is interrupted before `2Z-teardown`, just re-run +> `20-setup-deploy-shared-agent` — its `pre` hook downs any live project in the +> shared dir before redeploying, so resources won't be orphaned. + +## Notes + +- `files` and `sessions` are exercised as one lifecycle scenario per command + group (rather than one file per subcommand) to avoid cross-scenario ordering + dependencies — still one command at a time. +- `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` + uses two sessions (one to run, one to invoke `--local`). diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py new file mode 100644 index 00000000000..56b577b6277 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py @@ -0,0 +1,17 @@ +"""Minimal agent source fixture for `azd ai agent init --from-code` scenarios. + +This file exists only so the extension's from-code detection treats the working +directory as a Python agent project (it looks for requirements.txt or any .py +file, and uses app.py as the default entry point). The init flow scaffolds an +agent.yaml around this code; the body does not need to be a fully functional +agent for the scaffold-only Tier 1 scenarios. +""" + + +def handler(request: str) -> str: + """Echo the incoming request back to the caller.""" + return f"echo: {request}" + + +if __name__ == "__main__": + print(handler("hello")) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt new file mode 100644 index 00000000000..d94c3d032d6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt @@ -0,0 +1,4 @@ +# Minimal dependency set so `azd ai agent init --from-code` detects a Python +# project. Keep this lightweight — the Tier 1 from-code scenarios only scaffold +# and do not install or run the agent. +azure-ai-projects From 4fbaa67e4bc714125e792205230aa76424d1251a Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 2 Jun 2026 14:14:40 -0700 Subject: [PATCH 02/13] Some scenario edits Signed-off-by: trangevi --- .../10-init-deploy-mode-code.yaml | 5 ++- .../10-init-flags-agent-name-model.yaml | 11 +++--- .../10-init-from-code.yaml | 5 ++- .../10-init-from-manifest-url.yaml | 7 ++-- .../10-init-template-dotnet.yaml | 7 ++-- .../10-init-template-python.yaml | 7 ++-- .../20-setup-deploy-shared-agent.yaml | 5 ++- .../README.md | 38 ++++++++++++++++++- 8 files changed, 63 insertions(+), 22 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 647a7b34675..9b7d1d0fbf4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime # (instead of building a container image). name: "init-deploy-mode-code" @@ -21,11 +21,12 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 36f15bdaf00..df3792cd44a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,9 +1,9 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name qa-named-agent --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent --model gpt-4.1-mini" cwd: "~/working/azd-agents-t1-flags" env: @@ -16,14 +16,15 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - - "Accept any remaining model defaults (version, SKU, capacity, deployment name)." + - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name 'qa-named-agent' and the model 'gpt-4.1-mini' (the values passed via flags, not the manifest defaults)." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index cebe8f71fc3..4f8f4fd9db7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from existing code in the current directory. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Precondition: the cwd should already contain agent source code (and ideally an # agent manifest). The pre hooks seed a committed Python fixture so this is # guaranteed and the run is idempotent. Override the fixture location with @@ -21,9 +21,10 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 7f790eb49a7..95c9a84d0d2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest" @@ -15,13 +15,14 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." - - "Accept the defaults for any remaining model prompts (version, SKU, capacity, deployment name)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index a1f2c0487b3..901f0c53670 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. # -# Requires `azd auth login`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet" @@ -15,15 +15,16 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index d3614385d6f..c3a035828c5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -1,6 +1,6 @@ # Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. # -# Requires `azd auth login`. Reads subscriptions/Foundry projects but does NOT +# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Reads subscriptions/Foundry projects but does NOT # run `azd provision`, so no resources are created and no cost is incurred. name: "init-template-python" command: "azd ai agent init" @@ -16,15 +16,16 @@ pre: name: "reset working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 4ceb20913d4..f18400d226e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -26,15 +26,16 @@ pre: name: "clear the shared working dir" goals: + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, create a new one and follow the prompts." + - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - "If asked for a location/region, select 'East US 2'." - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity, and deployment name." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 895a892d7db..a609d8d4b23 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -1,4 +1,4 @@ - + # `azd ai agent` — cli-interactive-tester scenarios Goal-based scenarios for driving the `azure.ai.agents` extension through the @@ -47,6 +47,34 @@ Implications: On macOS/Linux these are simply native paths (no WSL involved). +## Authentication + +Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in +manually before** starting a run. The scenarios do **not** perform login +themselves, and the test-driving agent **cannot** complete it either: `az login` +opens a **separate browser window** for account selection that requires +human interaction outside the terminal the agent controls. Treat auth as a +one-time manual prerequisite, not a scenario step. + +Inside WSL, a human runs: + +``` +az login --tenant azdaiagent.onmicrosoft.com +``` + +This opens the interactive sign-in flow and then: + +1. **Browser account selection** — a separate browser window opens; the human + picks the account in the `azdaiagent.onmicrosoft.com` tenant. (The agent + cannot do this.) +2. **Subscription selection** — back in the terminal, select the + `azd ai agent development` subscription. + +Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL +session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of +them reuse that session credential. + + ## Tiers Scenarios are organized into three tiers by cost and prerequisites. @@ -68,7 +96,7 @@ in any order, any time. | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | ### Tier 1 — Auth, scaffold only (prefix `10-`) -Requires `azd auth login` (reads subscriptions/Foundry projects) but **does not +Requires Azure login (reads subscriptions/Foundry projects) but **does not provision** any resources and incurs no cost. Each completes a project scaffold and verifies the generated files, then stops before `azd provision`. @@ -116,6 +144,12 @@ so they operate on the same deployed agent. - **Subscription**: `azd ai agent development` - **Region**: `East US 2` - **Model**: `gpt-4.1-mini` (cheap/fast for testing) +- **Resource name prefix**: every newly created Azure resource (Foundry + project/account, azd environment, agent, model deployment, resource group) is + named with a `trangevi-` prefix so test resources are easy to identify and + clean up. Note that some fields lowercase the value and replace invalid + characters with hyphens — that normalization is expected (see + `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. From 61fc535a1bd21a3cd7e4aa27377e20c64400622e Mon Sep 17 00:00:00 2001 From: trangevi Date: Wed, 3 Jun 2026 09:41:48 -0700 Subject: [PATCH 03/13] Picking up recent tester tool updates Signed-off-by: trangevi --- .../.gitignore | 2 + .../00-doctor-empty-dir.yaml | 4 +- .../00-doctor-local-only.yaml | 4 +- .../00-init-picker-navigation.yaml | 4 +- .../00-init-validate-mutually-exclusive.yaml | 4 +- .../00-init-validate-no-prompt-missing.yaml | 4 +- .../10-init-deploy-mode-code.yaml | 8 +-- .../10-init-flags-agent-name-model.yaml | 10 +-- .../10-init-from-code.yaml | 8 +-- .../10-init-from-manifest-url.yaml | 6 +- .../10-init-template-dotnet.yaml | 6 +- .../10-init-template-python.yaml | 6 +- .../27-run-local-and-invoke-local.yaml | 26 ++++--- .../README.md | 68 +++++++++++++++++-- 14 files changed, 112 insertions(+), 48 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore new file mode 100644 index 00000000000..7fde892bf70 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore @@ -0,0 +1,2 @@ +# cli-interactive-tester run artifacts (screenshots, HTML reports, scrollback) +.reports/ diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml index 8c64eec4fe7..53aba1a8dbc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -1,12 +1,12 @@ # Tier 0 (offline) — `doctor` in an empty directory degrades gracefully. name: "doctor-empty-dir" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-doctor-empty" +cwd: "~/working/azd-agents-doctor-empty-{instance}" # Guarantee an empty working dir so the "no azd project" path is exercised. # start_session recreates the dir, so removing it is enough. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty" + - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 2f8ebd1a374..7c8662e6e3c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -1,11 +1,11 @@ # Tier 0 (offline) — `doctor --local-only` skips remote checks. name: "doctor-local-only" command: "azd ai agent doctor --local-only" -cwd: "~/working/azd-agents-doctor-empty" +cwd: "~/working/azd-agents-doctor-empty-{instance}" # Guarantee an empty working dir for a deterministic local-only run. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty" + - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index 1d4958cb13d..bbf2c0c7fd5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -5,14 +5,14 @@ # Foundry project prompts, so it needs no Azure auth and creates no resources. name: "init-picker-navigation" command: "azd ai agent init" -cwd: "~/working/azd-agents-picker" +cwd: "~/working/azd-agents-picker-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Clean dir so the abort-before-Azure check can't be confused by prior state. pre: - - run: "rm -rf ~/working/azd-agents-picker" + - run: "rm -rf ~/working/azd-agents-picker-{instance}" cwd: "~/working" name: "reset working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 1ce21ce14d8..6ee69d98636 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -1,14 +1,14 @@ # Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. name: "init-validate-mutually-exclusive" command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" -cwd: "~/working/azd-agents-validate" +cwd: "~/working/azd-agents-validate-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so leftover files can't affect the validation result. pre: - - run: "rm -rf ~/working/azd-agents-validate" + - run: "rm -rf ~/working/azd-agents-validate-{instance}" cwd: "~/working" name: "reset working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index 7843e58c31d..ac31b1f2127 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -1,14 +1,14 @@ # Tier 0 (offline) — `init --no-prompt` with no resolvable inputs fails helpfully. name: "init-validate-no-prompt-missing" command: "azd ai agent init --no-prompt" -cwd: "~/working/azd-agents-validate-noprompt" +cwd: "~/working/azd-agents-validate-noprompt-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Empty dir is a precondition: no existing code/manifest to resolve from. pre: - - run: "rm -rf ~/working/azd-agents-validate-noprompt" + - run: "rm -rf ~/working/azd-agents-validate-noprompt-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 9b7d1d0fbf4..89260d57ba8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -5,7 +5,7 @@ # (instead of building a container image). name: "init-deploy-mode-code" command: "azd ai agent init --from-code --deploy-mode code" -cwd: "~/working/azd-agents-t1-code-deploy" +cwd: "~/working/azd-agents-t1-code-deploy-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" @@ -13,15 +13,15 @@ env: # Seed a committed Python fixture so code-deploy has real source to package. # Override the fixture location with AZD_AGENTS_FIXTURES if needed. pre: - - run: "rm -rf ~/working/azd-agents-t1-code-deploy" + - run: "rm -rf ~/working/azd-agents-t1-code-deploy-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-code-deploy && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy/" + - run: "mkdir -p ~/working/azd-agents-t1-code-deploy-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-code-deploy-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index df3792cd44a..5c500ef9fdc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -3,20 +3,20 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent --model gpt-4.1-mini" -cwd: "~/working/azd-agents-t1-flags" +command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +cwd: "~/working/azd-agents-t1-flags-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-flags" + - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." @@ -25,6 +25,6 @@ goals: - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 4f8f4fd9db7..01026e394da 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -7,21 +7,21 @@ # AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. name: "init-from-code" command: "azd ai agent init --from-code" -cwd: "~/working/azd-agents-t1-from-code" +cwd: "~/working/azd-agents-t1-from-code-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" pre: - - run: "rm -rf ~/working/azd-agents-t1-from-code" + - run: "rm -rf ~/working/azd-agents-t1-from-code-{instance}" cwd: "~/working" name: "reset working dir" - - run: "mkdir -p ~/working/azd-agents-t1-from-code && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code/" + - run: "mkdir -p ~/working/azd-agents-t1-from-code-{instance} && cp -r \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/from-code/.\" ~/working/azd-agents-t1-from-code-{instance}/" cwd: "~/working" name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 95c9a84d0d2..6d5b8d38252 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -3,19 +3,19 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" -cwd: "~/working/azd-agents-t1-manifest" +cwd: "~/working/azd-agents-t1-manifest-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-manifest" + - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 901f0c53670..9c915254688 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -3,19 +3,19 @@ # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. name: "init-template-dotnet" command: "azd ai agent init" -cwd: "~/working/azd-agents-t1-dotnet" +cwd: "~/working/azd-agents-t1-dotnet-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-dotnet" + - run: "rm -rf ~/working/azd-agents-t1-dotnet-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index c3a035828c5..d9b41930c1d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -4,19 +4,19 @@ # run `azd provision`, so no resources are created and no cost is incurred. name: "init-template-python" command: "azd ai agent init" -cwd: "~/working/azd-agents-t1-python" +cwd: "~/working/azd-agents-t1-python-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" # Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. pre: - - run: "rm -rf ~/working/azd-agents-t1-python" + - run: "rm -rf ~/working/azd-agents-t1-python-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index f27e7b4267c..45a35130019 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -4,14 +4,20 @@ # `azd ai agent run` blocks the terminal, so this needs TWO sessions: one runs the # agent, a second invokes it with --local. name: "run-local-and-invoke-local" -command: "azd ai agent run --no-inspector" +command: "azd ai agent run --port {agent} --no-inspector" cwd: "~/working/azd-agents-shared" +# Reserve a free port per scenario run so parallel local runs don't collide on +# the default 8088, and so the run + invoke sessions find each other (a pool is +# shared across every start_session that passes the same scenario_path). +allocate_ports: [agent] + notes: | - Use two tester sessions: - - session_id "run": runs 'azd ai agent run --no-inspector' (this 'command'). - - session_id "invoke": a bash session for 'azd ai agent invoke --local'. - The agent listens on port 8088 by default. Stop the run session with Ctrl-C at the end. + Use two tester sessions that share this scenario's port pool — start BOTH with + the same scenario_path (and the same instance_id if running in parallel): + - session_id "run-{instance}": runs this 'command' (azd ai agent run --port {agent} --no-inspector). + - session_id "invoke-{instance}": a bash session for 'azd ai agent invoke --local --port {agent}'. + The agent listens on the allocated port {agent}. Stop the run session with Ctrl-C at the end. # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: @@ -21,10 +27,10 @@ pre: continue_on_error: true goals: - - "In the run session, wait until the local agent reports it is listening (look for a port 8088 / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." - - "Start a SECOND session (session_id 'invoke') running bash in the same cwd." - - "In the invoke session, run: azd ai agent invoke --local \"Hello from local!\" and wait for a response from the locally running agent." + - "In the run session, wait until the local agent reports it is listening (look for a port {agent} / 'listening' message). --no-inspector is used so the Agent Inspector is not launched." + - "Start a SECOND session (session_id 'invoke-{instance}', same scenario_path, same instance_id) running bash in the same cwd so it shares this scenario's allocated port {agent}." + - "In the invoke session, run: azd ai agent invoke --local --port {agent} \"Hello from local!\" and wait for a response from the locally running agent." - "Confirm the response comes from the local process (not Foundry) and is non-empty." - - "Take a screenshot of both the run session (showing it listening) and the invoke response." + - "Take a screenshot of both the run session (showing it listening on {agent}) and the invoke response." - "Stop the run session with Ctrl-C and confirm the local agent shuts down cleanly." - - "Report a finding if the local server fails to start, if --local invoke cannot reach it, or if shutdown is messy." + - "Report a finding if the local server fails to start, if --local invoke cannot reach it on port {agent}, or if shutdown is messy." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index a609d8d4b23..60a6cb3cf6b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -41,7 +41,9 @@ Implications: - `cwd` directories do not need to pre-exist — the tester creates them if missing. - The `cwd` convention is three-way by design: ephemeral `/tmp` for read-only scenarios that touch no project (`version`, `--help`, `sample list`); a unique - `~/working/azd-agents-*` dir per `init`/`doctor` scenario for isolation; and a + `~/working/azd-agents-*-{instance}` dir per `init`/`doctor` scenario for + isolation (the `{instance}` suffix keeps concurrent runs of the same scenario + apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they operate on the same deployed agent. @@ -74,6 +76,55 @@ Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of them reuse that session credential. +## Parallel-readiness & port allocation + +The tester can run **N concurrent instances of the same scenario** and can +**allocate free TCP ports** per run. Scenarios here are authored to take +advantage of both where it's safe. + +- **`{instance}` substitution.** `start_session(..., instance_id="1")` exposes + `{instance}` for substitution into `command`, `cwd`, `env`, hook fields, and + `goals`. It **defaults to `"main"`** when `instance_id` is omitted, so a single + run is unchanged (dirs/names just end in `-main`). +- **Which scenarios are parallel-ready:** + - **Tier 0 work-dir scenarios** (`doctor`, picker, validate) and **all Tier 1 + `init` scenarios** suffix their `cwd` (and hook paths) with `-{instance}`, so + concurrent instances get isolated working directories. + - **Tier 1 resource names** are suffixed with `-{instance}` too (via the + RESOURCE NAMING goal and the `--agent-name` flag), so parallel instances + don't collide on Azure resource names. + - **`27-run-local-and-invoke-local`** declares `allocate_ports: [agent]` and + binds `azd ai agent run`/`invoke --local` to `--port {agent}`. A port pool is + shared across every `start_session` with the same `scenario_path`, so the + `run` and `invoke` sessions find each other; parallel local runs each get a + distinct port instead of colliding on the default `8088`. +- **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), + plus `20-setup` and `2Z-teardown`, all share the one deployed agent in + `~/working/azd-agents-shared`. They are **not** parameterized with `{instance}` + (doing so would break the shared-agent assumption) and should be run serially. + +To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the +same `instance_id` for paired `run`/`invoke` sessions of one scenario). + +## Driving conventions + +These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent +should follow them so the runs actually *test* the CLI instead of papering over +its bugs: + +- **Don't verify/retry after a `select`.** These runs exist to catch picker + bugs; reading back the echo and "correcting" a pick hides the very defect the + test is for. Send the action and let downstream prompts surface any failure. +- **Treat a select miss as a hard failure.** The tester's `select_by_text` is + fail-loud: a missing target raises `LookupError`, surfaced as + `ERROR during 'select': …`. **Report a finding and stop** — do not retry with a + different `choice_text`/`choice_index` to work around it. +- **Prefer `choice_text` over `choice_index`** when the label is stable (indices + shift between releases). +- **Pause before the first cloud-creating action.** Provisioning is expensive and + irreversible-ish; confirm with the user before entering an `init`/`provision` + flow that creates real resources (especially when running in parallel). + ## Tiers @@ -146,10 +197,11 @@ so they operate on the same deployed agent. - **Model**: `gpt-4.1-mini` (cheap/fast for testing) - **Resource name prefix**: every newly created Azure resource (Foundry project/account, azd environment, agent, model deployment, resource group) is - named with a `trangevi-` prefix so test resources are easy to identify and - clean up. Note that some fields lowercase the value and replace invalid - characters with hyphens — that normalization is expected (see - `sanitizeAgentName` in the extension). + named with a `trangevi-` prefix (and, in parallel-ready Tier 1 scenarios, a + `-{instance}` suffix) so test resources are easy to identify, keep distinct + across concurrent runs, and clean up. Note that some fields lowercase the value + and replace invalid characters with hyphens — that normalization is expected + (see `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. @@ -227,4 +279,8 @@ be run back to back in any order within a tier: group (rather than one file per subcommand) to avoid cross-scenario ordering dependencies — still one command at a time. - `azd ai agent run` blocks the terminal; `27-run-local-and-invoke-local.yaml` - uses two sessions (one to run, one to invoke `--local`). + uses two sessions (one to run, one to invoke `--local`) that share an + allocated `{agent}` port (see + [Parallel-readiness](#parallel-readiness--port-allocation)). +- Run artifacts (screenshots, HTML reports) land in `.reports/`, which is + git-ignored. From 596826b623e2341e24aea3b04706774f88c3695e Mon Sep 17 00:00:00 2001 From: trangevi Date: Wed, 3 Jun 2026 15:43:25 -0700 Subject: [PATCH 04/13] Some scenario updates Signed-off-by: trangevi --- .../00-init-picker-navigation.yaml | 10 +++- .../00-init-validate-mutually-exclusive.yaml | 12 +++-- .../00-init-validate-no-prompt-missing.yaml | 2 +- .../10-init-deploy-mode-code.yaml | 8 ++-- .../10-init-flags-agent-name-model.yaml | 11 ++++- .../10-init-from-code.yaml | 5 +- .../10-init-from-manifest-url.yaml | 12 ++++- .../10-init-template-dotnet.yaml | 2 +- .../10-init-template-python.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 2 +- .../README.md | 48 +++++++++++++++---- .../fixtures/from-code/app.py | 2 +- .../fixtures/from-code/requirements.txt | 4 +- 13 files changed, 88 insertions(+), 32 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index bbf2c0c7fd5..d9907dde3c8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -10,14 +10,20 @@ cwd: "~/working/azd-agents-picker-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Clean dir so the abort-before-Azure check can't be confused by prior state. +# Clean dir, then seed a file so the dir is non-empty. The init-method picker +# ("Use the code in the current directory" / "Start new from a template") only +# appears when the working directory is NOT empty; on an empty dir the command +# auto-selects the template flow and skips the picker. pre: - run: "rm -rf ~/working/azd-agents-picker-{instance}" cwd: "~/working" name: "reset working dir" + - run: "mkdir -p ~/working/azd-agents-picker-{instance} && touch ~/working/azd-agents-picker-{instance}/placeholder.txt" + cwd: "~/working" + name: "seed a placeholder file so the init-method picker appears" goals: - - "Wait for the first prompt asking how to initialize (e.g. start from a template / use existing code / use a manifest)." + - "Wait for the first prompt asking how to initialize (it should appear because the directory is non-empty): 'Use the code in the current directory' / 'Start new from a template'." - "Select 'Start new from a template', then wait for the language prompt." - "Select a language (Python), then wait for the template list prompt." - "On the template list, type a partial search string to filter the list; confirm the list narrows to matching entries." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 6ee69d98636..5d5df023e7a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -1,6 +1,10 @@ -# Tier 0 (offline) — `init` rejects mutually exclusive --from-code and --manifest. +# Tier 0 (offline) — `init` rejects a positional argument combined with --manifest. +# +# There is no `--from-code` flag. A real, offline-detectable conflict is passing +# BOTH a positional manifest argument AND -m/--manifest, which the command +# rejects up front (CodeConflictingArguments) before any wizard or network call. name: "init-validate-mutually-exclusive" -command: "azd ai agent init --from-code -m https://example.com/agent.manifest.yaml" +command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" cwd: "~/working/azd-agents-validate-{instance}" env: @@ -14,7 +18,7 @@ pre: goals: - "Wait for the command to fail fast (it should not start the interactive wizard)." - - "Confirm the error message clearly states that --from-code and --manifest/-m are mutually exclusive." + - "Confirm the error message clearly states that you cannot pass both a positional argument and --manifest." - "Confirm the process exits non-zero and does NOT create or modify any project files in the directory." - "Take a screenshot of the error output." - - "Report a finding if the flags are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." + - "Report a finding if the conflicting arguments are silently accepted, if the wizard starts anyway, or if the error message is unclear about the conflict." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index ac31b1f2127..d5a46e55cb1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -15,6 +15,6 @@ pre: goals: - "Run init in --no-prompt mode in an empty directory with no flags and no existing code/manifest." - "Confirm the command does NOT hang waiting for input (no-prompt must never block on a prompt)." - - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --from-code, --manifest, or --project-id)." + - "Confirm it exits non-zero with a helpful message explaining what required value or decision could not be resolved automatically (e.g. needing --src, --manifest, or --project-id)." - "Take a screenshot of the error output." - "Report a finding if it hangs, prompts interactively despite --no-prompt, or gives an unhelpful error." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 89260d57ba8..4663a6dbc89 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -2,9 +2,10 @@ # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime -# (instead of building a container image). +# (instead of building a container image). There is no --from-code flag; the +# from-code flow is selected interactively at the init-method prompt. name: "init-deploy-mode-code" -command: "azd ai agent init --from-code --deploy-mode code" +command: "azd ai agent init --deploy-mode code" cwd: "~/working/azd-agents-t1-code-deploy-{instance}" env: @@ -21,7 +22,8 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 5c500ef9fdc..08797dbd7a2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,6 +1,9 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Also requires GitHub login: `gh auth login` (manifest download can fall back to +# the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails +# fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" @@ -9,14 +12,18 @@ cwd: "~/working/azd-agents-t1-flags-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" - run: "rm -rf ~/working/azd-agents-t1-flags-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index 01026e394da..bc8529c8853 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -6,7 +6,7 @@ # guaranteed and the run is idempotent. Override the fixture location with # AZD_AGENTS_FIXTURES if your repo is checked out elsewhere. name: "init-from-code" -command: "azd ai agent init --from-code" +command: "azd ai agent init" cwd: "~/working/azd-agents-t1-from-code-{instance}" env: @@ -21,7 +21,8 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 6d5b8d38252..d534205437f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,10 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # # Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Also requires GitHub login: `gh auth login`. Downloading the manifest (and its +# sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub +# API is rate-limited, which would otherwise drop into an interactive gh login +# mid-run. The pre hook fails fast if gh is not authenticated. name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" @@ -8,14 +12,18 @@ cwd: "~/working/azd-agents-t1-manifest-{instance}" env: AZD_DISABLE_AGENT_DETECT: "1" -# Start from a clean dir so a prior run's scaffold can't trigger overwrite prompts. +# Require GitHub auth up front (like az login), then start from a clean dir so a +# prior run's scaffold can't trigger overwrite prompts. pre: + - run: "gh auth status || { echo 'ERROR: GitHub CLI not authenticated. Run: gh auth login'; exit 1; }" + cwd: "~/working" + name: "require gh auth login (manifest download)" - run: "rm -rf ~/working/azd-agents-t1-manifest-{instance}" cwd: "~/working" name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 9c915254688..a9a2dbd69b8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -15,7 +15,7 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index d9b41930c1d..a8516b86b4f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -16,7 +16,7 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index f18400d226e..36e619802c7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -26,7 +26,7 @@ pre: name: "clear the shared working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 60a6cb3cf6b..1daf046970f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -76,6 +76,24 @@ Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of them reuse that session credential. +### GitHub login (manifest scenarios) + +The manifest scenarios (`10-init-from-manifest-url`, +`10-init-flags-agent-name-model`) download an agent manifest — and its sibling +files — from a public GitHub repo. The CLI first tries the anonymous GitHub API, +but when that's rate-limited (60 req/hr) it falls back to the `gh` CLI, which +would otherwise drop into an **interactive GitHub login** mid-run. Like +`az login`, this is a one-time manual prerequisite the agent can't complete, so a +human must run it once per WSL session **before** driving those scenarios: + +``` +gh auth login +``` + +Those scenarios include a `pre` hook that runs `gh auth status` and **fails fast** +if GitHub CLI isn't authenticated, so a missing login surfaces as a clear setup +error instead of a hung interactive prompt. + ## Parallel-readiness & port allocation The tester can run **N concurrent instances of the same scenario** and can @@ -121,6 +139,10 @@ its bugs: different `choice_text`/`choice_index` to work around it. - **Prefer `choice_text` over `choice_index`** when the label is stable (indices shift between releases). +- **Clear a pre-filled text field before typing.** Some prompts (e.g. the agent + name) come pre-populated with an editable default; typing without clearing + *appends* to it. Select-all then delete (or backspace) first so your value + replaces the default instead of producing `defaultyourvalue`. - **Pause before the first cloud-creating action.** Provisioning is expensive and irreversible-ish; confirm with the user before entering an `init`/`provision` flow that creates real resources (especially when running in parallel). @@ -142,7 +164,7 @@ in any order, any time. | `00-sample-list-json-filters.yaml` | `sample list` `--output json`, `--language`, `--type`, `--featured-only` | | `00-doctor-empty-dir.yaml` | `doctor` in an empty dir (graceful skips) | | `00-doctor-local-only.yaml` | `doctor --local-only` | -| `00-init-validate-mutually-exclusive.yaml` | `init` flag validation (`--from-code` + `-m`) | +| `00-init-validate-mutually-exclusive.yaml` | `init` arg validation (positional manifest + `-m`) | | `00-init-validate-no-prompt-missing.yaml` | `init --no-prompt` missing-input error | | `00-init-picker-navigation.yaml` | `init` interactive picker UX (abort before Azure) | @@ -155,9 +177,9 @@ and verifies the generated files, then stops before `azd provision`. |------|---------| | `10-init-template-python.yaml` | `init` new-from-template, Python | | `10-init-template-dotnet.yaml` | `init` new-from-template, C#/.NET | -| `10-init-from-manifest-url.yaml` | `init -m ` | -| `10-init-from-code.yaml` | `init --from-code` | -| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` | +| `10-init-from-manifest-url.yaml` | `init -m ` (needs `gh auth login`) | +| `10-init-from-code.yaml` | `init` → pick "Use the code in the current directory" | +| `10-init-flags-agent-name-model.yaml` | `init -m … --agent-name --model` (needs `gh auth login`) | | `10-init-deploy-mode-code.yaml` | `init --deploy-mode code` (entry-point/runtime) | ### Tier 2 — Cloud end-to-end (prefix `2x-`) — ⚠️ incurs Azure cost @@ -222,10 +244,15 @@ How they're used here: - **`pre` reset** — stateful Tier 0/1 scenarios `rm -rf` their own working dir so re-runs start clean. (`start_session` recreates the dir, so removing it is enough; the doctor/init scenarios just need an empty dir.) -- **`pre` fixture seed** — the `--from-code` scenarios +- **`pre` fixture seed** — the existing-code scenarios (`10-init-from-code`, `10-init-deploy-mode-code`) also copy a committed Python - fixture into the dir so the source exists before `init --from-code` inspects it - (see [Fixtures](#fixtures)). + fixture into the dir so the source exists before the wizard's "Use the code in + the current directory" flow inspects it (see [Fixtures](#fixtures)). +- **`pre` gh-auth guard** — the manifest scenarios (`10-init-from-manifest-url`, + `10-init-flags-agent-name-model`) run `gh auth status` and fail fast if GitHub + CLI isn't authenticated, because downloading the manifest can fall back to the + `gh` CLI (and an interactive login) when the anonymous GitHub API is + rate-limited. Run `gh auth login` first (see [Authentication](#authentication)). - **`pre` idempotent setup (Tier 2)** — `20-setup-deploy-shared-agent` first runs `azd down --force --purge` if a leftover project exists in the shared dir (so it never orphans live Azure resources), then clears the dir. The down hook uses @@ -238,9 +265,10 @@ How they're used here: ## Fixtures `fixtures/from-code/` holds a minimal Python agent source tree (`app.py` + -`requirements.txt`) that satisfies the extension's `--from-code` detection +`requirements.txt`) that satisfies the extension's existing-code detection (it looks for `requirements.txt` or any `.py`, and defaults the entry point to -`app.py`). The from-code scenarios copy it into the working dir via a `pre` hook. +`app.py`). The existing-code scenarios copy it into the working dir via a `pre` +hook, then select "Use the code in the current directory" at the init prompt. The hook references the fixture by absolute path with an overridable env var: @@ -250,7 +278,7 @@ cp -r "${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai If your clone lives somewhere other than `/mnt/c/Repos/azure-dev` (the WSL view of `C:\Repos\azure-dev`), export `AZD_AGENTS_FIXTURES` to the WSL path of this -`fixtures/` directory before running the from-code scenarios. +`fixtures/` directory before running the existing-code scenarios. ## Re-running scenarios (idempotency) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py index 56b577b6277..d0d8a0c1c0b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/app.py @@ -1,4 +1,4 @@ -"""Minimal agent source fixture for `azd ai agent init --from-code` scenarios. +"""Minimal agent source fixture for the `azd ai agent init` existing-code scenarios. This file exists only so the extension's from-code detection treats the working directory as a Python agent project (it looks for requirements.txt or any .py diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt index d94c3d032d6..b6ed8d63f87 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/from-code/requirements.txt @@ -1,4 +1,4 @@ -# Minimal dependency set so `azd ai agent init --from-code` detects a Python -# project. Keep this lightweight — the Tier 1 from-code scenarios only scaffold +# Minimal dependency set so `azd ai agent init` (existing-code flow) detects a Python +# project. Keep this lightweight — the Tier 1 existing-code scenarios only scaffold # and do not install or run the agent. azure-ai-projects From 71767dbfe6570364f464f1515190544845e82b55 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 10:54:35 -0700 Subject: [PATCH 05/13] Some more fixes to the scenarios Signed-off-by: trangevi --- .../10-init-flags-agent-name-model.yaml | 2 +- .../10-init-from-manifest-url.yaml | 2 +- .../20-setup-deploy-shared-agent.yaml | 15 ++++++++---- .../21-show-json.yaml | 6 ++--- .../21-show.yaml | 6 ++--- .../22-invoke-input-file.yaml | 6 ++--- .../22-invoke-new-session.yaml | 6 ++--- .../22-invoke-remote.yaml | 6 ++--- .../23-sessions-lifecycle.yaml | 6 ++--- .../24-files-lifecycle.yaml | 6 ++--- .../25-monitor-console.yaml | 6 ++--- .../25-monitor-system.yaml | 6 ++--- .../26-endpoint-update.yaml | 23 +++++++++++++++---- .../27-run-local-and-invoke-local.yaml | 6 ++--- .../28-eval-init-run-show.yaml | 9 ++++---- .../28-eval-update.yaml | 6 ++--- .../29-optimize-submit-status.yaml | 14 +++++++---- .../2A-doctor-provisioned-all-pass.yaml | 9 ++++---- .../2Z-teardown-down.yaml | 2 +- .../README.md | 20 +++++++++++----- .../fixtures/optimize-tasks.jsonl | 3 +++ 21 files changed, 102 insertions(+), 63 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 08797dbd7a2..93e79474fb8 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -6,7 +6,7 @@ # fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" cwd: "~/working/azd-agents-t1-flags-{instance}" env: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index d534205437f..fe28a2566fa 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -6,7 +6,7 @@ # API is rate-limited, which would otherwise drop into an interactive gh login # mid-run. The pre hook fails fast if gh is not authenticated. name: "init-from-manifest-url" -command: "azd ai agent init -m https://github.com/microsoft/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" env: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 36e619802c7..807b6721f00 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -1,8 +1,11 @@ # Tier 2 (cloud E2E) — SETUP: deploy the shared agent used by all 21-..2A scenarios. # -# ⚠️ Incurs Azure cost. Run this FIRST. The deployed agent lives in the shared -# working directory ~/working/azd-agents-shared and is reused by the targeted -# scenarios. Run 2Z-teardown-down.yaml LAST to clean up. +# ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared +# and scaffolds the project into a subdirectory named after the agent, so the +# deployed project lives in ~/working/azd-agents-shared/trangevi-basic-responses +# and is reused by the targeted scenarios. The agent name MUST be exactly +# 'trangevi-basic-responses' so that subdirectory path is deterministic. Run +# 2Z-teardown-down.yaml LAST to clean up. name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" @@ -16,7 +19,7 @@ env: # nothing to tear down). Re-using a clean path also avoids the resource-name hash # collision in issue #8360. pre: - - run: "if [ -f ~/working/azd-agents-shared/azure.yaml ]; then (cd ~/working/azd-agents-shared && azd down --force --purge); fi" + - run: "if [ -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml ]; then (cd ~/working/azd-agents-shared/trangevi-basic-responses && azd down --force --purge); fi" cwd: "~/working/azd-agents-shared" name: "tear down any leftover deployed agent" continue_on_error: true @@ -30,6 +33,7 @@ goals: - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." + - "When prompted for the AGENT NAME, set it to EXACTLY 'trangevi-basic-responses' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named 'trangevi-basic-responses'." - "When asked how to deploy, select 'Container' (hosted agent)." - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." @@ -38,7 +42,8 @@ goals: - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Run 'azd provision' and wait for it to succeed. This creates the real Azure resources and deploys the agent." + - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." + - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This creates the real Azure resources and deploys the agent." - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - "Take a screenshot of the successful provision and 'show' output." - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index 0f9dc491c3a..dc1ae9b47ad 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show-json" command: "azd ai agent show --output json" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index fb06d2ff454..4268464f912 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show" command: "azd ai agent show" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 79ab211dbcb..79766bfe9f4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-input-file" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index ce93cf77620..4aec58782e6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-new-session" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 9fedf175a0c..77c0f9d798e 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index f559df8cc08..fb5a5a781d7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -4,12 +4,12 @@ # Targets the `sessions` command group end-to-end in one run. name: "sessions-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index 4356ab7a54d..dbb81bfa47a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -5,12 +5,12 @@ # Note: file operations target a session — run an invoke first if no session exists. name: "files-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index d3288f76781..deff52e7d42 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists to stream logs from. name: "monitor-console" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 08c182ab1d3..85b64e0fbfe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists. name: "monitor-system" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 8cc712f582c..9b970cd50ce 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -3,17 +3,32 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "endpoint-update" command: "azd ai agent endpoint update" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true + # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors + # ("nothing to update") if neither is defined. The Basic Responses template + # defines neither, so inject a minimal agent_card (idempotently) before the run + # so there is something to patch. + - run: | + f="$(find ~/working/azd-agents-shared/trangevi-basic-responses -name agent.yaml | head -1)" + if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then + printf '\nagent_card:\n description: "trangevi endpoint-update test card"\n skills:\n - id: "trangevi-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" + echo "Injected agent_card into $f" + else + echo "agent_card already present or agent.yaml not found" + fi + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + name: "inject agent_card so there is something to patch" + continue_on_error: true goals: - - "Run endpoint update for the default (auto-detected) agent service." + - "Run endpoint update for the default (auto-detected) agent service. NOTE: a minimal agent_card was injected into agent.yaml during setup so the patch has content." - "Confirm it patches the existing deployed agent's endpoint/card configuration and explicitly does NOT create a new agent version." - "After it completes, run 'azd ai agent show' and confirm the agent version is unchanged from before the update." - "Take a screenshot of the update result and the post-update 'show' output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index 45a35130019..e0b10da49d9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -5,7 +5,7 @@ # agent, a second invokes it with --local. name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is @@ -21,8 +21,8 @@ notes: | # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml index df9dfcd33ff..aa93e385566 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml @@ -4,17 +4,18 @@ # (a deployed agent is required to generate and run evals). name: "eval-init-run-show" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true goals: - - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --eval-model gpt-4.1-mini --max-samples 15 and wait for it to generate an eval config and dataset." + - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --max-samples 15 and wait for it to generate an eval config and dataset. (Do NOT pass --eval-model — let the command prompt so a real, deployed model is selected.)" + - "At the eval-model selection prompt, accept the pre-filled default deployment. If no default is offered, choose 'Select another deployment' and pick the model deployment that was created during setup (the 'trangevi-'-prefixed gpt-4.1-mini deployment). Do NOT type a bare model name like 'gpt-4.1-mini' — the service resolves this value as a deployment name, and a non-existent deployment returns a 400 error." - "Confirm an eval.yaml (and dataset) is written to the project root." - "Run: azd ai agent eval run and wait for the evaluation run to complete." - "Run: azd ai agent eval list and confirm the run appears." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml index 7101c392f82..19abb9242c1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml @@ -3,12 +3,12 @@ # Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. name: "eval-update" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml index 1c53ded1649..07c46c3e685 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -3,17 +3,23 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "optimize-submit-status" command: "bash" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. +# Also copy a small dataset fixture into the project so `optimize` has tasks to +# submit (it requires --dataset, an eval.yaml, or interactive selection). pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true + - run: "cp \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/optimize-tasks.jsonl\" ~/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + name: "stage optimize dataset fixture" + continue_on_error: true goals: - - "Run: azd ai agent optimize --eval-model gpt-4.1-mini --max-iterations 1 --no-wait (agent auto-detected from the azd project). Provide any other values it prompts for using sensible defaults." + - "Run: azd ai agent optimize --dataset optimize-tasks.jsonl --max-iterations 1 --no-wait (agent auto-detected from the azd project; a dataset fixture was staged into the project root during setup). Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index cffe0288a63..166313fae5b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared" + - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/trangevi-basic-responses" name: "assert shared agent is deployed" continue_on_error: true @@ -17,5 +17,6 @@ goals: - "Confirm the local checks pass (project, azure.yaml, agent service config)." - "Confirm the remote checks pass (deployed agent reachable / active)." - "Confirm doctor suggests a sensible next command (e.g. invoke or run) and exits 0 (at least one check passed, none failed)." + - "KNOWN-ACCEPTABLE WARNING: a WARNING about agent identity / role assignments is EXPECTED in this test subscription (its ABAC conditional role-assignment policy blocks some role reads) and must NOT be treated as a failure. Only an actual FAILED check (not a warning) is a finding." - "Take a screenshot of the doctor report." - - "Report a finding if any check fails or is skipped unexpectedly for a healthy provisioned project, or if the suggested next step is wrong." + - "Report a finding if a check FAILS (red/error), or is skipped unexpectedly, for a healthy provisioned project, or if the suggested next step is wrong. Do NOT report the known identity/role-assignment warning described above." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index bcad6af5fc7..1432baa5212 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -4,7 +4,7 @@ # shared agent and its Azure resources to stop incurring cost. name: "teardown-down" command: "azd down --force --purge" -cwd: "~/working/azd-agents-shared" +cwd: "~/working/azd-agents-shared/trangevi-basic-responses" goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 1daf046970f..14f5d8a7177 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -45,7 +45,10 @@ Implications: isolation (the `{instance}` suffix keeps concurrent runs of the same scenario apart — see [Parallel-readiness](#parallel-readiness--port-allocation)); and a single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they - operate on the same deployed agent. + operate on the same deployed agent. `20-setup` runs `init` in that shared dir, + which scaffolds the project into a subdirectory named after the agent, so the + deployed project actually lives in `~/working/azd-agents-shared/trangevi-basic-responses`; + the reuse and teardown scenarios run with that subdirectory as their `cwd`. On macOS/Linux these are simply native paths (no WSL involved). @@ -117,9 +120,11 @@ advantage of both where it's safe. `run` and `invoke` sessions find each other; parallel local runs each get a distinct port instead of colliding on the default `8088`. - **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), - plus `20-setup` and `2Z-teardown`, all share the one deployed agent in - `~/working/azd-agents-shared`. They are **not** parameterized with `{instance}` - (doing so would break the shared-agent assumption) and should be run serially. + plus `20-setup` and `2Z-teardown`, all share the one deployed agent under + `~/working/azd-agents-shared` (the project itself lives in the + `trangevi-basic-responses` subdirectory created by `20-setup`). They are + **not** parameterized with `{instance}` (doing so would break the shared-agent + assumption) and should be run serially. To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the same `instance_id` for paired `run`/`invoke` sessions of one scenario). @@ -189,8 +194,11 @@ Provisions real resources. **Run order matters:** 2. Any `21-`…`2A-` targeted scenario (reuse the deployed agent). 3. `2Z-teardown-down.yaml` **last** — `azd down --force --purge`. -All Tier 2 scenarios share one working directory (`~/working/azd-agents-shared`) -so they operate on the same deployed agent. +All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` +so they operate on the same deployed agent. `20-setup` runs `init` there, which +scaffolds the project into the `trangevi-basic-responses` subdirectory; the +reuse and teardown scenarios run with `~/working/azd-agents-shared/trangevi-basic-responses` +as their `cwd`. | File | Targets | |------|---------| diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl new file mode 100644 index 00000000000..ec1f77655e3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl @@ -0,0 +1,3 @@ +{"prompt":"What is 2+2?","groundTruth":"4"} +{"prompt":"What is the capital of France?","groundTruth":"Paris"} +{"prompt":"Name a primary color.","groundTruth":"Red"} From df67a66389d8c58ac752536d4f7b34c901ac09ec Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 15:03:05 -0700 Subject: [PATCH 06/13] Some more improvements Signed-off-by: trangevi --- .../20-setup-deploy-shared-agent.yaml | 9 +- .../22-invoke-new-session.yaml | 16 +++- .../23-sessions-lifecycle.yaml | 5 +- .../29-optimize-submit-status.yaml | 2 +- .../README.md | 91 ++++++++++++++++++- 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 807b6721f00..8b8bada67dd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -43,7 +43,8 @@ goals: - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." - - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This creates the real Azure resources and deploys the agent." - - "After provision, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - - "Take a screenshot of the successful provision and 'show' output." - - "Report a finding if init or provision fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." + - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." + - "After provision succeeds, run 'azd deploy' (from the same subdirectory) and wait for it to succeed. This deploys the agent to the provisioned infrastructure — it is a required, separate step from provision before the agent can be shown or invoked." + - "After deploy, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." + - "Take a screenshot of the successful provision, deploy, and 'show' output." + - "Report a finding if init, provision, or deploy fails, hangs, or produces a confusing error. Do NOT run 'azd down' here — teardown is a separate scenario." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index 4aec58782e6..d56211ce4e6 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -1,7 +1,12 @@ -# Tier 2 (cloud E2E) — `invoke --new-session` resets conversation state. +# Tier 2 (cloud E2E) — `invoke` SESSION vs CONVERSATION memory semantics. +# +# A SESSION and a CONVERSATION are distinct concepts. For the responses-protocol +# agent (Basic Responses), multi-turn memory is bound to the CONVERSATION, not the +# session: `--new-session` starts a fresh session but reuses the saved conversation +# (memory persists), while `--new-conversation` is what actually resets memory. # # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -name: "invoke-new-session" +name: "invoke-session-vs-conversation" command: "bash" cwd: "~/working/azd-agents-shared/trangevi-basic-responses" @@ -14,7 +19,8 @@ pre: goals: - "Run: azd ai agent invoke \"My name is Quinn. Remember it.\" and wait for a response." - - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session reused the same conversation)." - - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent does NOT recall 'Quinn' — the new session discarded prior history." + - "Run: azd ai agent invoke \"What is my name?\" and confirm the agent recalls 'Quinn' (the persisted session and conversation were reused, so multi-turn memory works)." + - "Run: azd ai agent invoke --new-session \"What is my name?\" and confirm the agent STILL recalls 'Quinn'. This is EXPECTED, not a bug: a SESSION and a CONVERSATION are distinct concepts. For this responses-protocol agent, multi-turn memory is bound to the CONVERSATION, not the session. --new-session starts a fresh session but reuses the saved conversation, so memory persists." + - "Run: azd ai agent invoke --new-conversation \"What is my name?\" and confirm the agent does NOT recall 'Quinn'. This is the true memory reset: --new-conversation discards the conversation that holds multi-turn memory." - "Take a screenshot after each invoke." - - "Report a finding if --new-session still carries over prior conversation context, or if session persistence between the first two invokes does not work." + - "Report a finding if: the first two invokes fail to establish/recall memory; OR --new-conversation still recalls 'Quinn' (memory was not reset). Do NOT report a finding merely because --new-session still recalled 'Quinn' — that is the expected session-vs-conversation distinction." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index fb5a5a781d7..1bf3ef57e6c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -18,6 +18,7 @@ goals: - "Run: azd ai agent sessions list --output table. Confirm the newly created session appears in the list." - "Run: azd ai agent sessions show using the ID from create. Confirm session details (status, version) are shown." - "Run: azd ai agent sessions delete . Confirm the session is deleted (synchronously)." - - "Run: azd ai agent sessions list again and confirm the deleted session no longer appears." + - "Run: azd ai agent sessions list again. The service SOFT-DELETES sessions, so it is EXPECTED and acceptable for the entry to still appear with status 'deleted' (it should NOT appear as 'active'). Either the entry being gone OR present with status 'deleted' is a PASS." + - "Run: azd ai agent sessions show for the deleted session and confirm it reports the session as not found / deleted (this verifies the delete took effect even though list may still show it)." - "Take a screenshot after each step." - - "Report a finding if any subcommand errors, if the created session is missing from list, or if delete does not remove it." + - "Report a finding only if any subcommand errors, if the created session is missing from list before delete, or if after delete the session still shows as 'active' (a soft-deleted 'deleted' status is NOT a finding)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml index 07c46c3e685..ebbfa72f8fe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml @@ -19,7 +19,7 @@ pre: continue_on_error: true goals: - - "Run: azd ai agent optimize --dataset optimize-tasks.jsonl --max-iterations 1 --no-wait (agent auto-detected from the azd project; a dataset fixture was staged into the project root during setup). Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." + - "Run: azd ai agent optimize --dataset \"$HOME/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl\" --max-iterations 1 --no-wait (agent auto-detected from the azd project; the dataset fixture was staged into the project root during setup). Use an ABSOLUTE path for --dataset as shown: a bare relative name is resolved relative to the agent SERVICE subdirectory, not the project root, so the absolute path avoids a 'dataset not accessible' error. Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 14f5d8a7177..5568cb47ea1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -52,6 +52,23 @@ Implications: On macOS/Linux these are simply native paths (no WSL involved). +### This applies to MCP tool arguments too + +The same path-resolution rule applies to **every path-shaped argument an +orchestrator passes to the tester's MCP tools** — most importantly the `path:` +argument on `load_scenario`, `run_pre_hooks`, and `run_post_hooks`, and the +`scenario_path:` argument on `start_session`. The server resolves them on the +WSL side, **not** on the orchestrator side. On Windows hosts, pass a POSIX path: + +| Orchestrator OS | Pass to MCP tools | Don't pass | +| --- | --- | --- | +| Windows | `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml` | `C:\Repos\azure-dev\...\00-version.yaml` | +| macOS / Linux | native absolute path | — | + +**Failure-mode hint:** if `load_scenario` returns `Scenario file not found`, the +path style is almost certainly the cause — translate `C:\…` to `/mnt/c/…` and +retry one call before fanning out. + ## Authentication Tier 1 and Tier 2 scenarios read from / write to Azure, so a **human must log in @@ -129,6 +146,47 @@ advantage of both where it's safe. To fan out, pass a distinct `instance_id` per `start_session` call (and reuse the same `instance_id` for paired `run`/`invoke` sessions of one scenario). +## Orchestrating a fleet run + +When a driving agent wants to run **many scenarios concurrently** (e.g. via +parallel background sub-agents, one scenario per sub-agent), pick the right +fan-out primitive for the shape of the run: + +- **Different scenarios in parallel** (the common case for a full Tier 0/1 + sweep): give each sub-agent a distinct, descriptive `session_id` — e.g. + `fleet-00-version`, `fleet-10-init-from-code` — and call `start_session` with + the scenario's own `cwd`. **No `instance_id` is needed**: each scenario's `cwd` + already isolates itself via the `{instance}` substitution, which defaults to + `"main"` when `instance_id` is omitted. +- **Same scenario N times in parallel:** use `instance_id="1"`, `"2"`, … per + call. See [Parallel-readiness](#parallel-readiness--port-allocation) for which + scenarios are authored to support this. +- **Tier 2 ordering is fixed**, not parallel-friendly. Run `20-setup` first, + then the targeted `21-…2A-` scenarios **serially** (they share one deployed + agent and mutate shared state — sessions, files, endpoint configuration — + so parallel runs interfere), then `2Z-teardown` last. See the + [Tier 2](#tier-2--cloud-end-to-end-prefix-2x---%EF%B8%8F-incurs-azure-cost) + section. + +### Operational guardrails for the orchestrator + +A few hard-won lessons that apply regardless of fleet size: + +- **Validate the recipe with one sub-agent before fanning out.** Spend 30 + seconds confirming that `load_scenario`, `start_session`, and one + `send_action` round-trip work end-to-end for *one* scenario before launching + a wave. This is the cheapest way to catch wiring issues (wrong path style, + wrong tool surface, auth not set up) before they multiply across many agents. +- **Background sub-agents are typically not cancellable mid-run.** Once launched, + they will run to completion (or until the runtime times them out). For Tier 1 + and especially Tier 2 scenarios with Azure side effects, launch + conservatively — a stop request can't recall an in-flight `azd provision`. +- **Keep waves small.** The wall-clock bottleneck on a fleet run is per-agent + LLM time and per-account model concurrency, not the MCP server (which is + per-`session_id`-parallel by design). Launching 4–6 sub-agents at a time and + rolling forward typically finishes a sweep faster than launching everything + at once. + ## Driving conventions These mirror the tester's own `AGENTS.md` ("Driving the MCP") — the driving agent @@ -151,6 +209,35 @@ its bugs: - **Pause before the first cloud-creating action.** Provisioning is expensive and irreversible-ish; confirm with the user before entering an `init`/`provision` flow that creates real resources (especially when running in parallel). +- **Pass `run_name=` to every `start_session` call.** The + scenario stem is the YAML filename without `.yaml` (e.g. `00-version`, + `21-show-json`, `27-run-local-and-invoke-local`). Without `run_name` the + tester auto-names the run folder `agent_YYYYMMDD_HHMMSS`, which makes + archived runs in `.reports//tester-reports/` hard to cross-reference + with the scenario list. For scenarios that start two sessions + (e.g. `27-run-local-and-invoke-local`), suffix the run_name with a role tag + (`27-run-local-and-invoke-local-run`, `27-run-local-and-invoke-local-invoke`) + so each session gets its own clearly named folder. +- **Pass `output_dir` to every `start_session` call** so the tester writes + screenshots and HTML reports directly into this repo's archive layout + instead of its own working directory. Use the WSL path of the + `.reports//tester-reports/` folder under this scenarios + directory, with `` of the form `YYYYMMDD-HHMMSS`. Pick **one** + `` per suite run and reuse it across every session — this + groups all scenarios from one run under a single folder. The driving agent + should also write the final cross-scenario summary to + `.reports//FINAL-REPORT.md` at the end. Example + `output_dir` (the WSL view of this scenarios directory in this repo): + `/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.reports/20260603-171132/tester-reports`. + If your clone lives elsewhere, substitute the WSL path of *your* + `cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/`. +- **Record a time-to-complete per scenario.** Capture wall-clock duration for + every scenario (from `start_session` to `finish_session`, including pre/post + hooks) and include it as a `Duration` column in the per-scenario tables of + `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). + This makes regression slowdowns easy to spot across runs — Tier 2 in + particular has scenarios that legitimately take many minutes (provision, + deploy, eval dataset generation) and others that should complete in seconds. ## Tiers @@ -202,11 +289,11 @@ as their `cwd`. | File | Targets | |------|---------| -| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` (SETUP) | +| `20-setup-deploy-shared-agent.yaml` | `init` + `azd provision` + `azd deploy` (SETUP) | | `21-show.yaml` | `show` (table) | | `21-show-json.yaml` | `show --output json` | | `22-invoke-remote.yaml` | `invoke` (remote) | -| `22-invoke-new-session.yaml` | `invoke --new-session` | +| `22-invoke-new-session.yaml` | `invoke --new-session` / `--new-conversation` (session vs conversation memory) | | `22-invoke-input-file.yaml` | `invoke -f ` | | `23-sessions-lifecycle.yaml` | `sessions create/list/show/delete` | | `24-files-lifecycle.yaml` | `files upload/list/stat/mkdir/download/delete` | From 3eac215885617c1e22172264261494bf3ea3bee0 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 17:04:20 -0700 Subject: [PATCH 07/13] Remove optimization and evals because I don't understand them yet, will add back later Signed-off-by: trangevi --- .../28-eval-init-run-show.yaml | 24 ----------------- .../28-eval-update.yaml | 20 -------------- .../29-optimize-submit-status.yaml | 27 ------------------- .../README.md | 5 +--- .../fixtures/optimize-tasks.jsonl | 3 --- 5 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml delete mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml deleted file mode 100644 index aa93e385566..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-init-run-show.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Tier 2 (cloud E2E) — `eval` flow: init a suite, run it, then list/show results. -# -# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully -# (a deployed agent is required to generate and run evals). -name: "eval-init-run-show" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - -goals: - - "Run: azd ai agent eval init --gen-instruction \"This agent answers general questions concisely.\" --max-samples 15 and wait for it to generate an eval config and dataset. (Do NOT pass --eval-model — let the command prompt so a real, deployed model is selected.)" - - "At the eval-model selection prompt, accept the pre-filled default deployment. If no default is offered, choose 'Select another deployment' and pick the model deployment that was created during setup (the 'trangevi-'-prefixed gpt-4.1-mini deployment). Do NOT type a bare model name like 'gpt-4.1-mini' — the service resolves this value as a deployment name, and a non-existent deployment returns a 400 error." - - "Confirm an eval.yaml (and dataset) is written to the project root." - - "Run: azd ai agent eval run and wait for the evaluation run to complete." - - "Run: azd ai agent eval list and confirm the run appears." - - "Run: azd ai agent eval show and confirm run details / scores are displayed." - - "Take a screenshot after init, run, and show." - - "Report a finding if generation fails, eval.yaml is malformed, the run errors, or results are missing/unreadable." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml deleted file mode 100644 index 19abb9242c1..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/28-eval-update.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Tier 2 (cloud E2E) — `eval update` refreshes dataset and evaluators. -# -# Precondition: 28-eval-init-run-show.yaml has been run so an eval.yaml exists. -name: "eval-update" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - -goals: - - "Confirm an eval.yaml from a prior 'eval init' exists in the project root (run 28-eval-init-run-show.yaml first if not)." - - "Run: azd ai agent eval update --dataset-only and confirm only the dataset is updated (evaluators left unchanged)." - - "Run: azd ai agent eval update --evaluator-only and confirm only the evaluators are updated (dataset left unchanged)." - - "Take a screenshot after each update." - - "Report a finding if --dataset-only or --evaluator-only updates the wrong thing, errors, or prompts despite the scoping flag." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml deleted file mode 100644 index ebbfa72f8fe..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/29-optimize-submit-status.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Tier 2 (cloud E2E) — `optimize` submit (no-wait) then check status/list. -# -# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. -name: "optimize-submit-status" -command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - -# Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. -# Also copy a small dataset fixture into the project so `optimize` has tasks to -# submit (it requires --dataset, an eval.yaml, or interactive selection). -pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "assert shared agent is deployed" - continue_on_error: true - - run: "cp \"${AZD_AGENTS_FIXTURES:-/mnt/c/Repos/azure-dev/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures}/optimize-tasks.jsonl\" ~/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" - name: "stage optimize dataset fixture" - continue_on_error: true - -goals: - - "Run: azd ai agent optimize --dataset \"$HOME/working/azd-agents-shared/trangevi-basic-responses/optimize-tasks.jsonl\" --max-iterations 1 --no-wait (agent auto-detected from the azd project; the dataset fixture was staged into the project root during setup). Use an ABSOLUTE path for --dataset as shown: a bare relative name is resolved relative to the agent SERVICE subdirectory, not the project root, so the absolute path avoids a 'dataset not accessible' error. Do NOT pass --eval-model — let the command prompt and select a real, deployed model (accept the pre-filled default deployment, or pick the 'trangevi-'-prefixed deployment created during setup). Provide sensible defaults for anything else it prompts for." - - "Confirm a job is submitted and an optimization job ID is printed (the command returns immediately due to --no-wait)." - - "Run: azd ai agent optimize status using the printed ID and confirm a status is reported." - - "Run: azd ai agent optimize list and confirm the submitted job appears in recent runs." - - "Take a screenshot after submit, status, and list." - - "Report a finding if submission fails, the job ID is not surfaced, or status/list cannot find the job. (Optional: cancel with 'azd ai agent optimize cancel ' to avoid leaving it running.)" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 5568cb47ea1..0ffedc534fa 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -237,7 +237,7 @@ its bugs: `FINAL-REPORT.md`. Use `Hh Mm Ss` formatting (e.g. `3m 21s`, `1h 04m 12s`). This makes regression slowdowns easy to spot across runs — Tier 2 in particular has scenarios that legitimately take many minutes (provision, - deploy, eval dataset generation) and others that should complete in seconds. + deploy) and others that should complete in seconds. ## Tiers @@ -301,9 +301,6 @@ as their `cwd`. | `25-monitor-system.yaml` | `monitor --type system` | | `26-endpoint-update.yaml` | `endpoint update` | | `27-run-local-and-invoke-local.yaml` | `run` + `invoke --local` (two sessions) | -| `28-eval-init-run-show.yaml` | `eval init/run/list/show` | -| `28-eval-update.yaml` | `eval update` | -| `29-optimize-submit-status.yaml` | `optimize` + `optimize status/list` | | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl deleted file mode 100644 index ec1f77655e3..00000000000 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/fixtures/optimize-tasks.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"prompt":"What is 2+2?","groundTruth":"4"} -{"prompt":"What is the capital of France?","groundTruth":"Paris"} -{"prompt":"Name a primary color.","groundTruth":"Red"} From c18315cb27f7435048d315d6a9f91a67c790396c Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 4 Jun 2026 17:29:49 -0700 Subject: [PATCH 08/13] Add prompt to readme Signed-off-by: trangevi --- .../README.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 0ffedc534fa..2539711b89b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -24,6 +24,28 @@ the working dir or seeding a fixture), and a few declare **`post:` hooks** `run_post_hooks` MCP tools — `load_scenario` surfaces whether a scenario has any. See [Pre/post hooks](#prepost-hooks) below. +Here's also a sample prompt to run all of the scenarios, utilizing fleet mode: + +``` +Within the agents extension, there is a tests/cli-interactive-tester-scenarios directory, containing +a set of test scenarios for the cli-interactive-tester. I want you to use the cli-interactive-tester to; + load the scenarios, + start the session and accomplish the goals, + if the scenario declares pre or post hooks, run them before/after the session, + and take screenshots at each step. + +I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios +in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios +in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make +sure to take that into account when distributing the work. I want to run all of the tests regardless +of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. + +After all of these scenarios are run, create a final result report. + +Create a plan to accomplish this +``` + + ## Paths run inside WSL (on Windows) The cli-interactive-tester drives CLIs through **tmux**, which on Windows runs From 1d1e5f2d978bec55e324142db439a1419e62daec Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 12:34:16 -0700 Subject: [PATCH 09/13] Add parameterization support Signed-off-by: trangevi --- .../.gitignore | 4 + .../10-init-deploy-mode-code.yaml | 13 +- .../10-init-flags-agent-name-model.yaml | 17 +-- .../10-init-from-code.yaml | 13 +- .../10-init-from-manifest-url.yaml | 15 ++- .../10-init-template-dotnet.yaml | 15 ++- .../10-init-template-python.yaml | 19 +-- .../20-setup-deploy-shared-agent.yaml | 31 +++-- .../21-show-json.yaml | 6 +- .../21-show.yaml | 6 +- .../22-invoke-input-file.yaml | 6 +- .../22-invoke-new-session.yaml | 6 +- .../22-invoke-remote.yaml | 6 +- .../23-sessions-lifecycle.yaml | 6 +- .../24-files-lifecycle.yaml | 6 +- .../25-monitor-console.yaml | 6 +- .../25-monitor-system.yaml | 6 +- .../26-endpoint-update.yaml | 12 +- .../27-run-local-and-invoke-local.yaml | 6 +- .../2A-doctor-provisioned-all-pass.yaml | 6 +- .../2Z-teardown-down.yaml | 2 +- .../README.md | 124 ++++++++++++++---- .../profile.local.yaml.example | 27 ++++ .../profile.yaml | 25 ++++ 24 files changed, 261 insertions(+), 122 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore index 7fde892bf70..a3ca34ee539 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/.gitignore @@ -1,2 +1,6 @@ # cli-interactive-tester run artifacts (screenshots, HTML reports, scrollback) .reports/ + +# Per-developer / per-CI scenario profile (identifying values). Bootstrap +# from profile.local.yaml.example. See README "Profile / overrides". +profile.local.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 4663a6dbc89..501f2fdfb32 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — interactive code-deploy mode (entry point + runtime). # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Targets the --deploy-mode code path which prompts for entry-point and runtime # (instead of building a container image). There is no --from-code flag; the # from-code flow is selected interactively at the init-method prompt. @@ -22,16 +23,16 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory'." - "Wait for the tool to inspect the current directory's code with code-deploy (ZIP upload) mode selected." - "If an existing agent manifest is detected, confirm reuse." - "When prompted for an entry point, provide the main entry file (e.g. 'app.py')." - "When prompted for a runtime, select an appropriate runtime (e.g. 'python_3_13')." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml/agent.yaml reflect code-deploy mode with the chosen entry point and runtime, and that a .agentignore file controls ZIP packaging." - "Take a screenshot of the completed init output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index 93e79474fb8..cfc34c166fe 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -1,12 +1,13 @@ # Tier 1 (auth, scaffold only) — init from a manifest with explicit --agent-name and --model. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Also requires GitHub login: `gh auth login` (manifest download can fall back to # the gh CLI when the anonymous GitHub API is rate-limited). The pre hook fails # fast if gh is not authenticated. # Verifies that the override flags are honored in the generated files. name: "init-flags-agent-name-model" -command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name trangevi-qa-named-agent-{instance} --model gpt-4.1-mini" +command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" cwd: "~/working/azd-agents-t1-flags-{instance}" env: @@ -23,15 +24,15 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the manifest to download and parse." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "Accept any remaining model defaults (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Verify the overrides: confirm agent.yaml records the Foundry agent name as 'trangevi-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model 'gpt-4.1-mini' — the values passed via flags, not the manifest defaults." + - "Verify the overrides: confirm agent.yaml records the Foundry agent name as '{prefix}-qa-named-agent-{instance}' (the flag value passed via --agent-name) and the model '{model}' — the values passed via flags, not the manifest defaults." - "Take a screenshot of the completed init output." - "STOP here — do NOT run 'azd provision'. Report a finding if --agent-name or --model is ignored, or if the wizard still prompts for these values despite the flags." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index bc8529c8853..a34ed68a941 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from existing code in the current directory. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Precondition: the cwd should already contain agent source code (and ideally an # agent manifest). The pre hooks seed a committed Python fixture so this is # guaranteed and the run is idempotent. Override the fixture location with @@ -21,14 +22,14 @@ pre: name: "seed from-code agent fixture (app.py + requirements.txt)" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "At the first 'How do you want to initialize your agent?' prompt, select 'Use the code in the current directory' (this is the from-code flow; there is no --from-code flag)." - "Wait for the tool to inspect the current directory and treat its code as the agent source." - "If an existing agent manifest is detected, confirm that you want to reuse it (answer yes / confirm)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "If asked to select a model, choose 'gpt-4.1-mini' and accept the remaining model defaults." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "If asked to select a model, choose '{model}' and accept the remaining model defaults." - "Wait for initialization to complete — look for 'Next:' in the output." - "Verify the scaffold: confirm azure.yaml was created/updated to reference the local code as an azure.ai.agent service, and that a .agentignore file was generated." - "Take a screenshot of the completed init output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index fe28a2566fa..155477a364d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from an existing agent manifest URL. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. # Also requires GitHub login: `gh auth login`. Downloading the manifest (and its # sibling files) from GitHub falls back to the gh CLI when the anonymous GitHub # API is rate-limited, which would otherwise drop into an interactive gh login @@ -23,14 +24,14 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "Wait for the tool to fetch and parse the manifest from the provided URL." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini' (or accept the manifest's model if one is pinned)." - - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}' (or accept the manifest's model if one is pinned)." + - "Accept the defaults for any remaining model prompts (version, SKU, capacity). If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and the agent.yaml reflects the manifest's agent definition." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index a9a2dbd69b8..842de2de13a 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -1,6 +1,7 @@ # Tier 1 (auth, scaffold only) — init from a C#/.NET template, stop before provision. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Does NOT run `azd provision`; no cost incurred. +# Requires Azure login (see README "Authentication"). Does NOT run `azd provision`; +# no cost incurred. All `{name}` placeholders below come from the merged profile. name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet-{instance}" @@ -15,16 +16,16 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select C# / .NET as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that .NET project files were generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index a8516b86b4f..b6258a96d07 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -1,7 +1,10 @@ # Tier 1 (auth, scaffold only) — init from a Python template, stop before provision. # -# Requires Azure login (see README "Authentication"): `az login --tenant azdaiagent.onmicrosoft.com`. Reads subscriptions/Foundry projects but does NOT -# run `azd provision`, so no resources are created and no cost is incurred. +# Requires Azure login (see README "Authentication"). Reads subscriptions/Foundry +# projects but does NOT run `azd provision`, so no resources are created and no +# cost is incurred. All `{name}` placeholders below come from the merged +# profile (`profile.yaml` + `profile.local.yaml`); the orchestrator passes them +# as `session_vars` to every MCP call. name: "init-template-python" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-python-{instance}" @@ -16,16 +19,16 @@ pre: name: "reset working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' and suffixed with this run's instance id '-{instance}' (e.g. 'trangevi-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' and suffixed with this run's instance id '-{instance}' (e.g. '{prefix}-basic-responses-{instance}') so concurrent instances don't collide on resource names. Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Pick the first starter template in the list." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, choose to create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - "Verify the scaffold: confirm azure.yaml exists and references an azure.ai.agent service, and that an agent.yaml was generated." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 8b8bada67dd..039c7acaea7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -2,10 +2,13 @@ # # ⚠️ Incurs Azure cost. Run this FIRST. `init` runs in ~/working/azd-agents-shared # and scaffolds the project into a subdirectory named after the agent, so the -# deployed project lives in ~/working/azd-agents-shared/trangevi-basic-responses -# and is reused by the targeted scenarios. The agent name MUST be exactly -# 'trangevi-basic-responses' so that subdirectory path is deterministic. Run -# 2Z-teardown-down.yaml LAST to clean up. +# deployed project lives in ~/working/azd-agents-shared/{shared_agent_name} +# (where {shared_agent_name} = {prefix}-{shared_agent_suffix}, e.g. +# "alice-basic-responses") and is reused by the targeted scenarios. The agent +# name MUST be exactly that value so the subdirectory path is deterministic +# and the reuse scenarios can find it. Run 2Z-teardown-down.yaml LAST to +# clean up. All `{name}` placeholders come from the merged profile +# (`profile.yaml` + `profile.local.yaml`). name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" @@ -19,7 +22,7 @@ env: # nothing to tear down). Re-using a clean path also avoids the resource-name hash # collision in issue #8360. pre: - - run: "if [ -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml ]; then (cd ~/working/azd-agents-shared/trangevi-basic-responses && azd down --force --purge); fi" + - run: "if [ -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml ]; then (cd ~/working/azd-agents-shared/{shared_agent_name} && azd down --force --purge); fi" cwd: "~/working/azd-agents-shared" name: "tear down any leftover deployed agent" continue_on_error: true @@ -29,21 +32,21 @@ pre: name: "clear the shared working dir" goals: - - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with 'trangevi-' (e.g. 'trangevi-basic-responses'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." + - "RESOURCE NAMING: whenever you are prompted to NAME a new Azure resource you are creating (Foundry project/account, azd environment, agent, model deployment, or resource group), give it a name prefixed with '{prefix}-' (e.g. '{shared_agent_name}'). Some fields lowercase the value and replace invalid characters with hyphens; that normalization is expected. If a name prompt comes pre-filled with a default value, CLEAR it first (select-all then delete, or backspaces) before typing so your name replaces the default instead of appending to it." - "When asked how to initialize, select 'Start new from a template'." - "Select Python as the language." - "Select the 'Basic Responses' template from the list." - - "When prompted for the AGENT NAME, set it to EXACTLY 'trangevi-basic-responses' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named 'trangevi-basic-responses'." + - "When prompted for the AGENT NAME, set it to EXACTLY '{shared_agent_name}' (clear any pre-filled default first, then type it). This exact name is REQUIRED: init scaffolds the project into a subdirectory named after the agent, and the targeted reuse scenarios depend on that subdirectory being named '{shared_agent_name}'." - "When asked how to deploy, select 'Container' (hosted agent)." - - "If asked to select an Azure AI Foundry project, create a new one, naming it with the 'trangevi-' prefix, and follow the prompts." - - "If asked to select a subscription, search for and select the 'azd ai agent development' subscription." - - "If asked for a location/region, select 'East US 2'." - - "When asked to select a model, choose 'gpt-4.1-mini'." - - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a 'trangevi-'-prefixed name." + - "If asked to select an Azure AI Foundry project, create a new one, naming it with the '{prefix}-' prefix, and follow the prompts." + - "If asked to select a subscription, search for and select the '{subscription}' subscription." + - "If asked for a location/region, select '{region}'." + - "When asked to select a model, choose '{model}'." + - "Accept the defaults for model version, SKU, capacity. If prompted for a model deployment name, use a '{prefix}-'-prefixed name." - "If asked for container/resource size, select 'Small'." - "Wait for initialization to complete — look for 'Next:' or a success message." - - "Change directory into the new 'trangevi-basic-responses' subdirectory (run 'cd trangevi-basic-responses') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." - - "Run 'azd provision' (from inside the 'trangevi-basic-responses' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." + - "Change directory into the new '{shared_agent_name}' subdirectory (run 'cd {shared_agent_name}') — init scaffolds the project into a subdirectory named after the agent, so azure.yaml lives there, not in the current directory." + - "Run 'azd provision' (from inside the '{shared_agent_name}' subdirectory) and wait for it to succeed. This provisions the Azure infrastructure (Foundry project/account, model deployment, etc.) but does NOT yet deploy the agent." - "After provision succeeds, run 'azd deploy' (from the same subdirectory) and wait for it to succeed. This deploys the agent to the provisioned infrastructure — it is a required, separate step from provision before the agent can be shown or invoked." - "After deploy, run 'azd ai agent show' and note the agent name and endpoint URL — record these for the targeted scenarios." - "Take a screenshot of the successful provision, deploy, and 'show' output." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index dc1ae9b47ad..d71400568c9 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show-json" command: "azd ai agent show --output json" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index 4268464f912..78e0a131bdc 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "show" command: "azd ai agent show" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 79766bfe9f4..74109e0c82f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-input-file" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index d56211ce4e6..839c37dea8b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -8,12 +8,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-session-vs-conversation" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 77c0f9d798e..0f42731b15c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index 1bf3ef57e6c..2b633eef537 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -4,12 +4,12 @@ # Targets the `sessions` command group end-to-end in one run. name: "sessions-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index dbb81bfa47a..6d1ebbcfa92 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -5,12 +5,12 @@ # Note: file operations target a session — run an invoke first if no session exists. name: "files-lifecycle" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index deff52e7d42..479db222209 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists to stream logs from. name: "monitor-console" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 85b64e0fbfe..8b6ebb63acf 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -4,12 +4,12 @@ # and at least one invoke has happened so a session exists. name: "monitor-system" command: "bash" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 9b970cd50ce..705f2ccee21 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "endpoint-update" command: "azd ai agent endpoint update" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true # `endpoint update` reads agent_endpoint/agent_card from agent.yaml and errors @@ -16,14 +16,14 @@ pre: # defines neither, so inject a minimal agent_card (idempotently) before the run # so there is something to patch. - run: | - f="$(find ~/working/azd-agents-shared/trangevi-basic-responses -name agent.yaml | head -1)" + f="$(find ~/working/azd-agents-shared/{shared_agent_name} -name agent.yaml | head -1)" if [ -n "$f" ] && ! grep -q '^agent_card:' "$f"; then - printf '\nagent_card:\n description: "trangevi endpoint-update test card"\n skills:\n - id: "trangevi-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" + printf '\nagent_card:\n description: "{prefix} endpoint-update test card"\n skills:\n - id: "{prefix}-echo"\n name: "Echo"\n description: "Echoes input back"\n' >> "$f" echo "Injected agent_card into $f" else echo "agent_card already present or agent.yaml not found" fi - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "inject agent_card so there is something to patch" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index e0b10da49d9..86d9d66a112 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -5,7 +5,7 @@ # agent, a second invokes it with --local. name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is @@ -21,8 +21,8 @@ notes: | # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index 166313fae5b..af2dea47182 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -3,12 +3,12 @@ # Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: - - run: "test -f ~/working/azd-agents-shared/trangevi-basic-responses/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" - cwd: "~/working/azd-agents-shared/trangevi-basic-responses" + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" name: "assert shared agent is deployed" continue_on_error: true diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index 1432baa5212..45e9eb6d4b1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -4,7 +4,7 @@ # shared agent and its Azure resources to stop incurring cost. name: "teardown-down" command: "azd down --force --purge" -cwd: "~/working/azd-agents-shared/trangevi-basic-responses" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 2539711b89b..e89a211e8cd 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -1,4 +1,4 @@ - + # `azd ai agent` — cli-interactive-tester scenarios Goal-based scenarios for driving the `azure.ai.agents` extension through the @@ -8,14 +8,37 @@ strict `goals:` list format so the run is repeatable and reviewable. ## How to run -Register the cli-interactive-tester MCP server (see its README), then ask -Copilot CLI to load a scenario and accomplish its goals, e.g.: +Register the cli-interactive-tester MCP server (see its README), then +**bootstrap your profile** (one-time, per checkout — see [Profile / overrides](#profile--overrides)): + +```sh +cd cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` and `subscription` at minimum +``` + +Then ask Copilot CLI to load a scenario and accomplish its goals. The +orchestrator must **load both profile files, merge them (local overrides +shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass +the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, +`start_session`, and `run_post_hooks` call** — the scenario YAMLs reference +those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, +`{tenant}` (optional), and `{shared_agent_name}` placeholders. Example +prompt: ``` -Use the cli-interactive-tester to load the scenario at -tests/cli-interactive-tester-scenarios/00-version.yaml. If it declares pre hooks, -run them first; then start the session, accomplish the goals, take screenshots at -each step, and run any post hooks after finishing. +Use the cli-interactive-tester to drive the scenario at +tests/cli-interactive-tester-scenarios/00-version.yaml. + +First, read tests/cli-interactive-tester-scenarios/profile.yaml and +profile.local.yaml and merge them (local overrides shared); also derive +shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call. + +If the scenario declares pre hooks, run them first; then start the session, +accomplish the goals, take screenshots at each step, and run any post hooks +after finishing. ``` Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting @@ -34,6 +57,12 @@ a set of test scenarios for the cli-interactive-tester. I want you to use the cl if the scenario declares pre or post hooks, run them before/after the session, and take screenshots at each step. +First, read tests/cli-interactive-tester-scenarios/profile.yaml and profile.local.yaml and merge +them (local overrides shared); also derive shared_agent_name = "{prefix}-{shared_agent_suffix}". +Pass the merged map as session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call — the scenarios reference {prefix}, {subscription}, {region}, {model}, +{tenant} (optional), and {shared_agent_name} placeholders. + I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make @@ -69,8 +98,10 @@ Implications: single shared `~/working/azd-agents-shared` dir for all Tier 2 scenarios so they operate on the same deployed agent. `20-setup` runs `init` in that shared dir, which scaffolds the project into a subdirectory named after the agent, so the - deployed project actually lives in `~/working/azd-agents-shared/trangevi-basic-responses`; - the reuse and teardown scenarios run with that subdirectory as their `cwd`. + deployed project actually lives in `~/working/azd-agents-shared/{shared_agent_name}` + (where `{shared_agent_name} = {prefix}-{shared_agent_suffix}` from your + [profile](#profile--overrides), e.g. `alice-basic-responses`); the reuse and + teardown scenarios run with that subdirectory as their `cwd`. On macOS/Linux these are simply native paths (no WSL involved). @@ -100,19 +131,21 @@ opens a **separate browser window** for account selection that requires human interaction outside the terminal the agent controls. Treat auth as a one-time manual prerequisite, not a scenario step. -Inside WSL, a human runs: +Inside WSL, a human runs (substituting `{tenant}` and `{subscription}` with +the values from their [profile](#profile--overrides) — omit `--tenant` +entirely if `tenant` isn't set in `profile.local.yaml`): ``` -az login --tenant azdaiagent.onmicrosoft.com +az login --tenant {tenant} # or just `az login` if {tenant} is unset ``` This opens the interactive sign-in flow and then: 1. **Browser account selection** — a separate browser window opens; the human - picks the account in the `azdaiagent.onmicrosoft.com` tenant. (The agent - cannot do this.) + picks the account in the `{tenant}` tenant (or any tenant, if `{tenant}` + isn't set). The agent cannot do this. 2. **Subscription selection** — back in the terminal, select the - `azd ai agent development` subscription. + `{subscription}` subscription. Tier 0 (`00-`) scenarios need no auth. Run this `az login` step once per WSL session **before** asking the agent to drive any Tier 1/Tier 2 scenario; all of @@ -161,7 +194,7 @@ advantage of both where it's safe. - **Single-instance by design:** the **Tier 2 reuse scenarios** (`21-`…`2A-`), plus `20-setup` and `2Z-teardown`, all share the one deployed agent under `~/working/azd-agents-shared` (the project itself lives in the - `trangevi-basic-responses` subdirectory created by `20-setup`). They are + `{shared_agent_name}` subdirectory created by `20-setup`). They are **not** parameterized with `{instance}` (doing so would break the shared-agent assumption) and should be run serially. @@ -305,8 +338,8 @@ Provisions real resources. **Run order matters:** All Tier 2 scenarios share one working tree under `~/working/azd-agents-shared` so they operate on the same deployed agent. `20-setup` runs `init` there, which -scaffolds the project into the `trangevi-basic-responses` subdirectory; the -reuse and teardown scenarios run with `~/working/azd-agents-shared/trangevi-basic-responses` +scaffolds the project into the `{shared_agent_name}` subdirectory; the +reuse and teardown scenarios run with `~/working/azd-agents-shared/{shared_agent_name}` as their `cwd`. | File | Targets | @@ -326,18 +359,57 @@ as their `cwd`. | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | +## Profile / overrides + +Developer- and environment-specific values (subscription, region, model, +resource-name prefix, optional tenant) are **not** hardcoded in the scenario +YAMLs. Instead, the scenarios reference them via `{name}` placeholders, and +the orchestrator supplies the values as `session_vars` on every tester call. + +Two files in this directory drive the values: + +| File | Tracked? | Contents | Notes | +|---|---|---|---| +| `profile.yaml` | ✅ checked in | repo-shared defaults | `region`, `model`, `shared_agent_suffix` | +| `profile.local.yaml` | ❌ gitignored | per-developer / per-CI overrides | required: `prefix`, `subscription`. optional: `tenant` (no default) | +| `profile.local.yaml.example` | ✅ checked in | starter template | copy to `profile.local.yaml` and edit | + +Variables exposed to scenarios via `session_vars`: + +| Placeholder | Source | Default | Notes | +|---|---|---|---| +| `{prefix}` | `profile.local.yaml` | **required** | resource-name prefix; should be lowercase + hyphen-friendly so `sanitizeAgentName` doesn't mutate it | +| `{subscription}` | `profile.local.yaml` | **required** | subscription display name | +| `{tenant}` | `profile.local.yaml` | optional, no default | only consumed by the `az login` guidance above; when unset, drop `--tenant` and rely on the user's default tenant | +| `{region}` | `profile.yaml` | `East US 2` | | +| `{model}` | `profile.yaml` | `gpt-4.1-mini` | cheap/fast for tests | +| `{shared_agent_suffix}` | `profile.yaml` | `basic-responses` | | +| `{shared_agent_name}` | derived by orchestrator | `{prefix}-{shared_agent_suffix}` | Tier 2 subdirectory name — orchestrator must compute and pass alongside the others | + +**Bootstrap (one-time per checkout):** + +```sh +cp profile.local.yaml.example profile.local.yaml +# edit profile.local.yaml — set `prefix` (lowercase, hyphen-friendly) and `subscription` +``` + +The orchestrator must load both files, merge (local overrides shared), derive +`shared_agent_name`, and pass the merged map as `session_vars=` on every +`load_scenario` / `run_pre_hooks` / `start_session` / `run_post_hooks` call. +Failing to thread `session_vars` leaves `{prefix}` etc. unresolved in goals and +the run will execute against literal placeholder strings. + ## Conventions -- **Subscription**: `azd ai agent development` -- **Region**: `East US 2` -- **Model**: `gpt-4.1-mini` (cheap/fast for testing) -- **Resource name prefix**: every newly created Azure resource (Foundry +- **Tunable values** (subscription, region, model, prefix, tenant) come from + the profile pair above — see [Profile / overrides](#profile--overrides). +- **Resource naming**: every newly created Azure resource (Foundry project/account, azd environment, agent, model deployment, resource group) is - named with a `trangevi-` prefix (and, in parallel-ready Tier 1 scenarios, a - `-{instance}` suffix) so test resources are easy to identify, keep distinct - across concurrent runs, and clean up. Note that some fields lowercase the value - and replace invalid characters with hyphens — that normalization is expected - (see `sanitizeAgentName` in the extension). + named with the `{prefix}-` value from your profile (and, in parallel-ready + Tier 1 scenarios, a `-{instance}` suffix) so test resources are easy to + identify, keep distinct across concurrent runs, and clean up. Note that some + fields lowercase the value and replace invalid characters with hyphens — that + normalization is expected (see `sanitizeAgentName` in the extension). - `command:` invokes the installed extension as `azd ai agent …`. - Init scenarios set `env: AZD_DISABLE_AGENT_DETECT: "1"` to disable agent auto-detection prompts. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example new file mode 100644 index 00000000000..234b8f217ff --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example @@ -0,0 +1,27 @@ +# Per-developer / per-CI scenario profile — identifying values. +# +# Bootstrap: +# cp profile.local.yaml.example profile.local.yaml +# # then edit profile.local.yaml +# +# profile.local.yaml is gitignored. CI populates it from pipeline secrets. +# Any key set here overrides the same key in profile.yaml. + +# REQUIRED. Prefix applied to every newly created Azure resource (Foundry +# project/account, azd environment, agent, model deployment, resource group) +# so test resources are easy to identify and clean up. Use lowercase and +# hyphens — some fields normalize the value (sanitizeAgentName lowercases +# and replaces invalid characters), and a clean prefix avoids surprises. +# Concurrent runs of the same Tier 1 scenario automatically suffix this +# with -{instance} to keep parallel resource names distinct. +prefix: "your-alias" + +# REQUIRED. Display name of the Azure subscription to select when scenarios +# prompt for one. Tier 1 and Tier 2 scenarios both read this. +subscription: "azd ai agent development" + +# OPTIONAL. Tenant for `az login --tenant ` (and `gh auth login`). +# When unset, the driving agent omits the --tenant flag entirely and relies +# on your default tenant. Set this only if you actually need to scope to a +# specific tenant. +# tenant: "azdaiagent.onmicrosoft.com" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml new file mode 100644 index 00000000000..30ef5c004d4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.yaml @@ -0,0 +1,25 @@ +# Repo-shared scenario profile — defaults for non-identifying values. +# +# This file is checked in. Identifying values (prefix, subscription, optional +# tenant) live in profile.local.yaml, which is gitignored and bootstrapped +# from profile.local.yaml.example. +# +# The driving agent loads both files (local overrides this one), validates +# that the required keys are present, derives `shared_agent_name` = +# "-", and then passes the merged map as +# `session_vars` to EVERY MCP tool call (load_scenario, run_pre_hooks, +# start_session, run_post_hooks) — see the "Profile / overrides" section +# of README.md. + +# Default Azure region for new resources created by Tier 1/2 scenarios. +region: "East US 2" + +# Default model deployment chosen during init / referenced by --model flags. +# gpt-4.1-mini is cheap and fast — appropriate for tests. +model: "gpt-4.1-mini" + +# Suffix appended to {prefix} to form the Tier 2 shared agent's name (and +# therefore the subdir name that `azd ai agent init` scaffolds into under +# ~/working/azd-agents-shared/). The shared name is exposed to scenarios +# as {shared_agent_name}. +shared_agent_suffix: "basic-responses" From d3cd9b5007066aa5c6a4fd2940b719f9140a516b Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:03:46 -0700 Subject: [PATCH 10/13] Add tags Signed-off-by: trangevi --- .../00-doctor-empty-dir.yaml | 1 + .../00-doctor-local-only.yaml | 1 + .../00-help-root.yaml | 1 + .../00-init-picker-navigation.yaml | 1 + .../00-init-validate-mutually-exclusive.yaml | 1 + .../00-init-validate-no-prompt-missing.yaml | 1 + .../00-sample-list-json-filters.yaml | 1 + .../00-sample-list-text.yaml | 1 + .../00-version.yaml | 1 + .../10-init-deploy-mode-code.yaml | 1 + .../10-init-flags-agent-name-model.yaml | 1 + .../10-init-from-code.yaml | 1 + .../10-init-from-manifest-url.yaml | 1 + .../10-init-template-dotnet.yaml | 1 + .../10-init-template-python.yaml | 1 + .../20-setup-deploy-shared-agent.yaml | 1 + .../21-show-json.yaml | 1 + .../21-show.yaml | 1 + .../22-invoke-input-file.yaml | 1 + .../22-invoke-new-session.yaml | 1 + .../22-invoke-remote.yaml | 1 + .../23-sessions-lifecycle.yaml | 1 + .../24-files-lifecycle.yaml | 1 + .../25-monitor-console.yaml | 1 + .../25-monitor-system.yaml | 1 + .../26-endpoint-update.yaml | 1 + .../27-run-local-and-invoke-local.yaml | 1 + .../2A-doctor-provisioned-all-pass.yaml | 1 + .../2Z-teardown-down.yaml | 1 + .../README.md | 108 +++++++++++++++--- 30 files changed, 119 insertions(+), 18 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml index 53aba1a8dbc..dfd555970bb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-empty-dir.yaml @@ -2,6 +2,7 @@ name: "doctor-empty-dir" command: "azd ai agent doctor" cwd: "~/working/azd-agents-doctor-empty-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir so the "no azd project" path is exercised. # start_session recreates the dir, so removing it is enough. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 7c8662e6e3c..05c010c5529 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -2,6 +2,7 @@ name: "doctor-local-only" command: "azd ai agent doctor --local-only" cwd: "~/working/azd-agents-doctor-empty-{instance}" +tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir for a deterministic local-only run. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml index bd9db3b63bd..eca212c3f81 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-help-root.yaml @@ -2,6 +2,7 @@ name: "help-root" command: "azd ai agent --help" cwd: "/tmp" +tags: ["tier:0", "cmd:help", "parallel-safe"] goals: - "Wait for the help output to render (it includes an ASCII-art banner and a 'Usage:' section)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml index d9907dde3c8..6e11833c869 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-picker-navigation.yaml @@ -6,6 +6,7 @@ name: "init-picker-navigation" command: "azd ai agent init" cwd: "~/working/azd-agents-picker-{instance}" +tags: ["tier:0", "cmd:init", "picker", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml index 5d5df023e7a..fafecf07551 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-mutually-exclusive.yaml @@ -6,6 +6,7 @@ name: "init-validate-mutually-exclusive" command: "azd ai agent init agent.manifest.yaml -m https://example.com/agent.manifest.yaml" cwd: "~/working/azd-agents-validate-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml index d5a46e55cb1..dd8e1b3c313 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-init-validate-no-prompt-missing.yaml @@ -2,6 +2,7 @@ name: "init-validate-no-prompt-missing" command: "azd ai agent init --no-prompt" cwd: "~/working/azd-agents-validate-noprompt-{instance}" +tags: ["tier:0", "cmd:init", "negative-path", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml index a3679f6ec8b..55711345ff0 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-json-filters.yaml @@ -2,6 +2,7 @@ name: "sample-list-json-filters" command: "bash" cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Run: azd ai agent sample list --output json. Confirm the output is valid JSON (an array/object of samples)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml index 51b05c29be6..76323265ec5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-sample-list-text.yaml @@ -2,6 +2,7 @@ name: "sample-list-text" command: "azd ai agent sample list" cwd: "/tmp" +tags: ["tier:0", "cmd:sample", "parallel-safe"] goals: - "Wait for the curated sample catalog to render as human-readable text." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml index a58a88b0076..b25e9ae8392 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-version.yaml @@ -2,6 +2,7 @@ name: "version" command: "azd ai agent version" cwd: "/tmp" +tags: ["tier:0", "cmd:version", "parallel-safe"] goals: - "Wait for the command to print a version string (e.g. a value like '0.1.x-preview' or a commit-based 'vdev' build identifier)." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml index 501f2fdfb32..cd46adb445c 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-deploy-mode-code.yaml @@ -8,6 +8,7 @@ name: "init-deploy-mode-code" command: "azd ai agent init --deploy-mode code" cwd: "~/working/azd-agents-t1-code-deploy-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml index cfc34c166fe..81a29eb4e44 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-flags-agent-name-model.yaml @@ -9,6 +9,7 @@ name: "init-flags-agent-name-model" command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml --agent-name {prefix}-qa-named-agent-{instance} --model {model}" cwd: "~/working/azd-agents-t1-flags-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml index a34ed68a941..84fbcbb5b47 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-code.yaml @@ -9,6 +9,7 @@ name: "init-from-code" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-from-code-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml index 155477a364d..e326894f897 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-from-manifest-url.yaml @@ -9,6 +9,7 @@ name: "init-from-manifest-url" command: "azd ai agent init -m https://github.com/microsoft-foundry/foundry-samples/blob/main/samples/python/hosted-agents/agent-framework/responses/01-basic/agent.manifest.yaml" cwd: "~/working/azd-agents-t1-manifest-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml index 842de2de13a..74b4bf1e555 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-dotnet.yaml @@ -5,6 +5,7 @@ name: "init-template-dotnet" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-dotnet-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml index b6258a96d07..da845744fc7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/10-init-template-python.yaml @@ -8,6 +8,7 @@ name: "init-template-python" command: "azd ai agent init" cwd: "~/working/azd-agents-t1-python-{instance}" +tags: ["tier:1", "cmd:init", "parallel-safe"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml index 039c7acaea7..94234d7c8b1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/20-setup-deploy-shared-agent.yaml @@ -12,6 +12,7 @@ name: "setup-deploy-shared-agent" command: "azd ai agent init" cwd: "~/working/azd-agents-shared" +tags: ["tier:2", "cmd:init", "cmd:provision", "cmd:deploy", "serial-only"] env: AZD_DISABLE_AGENT_DETECT: "1" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml index d71400568c9..8b565759c65 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show-json.yaml @@ -4,6 +4,7 @@ name: "show-json" command: "azd ai agent show --output json" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml index 78e0a131bdc..7766378d8e7 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/21-show.yaml @@ -4,6 +4,7 @@ name: "show" command: "azd ai agent show" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:show", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml index 74109e0c82f..cb4a19233fb 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-input-file.yaml @@ -4,6 +4,7 @@ name: "invoke-input-file" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml index 839c37dea8b..8d38ef84524 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-new-session.yaml @@ -9,6 +9,7 @@ name: "invoke-session-vs-conversation" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml index 0f42731b15c..5ed85dd4c08 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/22-invoke-remote.yaml @@ -4,6 +4,7 @@ name: "invoke-remote" command: "azd ai agent invoke \"Hello! Tell me a one-sentence fun fact.\"" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:invoke", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml index 2b633eef537..4c8068b1ff4 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/23-sessions-lifecycle.yaml @@ -5,6 +5,7 @@ name: "sessions-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:sessions", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml index 6d1ebbcfa92..bed637575e2 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/24-files-lifecycle.yaml @@ -6,6 +6,7 @@ name: "files-lifecycle" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:files", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml index 479db222209..4c7aabf36d1 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-console.yaml @@ -5,6 +5,7 @@ name: "monitor-console" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml index 8b6ebb63acf..0aa08b4dd72 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/25-monitor-system.yaml @@ -5,6 +5,7 @@ name: "monitor-system" command: "bash" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:monitor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml index 705f2ccee21..0ac9922bb4b 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/26-endpoint-update.yaml @@ -4,6 +4,7 @@ name: "endpoint-update" command: "azd ai agent endpoint update" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:endpoint", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml index 86d9d66a112..325fbe0c67d 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/27-run-local-and-invoke-local.yaml @@ -6,6 +6,7 @@ name: "run-local-and-invoke-local" command: "azd ai agent run --port {agent} --no-inspector" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:run", "cmd:invoke", "serial-only"] # Reserve a free port per scenario run so parallel local runs don't collide on # the default 8088, and so the run + invoke sessions find each other (a pool is diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml index af2dea47182..2b8d289ba5f 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2A-doctor-provisioned-all-pass.yaml @@ -4,6 +4,7 @@ name: "doctor-provisioned-all-pass" command: "azd ai agent doctor" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:doctor", "serial-only"] # Precondition guard: warn (do not hard-fail) if the shared agent is not deployed. pre: diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml index 45e9eb6d4b1..c2c845f1ec5 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2Z-teardown-down.yaml @@ -5,6 +5,7 @@ name: "teardown-down" command: "azd down --force --purge" cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:down", "serial-only"] goals: - "Run 'azd down --force --purge' in the shared project directory." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index e89a211e8cd..92a50795e44 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -23,23 +23,7 @@ shared), derive `shared_agent_name = {prefix}-{shared_agent_suffix}`, and pass the merged map as `session_vars` on every `load_scenario`, `run_pre_hooks`, `start_session`, and `run_post_hooks` call** — the scenario YAMLs reference those values via `{prefix}`, `{subscription}`, `{region}`, `{model}`, -`{tenant}` (optional), and `{shared_agent_name}` placeholders. Example -prompt: - -``` -Use the cli-interactive-tester to drive the scenario at -tests/cli-interactive-tester-scenarios/00-version.yaml. - -First, read tests/cli-interactive-tester-scenarios/profile.yaml and -profile.local.yaml and merge them (local overrides shared); also derive -shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as -session_vars on every load_scenario / run_pre_hooks / start_session / -run_post_hooks call. - -If the scenario declares pre hooks, run them first; then start the session, -accomplish the goals, take screenshots at each step, and run any post hooks -after finishing. -``` +`{tenant}` (optional), and `{shared_agent_name}` placeholders. Most scenarios here declare **`pre:` hooks** (host-side setup such as resetting the working dir or seeding a fixture), and a few declare **`post:` hooks** @@ -74,6 +58,9 @@ After all of these scenarios are run, create a final result report. Create a plan to accomplish this ``` +For more selective fan-outs (e.g. "just the `init` scenarios" or "everything +in Tier 0") the tester's `list_scenarios` MCP tool filters by `tags:`. See +[Tags](#tags) below for the taxonomy and an example tag-filtered prompt. ## Paths run inside WSL (on Windows) @@ -297,7 +284,10 @@ its bugs: ## Tiers -Scenarios are organized into three tiers by cost and prerequisites. +Scenarios are organized into three tiers by cost and prerequisites. Each +scenario also carries a `tags:` list that exposes the same axes plus the +command(s) under test — see [Tags](#tags) for the full taxonomy and how to +filter via `list_scenarios`. ### Tier 0 — Offline (prefix `00-`) No Azure auth, no network resource creation. Fast and deterministic. Safe to run @@ -359,6 +349,88 @@ as their `cwd`. | `2A-doctor-provisioned-all-pass.yaml` | `doctor` (all checks pass) | | `2Z-teardown-down.yaml` | `azd down --force --purge` (TEARDOWN) | +## Tags + +Every scenario carries a top-level `tags:` list so an orchestrator can pick +subsets via the tester's `list_scenarios` MCP tool. The tool's filter is **OR +across the requested tags, case-sensitive, exact match**: a scenario matches +when its `tags` contains at least one of the requested values. + +Three namespaces are used (all lowercase, kebab-case, colon-separated for +grouping — colons are treated as ordinary characters by the filter): + +| Namespace | Values | Meaning | +|---|---|---| +| `tier:N` | `tier:0`, `tier:1`, `tier:2` | The tier the scenario belongs to (same axis as the directory's three sections above). Use this to express cost / auth profile in one tag. | +| `cmd:*` | `cmd:init`, `cmd:show`, `cmd:invoke`, `cmd:sessions`, `cmd:files`, `cmd:monitor`, `cmd:endpoint`, `cmd:run`, `cmd:doctor`, `cmd:sample`, `cmd:down`, `cmd:provision`, `cmd:deploy`, `cmd:version`, `cmd:help` | The top-level `azd ai agent` (or `azd`) command(s) the scenario exercises. Multi-command scenarios (e.g. `27-run-local-and-invoke-local` runs both `run` and `invoke --local`; `20-setup` runs `init` + `provision` + `deploy`) carry multiple `cmd:*` tags. | +| traits | `parallel-safe`, `serial-only`, `negative-path`, `picker` | `parallel-safe` ↔ `serial-only` are mutually exclusive: all Tier 0 / Tier 1 scenarios are `parallel-safe`, all Tier 2 are `serial-only`. `negative-path` flags arg-/CLI-validation scenarios that assert errors or non-zero exit codes rather than happy-path success. `picker` flags scenarios whose primary purpose is exercising interactive picker UX. | + +**Examples** (the tool's `tags:` parameter is OR across the list): + +| Goal | `list_scenarios(tags=…)` | +|---|---| +| All `init` scenarios across every tier | `["cmd:init"]` | +| Everything offline (no Azure auth, no cost) | `["tier:0"]` | +| All Tier 2 cloud scenarios | `["tier:2"]` | +| Invoke + sessions reuse scenarios | `["cmd:invoke", "cmd:sessions"]` | +| CLI arg-validation scenarios only | `["negative-path"]` | +| Everything safe to run in parallel | `["parallel-safe"]` | + +Sample prompt that uses tag filtering: + +``` +Use the cli-interactive-tester to run every `init` scenario across all tiers. + +First, call list_scenarios with root="tests/cli-interactive-tester-scenarios" +and tags=["cmd:init"] to enumerate the matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and +profile.local.yaml and merge them (local overrides shared); also derive +shared_agent_name = "{prefix}-{shared_agent_suffix}". Pass the merged map as +session_vars on every load_scenario / run_pre_hooks / start_session / +run_post_hooks call. + +For each scenario returned by list_scenarios: load it, run any pre hooks, +start the session and accomplish the goals (take screenshots at each step), +finish the session, run any post hooks. The Tier 0/1 `init` scenarios are +parallel-safe (also tagged `parallel-safe`); fan them out via fleet mode. +The Tier 2 `init` scenario (`20-setup-deploy-shared-agent`) is `serial-only` +— run it on its own and only if I confirm I want to spend on Azure resources. +``` + +You can also get copilot to generate the tags list instead of manually specifying +it. For example, if you want to run all of the scenarios to test the changes +in a PR, modify the above prompt to start with something like: + +``` +Here's a PR: https://github.com/Azure/azure-dev/pull/8532. In the +tests\cli-interactive-tester-scenarios directory, there are a set of test scenarios, +with tags to categorize what they're testing. I want you to come up with a set of +tags which, when used to select these test scenarios, would properly test the +changes made by the PR provided. + +Next, call list_scenarios with those tags, to enumerate matching scenarios. + +Then read tests/cli-interactive-tester-scenarios/profile.yaml and .... + +``` + +And, if you're running these scenarios as a part of creating or reviewing a PR, +you can ask copilot to generate a summary report and add it as a comment directly +on the pull request. + +When adding a new scenario, give it a `tags:` list that follows this +taxonomy: at minimum a `tier:N`, at least one `cmd:*`, and either +`parallel-safe` or `serial-only`. `list_scenarios` prints `tags: []` for any +file missing a `tags:` field, so an empty list in its output signals a +regression to fix. + +> `list_scenarios` walks every `*.yaml` under the directory, including +> `profile.yaml` / `profile.local.yaml` (which surface as `(unnamed)` with +> `tags: (none)`). Filter by any `tier:*` / `cmd:*` / trait tag to exclude +> them — they intentionally carry no tags because they are configuration, +> not scenarios. + ## Profile / overrides Developer- and environment-specific values (subscription, region, model, From ea2bac4de33930d9a2977e74c9eb9d7a9233b05e Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:24:51 -0700 Subject: [PATCH 11/13] Agents.md update, to direct people to the testing Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index b03e3847b98..3ff5a3e6ec9 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -38,6 +38,46 @@ replace github.com/azure/azure-dev/cli/azd => ../../ That `replace` points this extension at your local `cli/azd` checkout instead of the version in `go.mod`. Do not merge the extension with that `replace` still present. +## Interactive CLI test scenarios + +This extension ships a suite of goal-based scenarios for the +[cli-interactive-tester](https://github.com/coreai-microsoft/cli-interactive-tester) +MCP server under `tests/cli-interactive-tester-scenarios/`. They drive real +`azd ai agent` flows end-to-end (init, provision, deploy, invoke, run, sessions, +files, monitor, endpoint, doctor, down) and are organized by tier: + +- **Tier 0** — offline, no Azure auth, no cost (help, version, validation, picker UX) +- **Tier 1** — local-only with Azure auth (init flows) +- **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) + +Each scenario caries a set of tags based on what is being tested and how. +See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, +profile setup, and orchestration rules. + +### Guidance for coding agents + +These scenarios are **never run automatically** — they require the +cli-interactive-tester MCP server, a populated `profile.local.yaml`, and +(for Tier 2) real Azure resources. Do not invoke them on your own. Instead: + +1. **Surface them to the user** when you make a change that touches a + user-facing command path covered by an existing scenario (anything under + `internal/cmd/` that maps to a `cmd:*` tag, or shared helpers used by those + commands). In your summary, point the user at the relevant scenario(s) + and suggest they run the tester against the matching tag set to validate + the change. + +2. **Add or update a scenario** when your change introduces a new command, + flag, prompt, or user-visible flow — or meaningfully alters an existing + one. Place the new YAML alongside the others, follow the tagging taxonomy + documented in the scenarios README, and mention the new/changed scenario + in the PR description so reviewers know to exercise it. + +3. **Do not modify scenarios to match buggy behavior.** Scenarios are + user-facing specifications of how the command should behave; if a scenario + fails because of your change, prefer fixing the code unless the behavior + change is intentional and documented. + ## Error handling This extension uses `internal/exterrors` so the azd host can show a useful message, attach an optional suggestion, and emit stable telemetry. From 8a63c1d3363563de5c38d277b551b28780a05cd5 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:32:41 -0700 Subject: [PATCH 12/13] cspell Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/cspell.yaml | 1 + .../tests/cli-interactive-tester-scenarios/README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 2d5c10d89d6..f6a93b191c0 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -41,6 +41,7 @@ words: - CLIENTSECRET - curr - dataagent + - defaultyourvalue - envkey - exterrors - goyaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md index 92a50795e44..b0ed50c84ef 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/README.md @@ -48,7 +48,7 @@ run_post_hooks call — the scenarios reference {prefix}, {subscription}, {regio {tenant} (optional), and {shared_agent_name} placeholders. I want this run on fleet mode, to parallelize the tests as much as possible. Each of the scenarios -in tiers 0 and 1 are compleatly indpendent of each other and can be run in parallel. The scenarios +in tiers 0 and 1 are completely independent of each other and can be run in parallel. The scenarios in tier 2 however rely on a setup scenario, and the teardown scenario should be run last, so make sure to take that into account when distributing the work. I want to run all of the tests regardless of tier, and I acknowledge that tier 2 has an azure cost implication, that's fine. From 4793e9ff5e7cb4e8cf72830aad8ff1c789cd90f7 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 5 Jun 2026 14:50:54 -0700 Subject: [PATCH 13/13] PR comments Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 2 +- .../00-doctor-local-only.yaml | 4 ++-- .../profile.local.yaml.example | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index 3ff5a3e6ec9..e0d8712aed4 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -50,7 +50,7 @@ files, monitor, endpoint, doctor, down) and are organized by tier: - **Tier 1** — local-only with Azure auth (init flows) - **Tier 2** — full cloud E2E against a deployed shared agent (incurs Azure cost) -Each scenario caries a set of tags based on what is being tested and how. +Each scenario carries a set of tags based on what is being tested and how. See `tests/cli-interactive-tester-scenarios/README.md` for the tag taxonomy, profile setup, and orchestration rules. diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml index 05c010c5529..dfba27f8b89 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-doctor-local-only.yaml @@ -1,12 +1,12 @@ # Tier 0 (offline) — `doctor --local-only` skips remote checks. name: "doctor-local-only" command: "azd ai agent doctor --local-only" -cwd: "~/working/azd-agents-doctor-empty-{instance}" +cwd: "~/working/azd-agents-doctor-local-{instance}" tags: ["tier:0", "cmd:doctor", "parallel-safe"] # Guarantee an empty working dir for a deterministic local-only run. pre: - - run: "rm -rf ~/working/azd-agents-doctor-empty-{instance}" + - run: "rm -rf ~/working/azd-agents-doctor-local-{instance}" cwd: "~/working" name: "reset to an empty working dir" diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example index 234b8f217ff..e20187f0c14 100644 --- a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/profile.local.yaml.example @@ -20,7 +20,7 @@ prefix: "your-alias" # prompt for one. Tier 1 and Tier 2 scenarios both read this. subscription: "azd ai agent development" -# OPTIONAL. Tenant for `az login --tenant ` (and `gh auth login`). +# OPTIONAL. Tenant for `az login --tenant `. # When unset, the driving agent omits the --tenant flag entirely and relies # on your default tenant. Set this only if you actually need to scope to a # specific tenant.