From 75cfd4fdebc5745e1cd0b0d73a07514b54b521b2 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Thu, 3 Sep 2026 16:49:53 -0700 Subject: [PATCH 1/5] Update CHANGELOG for version 2.6.0 release, add new samples, and improve public methods generation script --- sdk/ai/azure-ai-projects/CHANGELOG.md | 9 +- .../GeneratePublicMethods.ps1 | 256 ++++++++++++++++++ sdk/ai/azure-ai-projects/PostEmitter.ps1 | 8 + sdk/ai/azure-ai-projects/api.md | 53 ---- sdk/ai/azure-ai-projects/api.metadata.yml | 2 +- .../azure-ai-projects/docs/public-methods.md | 20 +- .../agents/tools/sample_toolbox_with_shell.py | 2 +- .../sample_toolbox_with_shell_and_skill.py | 166 ++++++++++++ .../agents/tools/sample_toolbox_with_skill.py | 153 +++++++++++ .../sample_synthetic_multiturn_evaluation.py | 39 +-- .../sample_responses_model_router.py | 56 ++++ 11 files changed, 667 insertions(+), 97 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 create mode 100644 sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell_and_skill.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py create mode 100644 sdk/ai/azure-ai-projects/samples/responses/sample_responses_model_router.py diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 7a4faece3d69..c722c132ee04 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 2.6.0 (Unreleased) +## 2.6.0 (2026-09-04) ### Features Added @@ -12,11 +12,14 @@ * Added `ShellToolboxTool` and supporting container environment and network policy models, with the new `ToolboxToolType.SHELL` enum member. * Added `WebIQPreviewTool` and `WebIQPreviewToolboxTool`, with new `ToolType.WEB_IQ_PREVIEW` and `ToolboxToolType.WEB_IQ_PREVIEW` enum members. * Added the optional `external_web_access` property to `WebSearchTool` and `WebSearchToolboxTool` for disabling live internet access. +* Added preview support for Model Router, which dynamically routes each request to an appropriate model. ### Sample updates -* Added `sample_toolbox_with_shell.py`, demonstrating a Prompt Agent invoking a `ShellToolboxTool`. -* Added `sample_synthetic_multiturn_evaluation.py`, demonstrating simulation seed generation from an agent followed by multi-turn conversation simulation and evaluation. +* Added `sample_toolbox_with_shell.py` under `samples/agents/tools/`, demonstrating a Prompt Agent invoking a `ShellToolboxTool`. +* Added `sample_toolbox_with_shell_and_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using an inline Skill with a `ShellToolboxTool` through a Toolbox MCP endpoint. +* Added `sample_synthetic_multiturn_evaluation.py` under `samples/evaluations/`, demonstrating simulation seed generation from an agent followed by multi-turn conversation simulation and evaluation. +* Added `sample_responses_model_router.py` under `samples/responses/`, demonstrating Responses API calls to a Model Router deployment and inspection of the selected models for each inference turn. ### Bugs Fixed diff --git a/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 new file mode 100644 index 000000000000..a939fef58387 --- /dev/null +++ b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 @@ -0,0 +1,256 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[CmdletBinding()] +param( + [string]$PythonExecutable = "python", + [string]$OutputPath = (Join-Path $PSScriptRoot "docs\public-methods.md") +) + +$ErrorActionPreference = "Stop" +$packageRoot = $PSScriptRoot +$temporaryScript = Join-Path ([System.IO.Path]::GetTempPath()) ("generate-public-methods-{0}.py" -f [guid]::NewGuid()) + +$pythonScript = @' +from __future__ import annotations + +import inspect +import os +from pathlib import Path +import sys +from typing import Any + + +package_root = Path(sys.argv[1]).resolve() +output_path = Path(sys.argv[2]).resolve() +sys.path.insert(0, str(package_root)) +os.chdir(package_root) + +from azure.core.credentials import AccessToken +from azure.ai.projects import AIProjectClient +from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient +import azure.ai.projects as projects_package + + +class FakeCredential: + def get_token(self, *args: Any, **kwargs: Any) -> AccessToken: + return AccessToken("fake-token", 2**31) + + +class AsyncFakeCredential: + async def get_token(self, *args: Any, **kwargs: Any) -> AccessToken: + return AccessToken("fake-token", 2**31) + + +def assert_local_import() -> None: + imported_path = Path(projects_package.__file__).resolve() + if not imported_path.is_relative_to(package_root): + raise RuntimeError( + f"Expected azure.ai.projects from {package_root}, but imported {imported_path}" + ) + + +def unwrap_operation(value: Any) -> Any: + return getattr(value, "_operation", value) + + +def operation_instances(container: Any, *, exclude: set[str] | None = None) -> dict[str, Any]: + excluded = exclude or set() + operations: dict[str, Any] = {} + for name, value in vars(container).items(): + if name.startswith("_") or name in excluded: + continue + operation = unwrap_operation(value) + if type(operation).__name__.endswith("Operations"): + operations[name] = operation + return operations + + +def is_handwritten_method(cls: type[Any], name: str) -> bool: + owner = next((base for base in cls.__mro__ if name in vars(base)), None) + if owner is None: + raise RuntimeError(f"Unable to find the class that defines {cls.__name__}.{name}") + source_path = inspect.getsourcefile(owner) + return source_path is not None and "_patch" in Path(source_path).name + + +def public_methods(instance: Any) -> dict[str, bool]: + methods: dict[str, bool] = {} + for name, member in inspect.getmembers(type(instance), predicate=callable): + if name.startswith("_"): + continue + methods[name] = is_handwritten_method(type(instance), name) + return methods + + +def client_methods(client: Any) -> dict[str, bool]: + included_dunders = {"__enter__", "__exit__"} + methods: dict[str, bool] = {} + for name, member in inspect.getmembers(type(client), predicate=callable): + if name.startswith("_") and name not in included_dunders: + continue + methods[name] = is_handwritten_method(type(client), name) + return methods + + +def method_label(prefix: str, name: str, handwritten: bool) -> str: + return f".{prefix}{name}{'*' if handwritten else ''}" + + +def validate_async_parity( + sync_operations: dict[str, Any], + async_operations: dict[str, Any], + group_name: str, +) -> None: + if sync_operations.keys() != async_operations.keys(): + sync_only = sorted(sync_operations.keys() - async_operations.keys()) + async_only = sorted(async_operations.keys() - sync_operations.keys()) + raise RuntimeError( + f"{group_name} sub-client mismatch; sync-only={sync_only}, async-only={async_only}" + ) + + for name in sorted(sync_operations): + sync_methods = set(public_methods(sync_operations[name])) + async_methods = set(public_methods(async_operations[name])) + if sync_methods != async_methods: + raise RuntimeError( + f"{group_name}.{name} method mismatch; " + f"sync-only={sorted(sync_methods - async_methods)}, " + f"async-only={sorted(async_methods - sync_methods)}" + ) + + +def table(lines: list[str], rows: list[tuple[str, str, int]]) -> None: + lines.extend( + [ + "| Subclient | Class Name | Methods Count |", + "| --- | --- | --- |", + ] + ) + lines.extend(f"| `{name}` | {class_name} | {count} |" for name, class_name, count in rows) + + +assert_local_import() +endpoint = "https://example.services.ai.azure.com/api/projects/example" +sync_client = AIProjectClient(endpoint=endpoint, credential=FakeCredential(), allow_preview=True) +async_client = AsyncAIProjectClient(endpoint=endpoint, credential=AsyncFakeCredential(), allow_preview=True) + +try: + sync_stable = operation_instances(sync_client, exclude={"beta"}) + async_stable = operation_instances(async_client, exclude={"beta"}) + sync_beta = operation_instances(sync_client.beta) + async_beta = operation_instances(async_client.beta) + + validate_async_parity(sync_stable, async_stable, "stable") + validate_async_parity(sync_beta, async_beta, "beta") + + stable_methods = {name: public_methods(instance) for name, instance in sync_stable.items()} + beta_methods = {name: public_methods(instance) for name, instance in sync_beta.items()} + direct_methods = client_methods(sync_client) + + stable_count = sum(len(methods) for methods in stable_methods.values()) + beta_count = sum(len(methods) for methods in beta_methods.values()) + total_count = len(direct_methods) + stable_count + beta_count + + lines = [ + "# Public AIProjectClient methods", + "", + "", + "", + "This document lists all public methods available on `AIProjectClient` and its sub-clients. " + "Overload methods are not counted. Only synchronous methods are counted (but each one has an " + "equivalent asynchronous method).", + "", + "## Summary", + "", + f"There are a total of {total_count} unique public methods:", + "", + f"- {len(direct_methods)} stable methods on the client", + f"- {stable_count} stable methods on top-level sub-clients", + f"- {beta_count} beta methods on nested beta sub-clients", + "", + "### Top-level sub-clients (stable operations)", + "", + ] + + stable_rows = [ + (name, type(sync_stable[name]).__name__, len(stable_methods[name])) + for name in sorted(sync_stable) + ] + table(lines, stable_rows) + lines.extend(["", "### Nested sub-clients (beta operations)", ""]) + beta_rows = [ + (f"beta.{name}", type(sync_beta[name]).__name__, len(beta_methods[name])) + for name in sorted(sync_beta) + ] + table(lines, beta_rows) + + lines.extend( + [ + "", + "## Stable methods on the client", + "", + "Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method.", + "", + "```text", + ] + ) + lines.extend(method_label("", name, direct_methods[name]) for name in sorted(direct_methods)) + lines.extend( + [ + "```", + "", + "## Stable methods on top-level sub clients", + "", + "Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method.", + "", + "```text", + ] + ) + for index, subclient_name in enumerate(sorted(stable_methods)): + if index: + lines.append("") + methods = stable_methods[subclient_name] + lines.extend( + method_label(f"{subclient_name}.", name, methods[name]) for name in sorted(methods) + ) + lines.extend( + [ + "```", + "", + "## Beta methods on nested sub-clients", + "", + "Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method.", + "", + "```text", + ] + ) + for index, subclient_name in enumerate(sorted(beta_methods)): + if index: + lines.append("") + methods = beta_methods[subclient_name] + lines.extend( + method_label(f"beta.{subclient_name}.", name, methods[name]) for name in sorted(methods) + ) + lines.extend(["```", ""]) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("\n".join(lines), encoding="utf-8", newline="\n") + print( + f"Generated {output_path} with {total_count} methods " + f"({len(direct_methods)} client, {stable_count} stable, {beta_count} beta)." + ) +finally: + sync_client.close() +'@ + +try { + [System.IO.File]::WriteAllText($temporaryScript, $pythonScript, [System.Text.UTF8Encoding]::new($false)) + & $PythonExecutable $temporaryScript $packageRoot $OutputPath + if ($LASTEXITCODE -ne 0) { + throw "Public method generation failed with exit code $LASTEXITCODE." + } +} +finally { + Remove-Item $temporaryScript -Force -ErrorAction SilentlyContinue +} \ No newline at end of file diff --git a/sdk/ai/azure-ai-projects/PostEmitter.ps1 b/sdk/ai/azure-ai-projects/PostEmitter.ps1 index 4a7f6f2e519e..129b8b4bbd12 100644 --- a/sdk/ai/azure-ai-projects/PostEmitter.ps1 +++ b/sdk/ai/azure-ai-projects/PostEmitter.ps1 @@ -89,3 +89,11 @@ foreach ($f in $files) { # Finishing by running 'black' tool to format code. pip install black black --config ../../../eng/black-pyproject.toml . + +# Regenerate API review artifacts and the public method inventory. +azpysdk apistub . +$apiStubExitCode = $LASTEXITCODE +.\GeneratePublicMethods.ps1 +if ($apiStubExitCode -ne 0) { + throw "API stub generation failed with exit code $apiStubExitCode." +} diff --git a/sdk/ai/azure-ai-projects/api.md b/sdk/ai/azure-ai-projects/api.md index e251c65f2356..56a558a575ea 100644 --- a/sdk/ai/azure-ai-projects/api.md +++ b/sdk/ai/azure-ai-projects/api.md @@ -8616,14 +8616,12 @@ namespace azure.ai.projects.models class azure.ai.projects.models.RaiConfig(_Model): - invocations_moderation: Optional[RaiInvocationModeration] rai_policy_name: str @overload def __init__( self, *, - invocations_moderation: Optional[RaiInvocationModeration] = ..., rai_policy_name: str ) -> None: ... @@ -8631,57 +8629,6 @@ namespace azure.ai.projects.models def __init__(self, mapping: Mapping[str, Any]) -> None: ... - class azure.ai.projects.models.RaiInvocationContentType(str, Enum, metaclass=CaseInsensitiveEnumMeta): - JSON = "json" - TEXT = "text" - - - class azure.ai.projects.models.RaiInvocationMode(str, Enum, metaclass=CaseInsensitiveEnumMeta): - BOTH = "both" - NON_STREAMING = "non_streaming" - STREAMING = "streaming" - - - class azure.ai.projects.models.RaiInvocationModeration(_Model): - input_content_type: Optional[Union[str, RaiInvocationContentType]] - input_paths: Optional[list[str]] - output_content_type: Optional[Union[str, RaiInvocationContentType]] - output_paths: Optional[list[str]] - response_mode: Union[str, RaiInvocationMode] - stream_selectors: Optional[list[RaiSseTextSelector]] - - @overload - def __init__( - self, - *, - input_content_type: Optional[Union[str, RaiInvocationContentType]] = ..., - input_paths: Optional[list[str]] = ..., - output_content_type: Optional[Union[str, RaiInvocationContentType]] = ..., - output_paths: Optional[list[str]] = ..., - response_mode: Union[str, RaiInvocationMode], - stream_selectors: Optional[list[RaiSseTextSelector]] = ... - ) -> None: ... - - @overload - def __init__(self, mapping: Mapping[str, Any]) -> None: ... - - - class azure.ai.projects.models.RaiSseTextSelector(_Model): - event_type: str - text_field: Optional[str] - - @overload - def __init__( - self, - *, - event_type: str, - text_field: Optional[str] = ... - ) -> None: ... - - @overload - def __init__(self, mapping: Mapping[str, Any]) -> None: ... - - class azure.ai.projects.models.RankerVersionType(str, Enum, metaclass=CaseInsensitiveEnumMeta): AUTO = "auto" DEFAULT_2024_11_15 = "default-2024-11-15" diff --git a/sdk/ai/azure-ai-projects/api.metadata.yml b/sdk/ai/azure-ai-projects/api.metadata.yml index b118de70aecf..ea465463f31b 100644 --- a/sdk/ai/azure-ai-projects/api.metadata.yml +++ b/sdk/ai/azure-ai-projects/api.metadata.yml @@ -1,4 +1,4 @@ -apiMdSha256: 565d431df41079ca560a9b1c4fda23578362e831b819fc37dd1db36cd4ab0b82 +apiMdSha256: 5d405fa9c99c19e09c66ec01f262504083883917fbd849199f1a58de4a21c81e packageVersion: 2.6.0 parserVersion: 0.3.31 pythonVersion: 3.12.10 diff --git a/sdk/ai/azure-ai-projects/docs/public-methods.md b/sdk/ai/azure-ai-projects/docs/public-methods.md index fbf9595f4226..c057b80a9581 100644 --- a/sdk/ai/azure-ai-projects/docs/public-methods.md +++ b/sdk/ai/azure-ai-projects/docs/public-methods.md @@ -1,10 +1,13 @@ # Public AIProjectClient methods + + This document lists all public methods available on `AIProjectClient` and its sub-clients. Overload methods are not counted. Only synchronous methods are counted (but each one has an equivalent asynchronous method). ## Summary There are a total of 157 unique public methods: + - 5 stable methods on the client - 58 stable methods on top-level sub-clients - 94 beta methods on nested beta sub-clients @@ -12,7 +15,7 @@ There are a total of 157 unique public methods: ### Top-level sub-clients (stable operations) | Subclient | Class Name | Methods Count | -|-----------|------------|----------------| +| --- | --- | --- | | `agents` | AgentsOperations | 26 | | `connections` | ConnectionsOperations | 3 | | `datasets` | DatasetsOperations | 9 | @@ -25,7 +28,7 @@ There are a total of 157 unique public methods: ### Nested sub-clients (beta operations) | Subclient | Class Name | Methods Count | -|-----------|------------|----------------| +| --- | --- | --- | | `beta.agent_insight_monitors` | BetaAgentInsightMonitorsOperations | 13 | | `beta.agents` | BetaAgentsOperations | 5 | | `beta.datasets` | BetaDatasetsOperations | 5 | @@ -39,12 +42,11 @@ There are a total of 157 unique public methods: | `beta.schedules` | BetaSchedulesOperations | 6 | | `beta.skills` | BetaSkillsOperations | 11 | - ## Stable methods on the client -Alphabetically sorted. An asterisk at the end of the method name means is a hand-written method. +Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method. -``` +```text .__enter__ .__exit__ .close @@ -54,9 +56,9 @@ Alphabetically sorted. An asterisk at the end of the method name means is a hand ## Stable methods on top-level sub clients -Alphabetically sorted. An asterisk at the end of the method name means is a hand-written method. +Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method. -``` +```text .agents.create_session .agents.create_version* .agents.create_version_from_code* @@ -126,9 +128,9 @@ Alphabetically sorted. An asterisk at the end of the method name means is a hand ## Beta methods on nested sub-clients -Alphabetically sorted. An asterisk at the end of the method name means is a hand-written method. +Alphabetically sorted. An asterisk at the end of the method name means it is a hand-written method. -``` +```text .beta.agent_insight_monitors.begin_create_run* .beta.agent_insight_monitors.cancel_run .beta.agent_insight_monitors.create diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell.py index 2436edd9e24a..5f4b176caaa8 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell.py @@ -21,7 +21,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.3.0" python-dotenv openai + pip install "azure-ai-projects>=2.6.0" python-dotenv openai Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell_and_skill.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell_and_skill.py new file mode 100644 index 000000000000..3501e98e24ba --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shell_and_skill.py @@ -0,0 +1,166 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates how to put a shell tool and an inline Skill in a + Toolbox and use them with a Prompt Agent. The Skill instructs the shell to + print "Welcome to shell tool" before reporting the Python version and the + working-directory contents of its auto-provisioned, network-isolated container. + + The sample downloads the persisted Skill's `SKILL.md` into the Prompt Agent + instructions. The agent reaches the shell through an `MCPTool` pointed at + the Toolbox's versioned `/mcp` URL. The sample prints the tools exposed by + the MCP server, each shell command's arguments and output, and the agent's + final response. + +USAGE: + python sample_toolbox_with_shell_and_skill.py + + Before running the sample: + + pip install "azure-ai-projects>=2.6.0" python-dotenv openai + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Microsoft Foundry portal. + 2) FOUNDRY_MODEL_NAME - The deployment name of the AI model, as found under the "Name" column in + the "Models + endpoints" tab in your Microsoft Foundry project. + 3) FOUNDRY_AGENT_NAME - Optional. The name of the AI agent. If not set, defaults to "MyAgent". +""" + +import io +import os +import zipfile +from dotenv import load_dotenv +from util import create_version_with_endpoint +from azure.core.exceptions import ResourceNotFoundError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + MCPTool, + PromptAgentDefinition, + ShellToolboxTool, + SkillInlineContent, + ToolSearchToolboxTool, + ToolboxSkillReference, + ToolboxShellContainerAutoEnvironment, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + +SKILL_NAME = "sandbox-environment-inspection" +TOOLBOX_NAME = "toolbox_with_shell_tool" +TOOLBOX_MCP_LABEL = "shell-toolbox" +agent_name = os.environ.get("FOUNDRY_AGENT_NAME") or "MyAgent" + + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + project_client.get_openai_client(agent_name=agent_name) as openai_client, +): + try: + project_client.toolboxes.delete(TOOLBOX_NAME) + print(f"Deleted pre-existing toolbox `{TOOLBOX_NAME}`") + except ResourceNotFoundError: + pass + + try: + project_client.beta.skills.delete(SKILL_NAME) + print(f"Deleted pre-existing skill `{SKILL_NAME}`") + except ResourceNotFoundError: + pass + + skill_version = project_client.beta.skills.create( + name=SKILL_NAME, + inline_content=SkillInlineContent( + description="Inspect the runtime environment of a sandboxed shell container.", + instructions=( + "When asked to inspect the sandbox environment, first run " + "`printf 'Welcome to shell tool\\n'` with the shell tool. Then run the relevant " + "commands. For Python version and working-directory contents, run " + "`python --version`, `pwd`, and `ls -la`. Report the exact command output." + ), + ), + ) + print(f"Created skill `{skill_version.name}` (version {skill_version.version}).") + + skill_archive = b"".join( + project_client.beta.skills.download_version(name=skill_version.name, version=skill_version.version) + ) + with zipfile.ZipFile(io.BytesIO(skill_archive)) as archive: + skill_instructions = archive.read("SKILL.md").decode("utf-8") + print(f"Loaded instructions from skill `{skill_version.name}` version {skill_version.version}.") + + shell_tool = ShellToolboxTool( + name="shell", + description="Runs shell commands in a sandboxed container.", + environment=ToolboxShellContainerAutoEnvironment(), + ) + + try: + toolbox_version = project_client.toolboxes.create_version( + name=TOOLBOX_NAME, + description="Toolbox with a shell tool and an environment-inspection skill.", + tools=[shell_tool, ToolSearchToolboxTool(name="skill_search")], + skills=[ToolboxSkillReference(name=skill_version.name, version=skill_version.version)], + ) + print(f"Created toolbox `{TOOLBOX_NAME}` (version {toolbox_version.version}).") + + toolbox_mcp_url = f"{endpoint}/toolboxes/{TOOLBOX_NAME}/versions/{toolbox_version.version}/mcp?api-version=v1" + token = credential.get_token("https://ai.azure.com/.default").token + + toolbox_mcp_tool = MCPTool( + server_label=TOOLBOX_MCP_LABEL, + server_url=toolbox_mcp_url, + authorization=token, + require_approval="never", + ) + + with create_version_with_endpoint( + project_client=project_client, + agent_name=agent_name, + definition=PromptAgentDefinition( + model=os.environ["FOUNDRY_MODEL_NAME"], + instructions=( + "You have a shell tool that runs commands in a sandboxed container with no " + "network access. Follow the loaded skill instructions below and report the exact " + f"command output back to the user.\n\n{skill_instructions}" + ), + tools=[toolbox_mcp_tool], + ), + ): + response = openai_client.responses.create( + input=( + "Use the sandbox environment inspection skill to determine which Python version " + "is installed and what is in the working directory." + ), + ) + + for item in response.output: + if item.type == "mcp_list_tools": + print(f"server_label={item.server_label}, tools={[tool.name for tool in (item.tools or [])]}") + elif item.type == "mcp_call": + print(f"server_label={item.server_label}, name={item.name}, error={item.error}") + print(f" arguments: {item.arguments}") + print(f" output: {item.output}") + + print(f"\nResponse: {response.output_text}") + finally: + try: + project_client.toolboxes.delete(TOOLBOX_NAME) + print(f"\nDeleted toolbox `{TOOLBOX_NAME}`") + except ResourceNotFoundError: + pass + finally: + try: + project_client.beta.skills.delete(SKILL_NAME) + print(f"Deleted skill `{SKILL_NAME}`") + except ResourceNotFoundError: + pass diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py new file mode 100644 index 000000000000..d679e6d81495 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py @@ -0,0 +1,153 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates how to create a shipping-cost Skill, include it in + a Toolbox, and use its persisted instructions with a Prompt Agent. The + Toolbox is exposed to the agent through its versioned MCP endpoint. + + Prompt Agent definitions do not have a native skill-reference field, so the + sample downloads the immutable Skill version and adds its ``SKILL.md`` + content to the agent instructions. A marker known only to the persisted + Skill proves that the agent applied those instructions. + +USAGE: + python sample_toolbox_with_skill.py + + Before running the sample: + + pip install "azure-ai-projects>=2.5.0" python-dotenv openai + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Microsoft Foundry portal. + 2) FOUNDRY_MODEL_NAME - The deployment name of the AI model, as found under the "Name" column in + the "Models + endpoints" tab in your Microsoft Foundry project. + 3) FOUNDRY_AGENT_NAME - Optional. The name of the AI agent. If not set, defaults to "MyAgent". +""" + +import io +import os +import zipfile + +from dotenv import load_dotenv +from util import create_version_with_endpoint + +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + MCPTool, + PromptAgentDefinition, + SkillInlineContent, + ToolSearchToolboxTool, + ToolboxSkillReference, +) +from azure.core.exceptions import ResourceNotFoundError +from azure.identity import DefaultAzureCredential + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ.get("FOUNDRY_AGENT_NAME") or "MyAgent" + +SKILL_NAME = "shipping-cost-skill" +SKILL_PROOF_MARKER = "SHIPPING_COST_SKILL_APPLIED" +TOOLBOX_NAME = "toolbox_with_skill_prompt_agent" +TOOLBOX_MCP_LABEL = "shipping-toolbox" + + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client(agent_name=agent_name) as openai_client, +): + try: + project_client.toolboxes.delete(TOOLBOX_NAME) + except ResourceNotFoundError: + pass + + try: + project_client.beta.skills.delete(SKILL_NAME) + except ResourceNotFoundError: + pass + + skill_version = project_client.beta.skills.create( + name=SKILL_NAME, + inline_content=SkillInlineContent( + description="Compute shipping cost for a package given weight and destination.", + instructions=( + f"Begin your answer with `{SKILL_PROOF_MARKER}`. Compute shipping cost using " + "cost (USD) = 5 + 2 * weight_kg for domestic destinations, and " + "cost (USD) = 15 + 4 * weight_kg for international destinations. " + "Always state the formula you used." + ), + metadata={"revision": "1"}, + ), + ) + print(f"Created skill `{skill_version.name}` (version {skill_version.version}).") + + try: + skill_archive = b"".join( + project_client.beta.skills.download_version(name=skill_version.name, version=skill_version.version) + ) + with zipfile.ZipFile(io.BytesIO(skill_archive)) as archive: + skill_instructions = archive.read("SKILL.md").decode("utf-8") + print(f"Loaded instructions from skill `{skill_version.name}` version {skill_version.version}.") + + toolbox_version = project_client.toolboxes.create_version( + name=TOOLBOX_NAME, + description="Toolbox exposing a shipping-cost skill to a Prompt Agent.", + tools=[ToolSearchToolboxTool(name="skill_search")], + skills=[ToolboxSkillReference(name=skill_version.name, version=skill_version.version)], + ) + print(f"Created toolbox `{toolbox_version.name}` (version {toolbox_version.version}).") + + toolbox_mcp_url = f"{endpoint}/toolboxes/{TOOLBOX_NAME}/versions/{toolbox_version.version}/mcp?api-version=v1" + toolbox_mcp_tool = MCPTool( + server_label=TOOLBOX_MCP_LABEL, + server_url=toolbox_mcp_url, + authorization=credential.get_token("https://ai.azure.com/.default").token, + require_approval="never", + ) + + with create_version_with_endpoint( + project_client=project_client, + agent_name=agent_name, + definition=PromptAgentDefinition( + model=os.environ["FOUNDRY_MODEL_NAME"], + instructions=("Follow the persisted Skill instructions below.\n\n" f"{skill_instructions}"), + tools=[toolbox_mcp_tool], + ), + ): + user_input = "Compute the shipping cost for a 3 kg package shipped domestically." + print(f"User: {user_input}") + response = openai_client.responses.create(input=user_input) + + for item in response.output: + if item.type == "mcp_list_tools": + print(f"server_label={item.server_label}, tools={[tool.name for tool in (item.tools or [])]}") + elif item.type == "mcp_call": + print(f"server_label={item.server_label}, name={item.name}, error={item.error}") + print(f" arguments: {item.arguments}") + print(f" output: {item.output}") + + if SKILL_PROOF_MARKER not in (response.output_text or ""): + raise RuntimeError("The response did not contain evidence that the skill instructions were applied.") + + print(f"Verified skill instructions with marker `{SKILL_PROOF_MARKER}`.") + print(f"\nResponse: {response.output_text}") + finally: + try: + project_client.toolboxes.delete(TOOLBOX_NAME) + print(f"\nDeleted toolbox `{TOOLBOX_NAME}`") + except ResourceNotFoundError: + pass + finally: + try: + project_client.beta.skills.delete(SKILL_NAME) + print(f"Deleted skill `{SKILL_NAME}`") + except ResourceNotFoundError: + pass diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_multiturn_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_multiturn_evaluation.py index e4c8ad60b306..84ba4efebef4 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_multiturn_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_synthetic_multiturn_evaluation.py @@ -108,13 +108,9 @@ def main() -> None: ], options=SimulationSeedDataGenerationJobOptions( max_samples=SEED_COUNT, - model_options=DataGenerationModelOptions( - model=model_deployment_name - ), - ), - output_options=DataGenerationJobOutputOptions( - name=f"{agent_name}-simulation-seeds" + model_options=DataGenerationModelOptions(model=model_deployment_name), ), + output_options=DataGenerationJobOutputOptions(name=f"{agent_name}-simulation-seeds"), ), ), polling_interval=10, @@ -122,9 +118,7 @@ def main() -> None: generation_result = poller.result() seeds = generation_result.outputs[0] if generation_result.outputs else None - assert isinstance( - seeds, DatasetDataGenerationJobOutput - ), "Expected a dataset output from the generation job" + assert isinstance(seeds, DatasetDataGenerationJobOutput), "Expected a dataset output from the generation job" assert seeds.id is not None, "Generation job returned a dataset without an id" print(f"Generated {generation_result.generated_samples} seed scenarios") @@ -205,14 +199,10 @@ def main() -> None: print("Simulation runs can take several minutes. Polling...") while True: - run = client.evals.runs.retrieve( - run_id=eval_run.id, eval_id=eval_object.id - ) + run = client.evals.runs.retrieve(run_id=eval_run.id, eval_id=eval_object.id) if run.status in ("completed", "failed", "canceled"): break - print( - f"Waiting for simulation to complete... current status: {run.status}" - ) + print(f"Waiting for simulation to complete... current status: {run.status}") time.sleep(10) if run.status != "completed": @@ -221,28 +211,17 @@ def main() -> None: print("\nSynthetic multi-turn evaluation completed successfully.") print(f"Result Counts: {run.result_counts}") if run.result_counts.errored: - raise RuntimeError( - f"{run.result_counts.errored} evaluation item(s) errored" - ) + raise RuntimeError(f"{run.result_counts.errored} evaluation item(s) errored") - expected_conversations = ( - generation_result.generated_samples * CONVERSATIONS_PER_SEED - ) + expected_conversations = generation_result.generated_samples * CONVERSATIONS_PER_SEED print( f"Expected: {expected_conversations} conversations " f"({generation_result.generated_samples} generated scenarios x " f"{CONVERSATIONS_PER_SEED} per scenario)" ) - output_items = list( - client.evals.runs.output_items.list( - run_id=run.id, eval_id=eval_object.id - ) - ) - if ( - run.result_counts.total != expected_conversations - or len(output_items) != expected_conversations - ): + output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) + if run.result_counts.total != expected_conversations or len(output_items) != expected_conversations: raise RuntimeError( f"Expected {expected_conversations} conversations, got " f"{run.result_counts.total} results and {len(output_items)} output items" diff --git a/sdk/ai/azure-ai-projects/samples/responses/sample_responses_model_router.py b/sdk/ai/azure-ai-projects/samples/responses/sample_responses_model_router.py new file mode 100644 index 000000000000..3c22053096fb --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/responses/sample_responses_model_router.py @@ -0,0 +1,56 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates how to invoke a Microsoft Foundry Model Router deployment + using the Responses API and inspect the model selected by the router. + +USAGE: + python sample_responses_model_router.py + + Before running the sample: + + pip install "azure-ai-projects>=2.6.0" python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Microsoft Foundry portal. +""" + +import os + +from dotenv import load_dotenv + +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_router_deployment = "model-router" + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client() as openai_client, +): + print(f"Sending request to Model Router deployment: {model_router_deployment}") + + response = openai_client.responses.create( + model=model_router_deployment, + input="Explain why the sky appears blue in three concise sentences.", + extra_headers={"Foundry-Features": "ModelRouterControls=V1Preview"}, + ) + + print(f"\nResponse output:\n{response.output_text}") + print("\nModel Router result:") + print(f" Response ID: {response.id}") + print(f" Status: {response.status}") + print(f" Selected model: {response.model}") + if response.usage: + print(f" Input tokens: {response.usage.input_tokens}") + print(f" Output tokens: {response.usage.output_tokens}") + print(f" Total tokens: {response.usage.total_tokens}") From 99a2f4ab2490604a0324abb5caaa8420461877b7 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Thu, 3 Sep 2026 17:28:19 -0700 Subject: [PATCH 2/5] Add new sample scripts for Prompt Agents and evaluations Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index c722c132ee04..d98b2c933280 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -17,6 +17,7 @@ ### Sample updates * Added `sample_toolbox_with_shell.py` under `samples/agents/tools/`, demonstrating a Prompt Agent invoking a `ShellToolboxTool`. +* Added `sample_toolbox_with_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a persisted inline Skill through a Toolbox MCP endpoint. * Added `sample_toolbox_with_shell_and_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using an inline Skill with a `ShellToolboxTool` through a Toolbox MCP endpoint. * Added `sample_synthetic_multiturn_evaluation.py` under `samples/evaluations/`, demonstrating simulation seed generation from an agent followed by multi-turn conversation simulation and evaluation. * Added `sample_responses_model_router.py` under `samples/responses/`, demonstrating Responses API calls to a Model Router deployment and inspection of the selected models for each inference turn. From b00592294e81112065197b41d58b762b7aeab5db Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Thu, 3 Sep 2026 18:08:28 -0700 Subject: [PATCH 3/5] Remove deprecated test for responses samples in TestSamples class --- sdk/ai/azure-ai-projects/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index 7cb1cff54b3b..109077173f74 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_1cf1cb56ef" + "Tag": "python/ai/azure-ai-projects_8a61a85d30" } From 19c559dc4639108d48eda097f1c8dedd72097255 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Thu, 3 Sep 2026 19:37:40 -0700 Subject: [PATCH 4/5] Update CHANGELOG and add shipping skill sample for Prompt Agent --- sdk/ai/azure-ai-projects/CHANGELOG.md | 2 +- sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 | 4 ++-- sdk/ai/azure-ai-projects/assets.json | 2 +- ...ox_with_skill.py => sample_toolbox_with_shipping_skill.py} | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename sdk/ai/azure-ai-projects/samples/agents/tools/{sample_toolbox_with_skill.py => sample_toolbox_with_shipping_skill.py} (99%) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index d98b2c933280..c01bf43beb1b 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -17,7 +17,7 @@ ### Sample updates * Added `sample_toolbox_with_shell.py` under `samples/agents/tools/`, demonstrating a Prompt Agent invoking a `ShellToolboxTool`. -* Added `sample_toolbox_with_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a persisted inline Skill through a Toolbox MCP endpoint. +* Added `sample_toolbox_with_shipping_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a persisted inline Skill through a Toolbox MCP endpoint. * Added `sample_toolbox_with_shell_and_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using an inline Skill with a `ShellToolboxTool` through a Toolbox MCP endpoint. * Added `sample_synthetic_multiturn_evaluation.py` under `samples/evaluations/`, demonstrating simulation seed generation from an agent followed by multi-turn conversation simulation and evaluation. * Added `sample_responses_model_router.py` under `samples/responses/`, demonstrating Responses API calls to a Model Router deployment and inspection of the selected models for each inference turn. diff --git a/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 index a939fef58387..938e257ed608 100644 --- a/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 +++ b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 @@ -84,10 +84,10 @@ def public_methods(instance: Any) -> dict[str, bool]: def client_methods(client: Any) -> dict[str, bool]: - included_dunders = {"__enter__", "__exit__"} + included_special_methods = {"__enter__", "__exit__"} methods: dict[str, bool] = {} for name, member in inspect.getmembers(type(client), predicate=callable): - if name.startswith("_") and name not in included_dunders: + if name.startswith("_") and name not in included_special_methods: continue methods[name] = is_handwritten_method(type(client), name) return methods diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index 109077173f74..caef94c4b521 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_8a61a85d30" + "Tag": "python/ai/azure-ai-projects_80e35fe70b" } diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shipping_skill.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py rename to sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shipping_skill.py index d679e6d81495..25f14a49518b 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_skill.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_toolbox_with_shipping_skill.py @@ -16,7 +16,7 @@ Skill proves that the agent applied those instructions. USAGE: - python sample_toolbox_with_skill.py + python sample_toolbox_with_shipping_skill.py Before running the sample: From 3484e9d9d25a9cbe4f74b2f78c08650a4e5b0be5 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Thu, 3 Sep 2026 20:28:12 -0700 Subject: [PATCH 5/5] Update sample descriptions in CHANGELOG for clarity and consistency --- sdk/ai/azure-ai-projects/CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index c01bf43beb1b..7f27056895c8 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -17,10 +17,10 @@ ### Sample updates * Added `sample_toolbox_with_shell.py` under `samples/agents/tools/`, demonstrating a Prompt Agent invoking a `ShellToolboxTool`. -* Added `sample_toolbox_with_shipping_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a persisted inline Skill through a Toolbox MCP endpoint. -* Added `sample_toolbox_with_shell_and_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using an inline Skill with a `ShellToolboxTool` through a Toolbox MCP endpoint. +* Added `sample_toolbox_with_shipping_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a skill through a Toolbox MCP endpoint. +* Added `sample_toolbox_with_shell_and_skill.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using a skill with a `ShellToolboxTool` through a Toolbox MCP endpoint. * Added `sample_synthetic_multiturn_evaluation.py` under `samples/evaluations/`, demonstrating simulation seed generation from an agent followed by multi-turn conversation simulation and evaluation. -* Added `sample_responses_model_router.py` under `samples/responses/`, demonstrating Responses API calls to a Model Router deployment and inspection of the selected models for each inference turn. +* Added `sample_responses_model_router.py` under `samples/responses/`, demonstrating a Responses API request to a model router deployment and selection of a model by the router. ### Bugs Fixed