diff --git a/.agents/skills/integration-tests/SKILL.md b/.agents/skills/integration-tests/SKILL.md new file mode 100644 index 0000000000..1866d674cb --- /dev/null +++ b/.agents/skills/integration-tests/SKILL.md @@ -0,0 +1,64 @@ +--- +name: integration-tests +description: Run the packaged OpenAI Agents Python SDK integration tests from clean wheel and source-distribution environments. Use for release readiness, live OpenAI regression checks, package import compatibility, optional-extra validation, or when asked to run integration tests after examples-auto-run. +--- + +# Integration Tests + +## Overview + +Run the release-oriented integration suite against the exact wheel and source distribution produced by `uv build`. The runner installs both artifacts into isolated environments and validates supported imports, optional extras, OpenAI model adapters, hosted tools, Realtime, and voice workflows. + +## Execution requirements + +- Fresh isolated environments download optional dependencies from PyPI and connect to the configured API providers. +- When the execution environment requires approval for package downloads or configured provider connections, request elevated command execution (`sandbox_permissions=require_escalated`). Retry with the required network permissions before classifying a connectivity failure as an SDK regression. + +## Release workflow + +Run this command from the repository root: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple \ + OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS=1 \ + OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS=0 \ + make integration-tests-release +``` + +- Use the release profile as the default whenever `$integration-tests` is invoked without a narrower request. +- Use OpenRouter as the standard multi-provider gateway. Add provider-specific direct connections only when the user explicitly requests that additional credential matrix. +- Use existing `OPENAI_API_KEY` and `OPENROUTER_API_KEY` values without printing them. Missing optional service configuration may skip capability-specific tests unless strict mode was explicitly requested. +- The command rebuilds the wheel and source distribution, creates isolated virtual environments, checks public imports and optional dependencies, and runs the release-oriented live suites. +- Do not run watch mode, modify source files, create a branch, commit, push, or open a pull request as part of this skill. + +## Paired release validation + +When the user requests both pre-release checks, run `$examples-auto-run` first and follow that skill's required per-example behavioral validation. Then run the command above and report the examples and integration outcomes separately. Invoking `$integration-tests` alone does not implicitly start the examples suite. + +## Focused commands + +Use a focused target only when the user specifically asks to narrow the run: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-packaging +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-core +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-providers +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-hosted +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-realtime +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-voice +env UV_DEFAULT_INDEX=https://pypi.org/simple make integration-tests-extras +``` + +For the minimum supported Python package boundary, use: + +```bash +env UV_DEFAULT_INDEX=https://pypi.org/simple \ + OPENAI_AGENTS_INTEGRATION_PYTHON=3.10 \ + make integration-tests-packaging +``` + +Nightly and manual profiles include additional capability-specific or higher-cost checks. Run them only when explicitly requested; use the configured OpenRouter matrix by default and include direct providers only when explicitly selected. + +## Reporting + +Report the final pass, fail, skip, and deselection counts for each isolated environment. If a command fails, identify the exact profile, package environment, failing test, and actionable error. Separate product regressions from missing credentials, unsupported hosted features, dependency installation failures, and execution-environment restrictions. diff --git a/.agents/skills/integration-tests/agents/openai.yaml b/.agents/skills/integration-tests/agents/openai.yaml new file mode 100644 index 0000000000..cd918c14f2 --- /dev/null +++ b/.agents/skills/integration-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Integration Tests" + short_description: "Run packaged Python SDK integration tests" + default_prompt: "Use $integration-tests to run the packaged Python SDK integration suite." diff --git a/.github/scripts/run_integration_tests.py b/.github/scripts/run_integration_tests.py new file mode 100644 index 0000000000..28aa259d71 --- /dev/null +++ b/.github/scripts/run_integration_tests.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKSPACE = ROOT / ".tmp" / "integration-tests" +DIST = WORKSPACE / "dist" +TESTS = ROOT / "integration_tests" +EXTRAS = "any-llm,litellm,realtime,voice" +OPTIONAL_EXTRAS = ( + "any-llm", + "litellm", + "realtime", + "voice", + "sqlalchemy", + "encrypt", + "redis", + "viz", + "s3", +) +PROFILES = ( + "packaging", + "core", + "providers", + "realtime", + "voice", + "hosted", + "extras", + "full", + "release", + "nightly", + "manual", +) + + +def run(command: list[str], *, env: dict[str, str] | None = None) -> None: + print(f"[integration] {' '.join(command)}", flush=True) + subprocess.run(command, cwd=ROOT, env=env, check=True) + + +def build_distributions() -> tuple[Path, Path]: + DIST.mkdir(parents=True, exist_ok=True) + run(["uv", "build", "--out-dir", str(DIST)]) + wheels = sorted(DIST.glob("openai_agents-*.whl"), key=lambda path: path.stat().st_mtime) + sdists = sorted(DIST.glob("openai_agents-*.tar.gz"), key=lambda path: path.stat().st_mtime) + if not wheels or not sdists: + raise RuntimeError("uv build did not produce both an openai-agents wheel and sdist.") + return wheels[-1], sdists[-1] + + +def _any_llm_provider_extras( + *, external_providers_enabled: bool, direct_providers_enabled: bool +) -> list[str]: + provider_extras: set[str] = set() + configured_models = os.environ.get("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "") + for model in configured_models.split(","): + provider = model.strip().partition("/")[0] + if provider in {"anthropic", "openrouter"}: + provider_extras.add(provider) + elif provider in {"gemini", "google"}: + provider_extras.add("gemini") + + if external_providers_enabled: + if direct_providers_enabled and os.environ.get("ANTHROPIC_API_KEY"): + provider_extras.add("anthropic") + if direct_providers_enabled and ( + os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + ): + provider_extras.add("gemini") + if os.environ.get("OPENROUTER_API_KEY"): + provider_extras.add("openrouter") + + return sorted(provider_extras) + + +def create_environment( + name: str, distribution: Path, *, extras: bool = False, optional_extra: str | None = None +) -> Path: + environment = WORKSPACE / name + venv_command = ["uv", "venv", "--clear", str(environment)] + if python_version := os.environ.get("OPENAI_AGENTS_INTEGRATION_PYTHON"): + venv_command.extend(["--python", python_version]) + run(venv_command) + python = environment / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python") + selected_extra = EXTRAS if extras else optional_extra + requirement = f"{distribution}[{selected_extra}]" if selected_extra else str(distribution) + requirements = [requirement, "pytest", "pytest-asyncio", "pytest-timeout"] + external_providers_enabled = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "" + ).lower() in {"1", "true", "yes"} + direct_providers_enabled = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "" + ).lower() in {"1", "true", "yes"} + if extras: + any_llm_extras = _any_llm_provider_extras( + external_providers_enabled=external_providers_enabled, + direct_providers_enabled=direct_providers_enabled, + ) + if any_llm_extras: + requirements.append(f"any-llm-sdk[{','.join(any_llm_extras)}]") + proxy_values = [ + os.environ.get(name, "") + for name in ( + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + ) + ] + if any(value.lower().startswith("socks") for value in proxy_values): + requirements.append("httpx[socks]") + run(["uv", "pip", "install", "--python", str(python), *requirements]) + return python + + +def run_suite( + python: Path, + wheel: Path, + sdist: Path, + *, + selection: str, + environment_kind: str, +) -> None: + child_env = dict(os.environ) + child_env.pop("PYTHONPATH", None) + if child_env.get("OPENAI_AGENTS_INTEGRATION_DISABLE_PROXY", "").lower() in { + "1", + "true", + "yes", + }: + for variable in ( + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + ): + child_env.pop(variable, None) + child_env["PYTHONNOUSERSITE"] = "1" + child_env["OPENAI_AGENTS_INTEGRATION_WHEEL"] = str(wheel) + child_env["OPENAI_AGENTS_INTEGRATION_SDIST"] = str(sdist) + child_env["OPENAI_AGENTS_INTEGRATION_ENVIRONMENT"] = environment_kind + if environment_kind.startswith("extra-"): + child_env["OPENAI_AGENTS_INTEGRATION_EXTRA"] = environment_kind.removeprefix("extra-") + if not os.environ.get("OPENAI_AGENTS_INTEGRATION_ENABLE_TRACING"): + child_env["OPENAI_AGENTS_DISABLE_TRACING"] = "1" + command = [ + str(python), + "-I", + "-m", + "pytest", + "-c", + str(TESTS / "pytest.ini"), + str(TESTS), + "-v", + "--tb=short", + "-m", + selection, + ] + run(command, env=child_env) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run packaged openai-agents integration tests.") + parser.add_argument("--profile", choices=PROFILES, default="full") + parser.add_argument( + "--all", + action="store_true", + help="Include configured direct Anthropic and Gemini providers alongside OpenRouter.", + ) + args = parser.parse_args() + if args.all: + os.environ["OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS"] = "1" + os.environ["OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS"] = "1" + wheel, sdist = build_distributions() + print(f"[integration] wheel={wheel.name} sdist={sdist.name} profile={args.profile}") + + if args.profile in {"packaging", "core", "hosted", "full", "release", "nightly", "manual"}: + python = create_environment("core", wheel) + selections = { + "packaging": "packaging", + "core": "packaging or core", + "hosted": "packaging or hosted", + "full": "packaging or ((core or hosted) and not nightly and not manual)", + "release": "packaging or ((core or hosted) and not nightly and not manual)", + "nightly": "packaging or ((core or hosted) and not manual)", + "manual": "packaging or core or hosted", + } + run_suite( + python, + wheel, + sdist, + selection=selections[args.profile], + environment_kind="core", + ) + + if args.profile in {"providers", "realtime", "voice", "full", "release", "nightly", "manual"}: + python = create_environment("extended", wheel, extras=True) + if args.profile in {"full", "release"}: + selection = "(providers or realtime or voice) and not nightly and not manual" + elif args.profile == "nightly": + selection = "(providers or realtime or voice) and not manual" + elif args.profile == "manual": + selection = "providers or realtime or voice" + else: + selection = args.profile + run_suite( + python, + wheel, + sdist, + selection=selection, + environment_kind="extended", + ) + + if args.profile in {"packaging", "full", "release", "nightly", "manual"}: + python = create_environment("sdist", sdist) + run_suite(python, wheel, sdist, selection="packaging", environment_kind="sdist") + + if args.profile in {"extras", "full", "release", "nightly", "manual"}: + for optional_extra in OPTIONAL_EXTRAS: + environment_kind = f"extra-{optional_extra}" + python = create_environment(environment_kind, wheel, optional_extra=optional_extra) + run_suite(python, wheel, sdist, selection="extras", environment_kind=environment_kind) + + +if __name__ == "__main__": + main() diff --git a/Makefile b/Makefile index e0f2b64383..daa1745f56 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,62 @@ tests-parallel: tests-serial: uv run pytest -m serial +.PHONY: integration-tests +integration-tests: + uv run python .github/scripts/run_integration_tests.py --profile full $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-release +integration-tests-release: + uv run python .github/scripts/run_integration_tests.py --profile release $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-nightly +integration-tests-nightly: + uv run python .github/scripts/run_integration_tests.py --profile nightly $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-manual +integration-tests-manual: + uv run python .github/scripts/run_integration_tests.py --profile manual $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-packaging +integration-tests-packaging: + uv run python .github/scripts/run_integration_tests.py --profile packaging + +.PHONY: integration-tests-core +integration-tests-core: + uv run python .github/scripts/run_integration_tests.py --profile core + +.PHONY: integration-tests-providers +integration-tests-providers: + uv run python .github/scripts/run_integration_tests.py --profile providers $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-providers-external +integration-tests-providers-external: + OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS=1 uv run python .github/scripts/run_integration_tests.py --profile providers $(filter --all,$(MAKECMDGOALS)) + +.PHONY: integration-tests-providers-all +integration-tests-providers-all: + uv run python .github/scripts/run_integration_tests.py --profile providers --all + +.PHONY: --all +--all: + @: + +.PHONY: integration-tests-realtime +integration-tests-realtime: + uv run python .github/scripts/run_integration_tests.py --profile realtime + +.PHONY: integration-tests-voice +integration-tests-voice: + uv run python .github/scripts/run_integration_tests.py --profile voice + +.PHONY: integration-tests-hosted +integration-tests-hosted: + uv run python .github/scripts/run_integration_tests.py --profile hosted + +.PHONY: integration-tests-extras +integration-tests-extras: + uv run python .github/scripts/run_integration_tests.py --profile extras + .PHONY: coverage coverage: diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 0000000000..4d5db77f50 --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1,28 @@ +# Packaged live integration tests + +These tests exercise the exact wheel produced by `uv build` after installing it into clean virtual environments. The `integration_tests/` directory, repository automation metadata, and local dependency/type-checking caches are excluded from published distributions. + +Run the complete release-oriented matrix with: + + export UV_DEFAULT_INDEX=https://pypi.org/simple + make integration-tests + +`make integration-tests-release` runs the same release-safe matrix explicitly. `make integration-tests-nightly` also includes extended capability and transport checks, while `make integration-tests-manual` includes checks reserved for an intentionally configured manual run. Focused entry points are `make integration-tests-packaging`, `make integration-tests-core`, `make integration-tests-providers`, `make integration-tests-providers-external`, `make integration-tests-providers-all`, `make integration-tests-realtime`, `make integration-tests-voice`, `make integration-tests-hosted`, and `make integration-tests-extras`. + +Invoke the repository-local `$integration-tests` skill to run the release profile with configured OpenRouter-backed provider checks. OpenRouter provides a single configured gateway for the standard multi-provider matrix; provider-specific direct connections are optional extensions selected explicitly. When a release review also requires runnable examples, run `$examples-auto-run` first and then `$integration-tests`. + +Set `OPENAI_API_KEY` for live OpenAI calls. Override `OPENAI_AGENTS_INTEGRATION_MODEL`, `OPENAI_AGENTS_INTEGRATION_REALTIME_MODEL`, `OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS`, and `OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS` when testing different models or configured providers. Provider model lists contain comma-separated adapter model names and require the credentials matching each selected provider. Set `OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL` to use another trusted DeepWiki-compatible hosted MCP server that exposes the `ask_question` tool and can answer questions about the `openai/openai-agents-python` repository. + +Run `make integration-tests-providers-external` with `OPENROUTER_API_KEY` to exercise current OpenAI, Anthropic, and Google models through one provider gateway. To extend the matrix with separately configured direct-provider credentials, use `make integration-tests-providers-external -- --all`, `make integration-tests-providers-all`, or `uv run python .github/scripts/run_integration_tests.py --profile providers --all`. Set `ANTHROPIC_API_KEY` and `GEMINI_API_KEY` or `GOOGLE_API_KEY` for the direct providers you want to include. Override `OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL`, `OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL`, or the comma-separated `OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS` to select provider models. + +The default general model is `gpt-5.6`, while LiteLLM function-tool cases use the Chat Completions-native `openai/gpt-4.1-mini`. This avoids LiteLLM's separate Responses API bridge and keeps the adapter regression focused on its actual Chat Completions contract. + +When the host requires a SOCKS proxy, the runner installs `httpx[socks]` as a test-harness dependency without changing the SDK's published requirements. Set `OPENAI_AGENTS_INTEGRATION_DISABLE_PROXY=1` when the selected environment should connect without inherited proxy settings. + +Set `OPENAI_AGENTS_INTEGRATION_STRICT=1` to fail rather than skip when a requested live feature is not configured. Integration tests never run as part of ordinary `make tests`. + +Each live test has a 75-second timeout so a stalled provider connection cannot block a release review indefinitely. + +Set `OPENAI_AGENTS_INTEGRATION_PYTHON` to choose the Python interpreter used for isolated environments. For example, `OPENAI_AGENTS_INTEGRATION_PYTHON=3.10 make integration-tests-packaging` verifies the minimum supported Python package and import boundary; use Python 3.11 or newer for the full adapter matrix because the AnyLLM extra requires Python 3.11. + +The release suite also covers canonical and supported legacy public-import identity, client-side handoffs, nested agents as tools, custom and shell tools, namespaced tool search, approval/rejection plus serialized `RunState` resume, durable SQLite sessions, explicit and server-managed conversation continuation, controlled retries, input/output and tool guardrails, explicit prompt caching, structured streaming output, provider token logprobs, hosted web search/MCP approval, hosted multi-agent streaming, programmatic-tool streaming/handoffs, multi-turn Realtime history, usage, handoffs, agent updates, voice failure propagation, and independent installation of each selected optional dependency group. The nightly profile adds extended approval matrices, parallel tool concurrency, stateless reasoning replay, reusable Responses WebSocket sessions, collected trace trees, streamed provider tool calls, Realtime audio/guardrails, and streamed-input voice pipelines. diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py new file mode 100644 index 0000000000..9eeeb2d7d6 --- /dev/null +++ b/integration_tests/conftest.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import importlib +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest + +LIVE_MARKERS = frozenset({"core", "providers", "realtime", "voice", "hosted"}) + + +@dataclass(frozen=True) +class ExternalProvider: + name: str + model: str + api_key_name: str + + @property + def api_key(self) -> str: + return os.environ[self.api_key_name] + + +def _external_providers_enabled() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "").lower() in { + "1", + "true", + "yes", + } + + +def _direct_providers_enabled() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "").lower() in { + "1", + "true", + "yes", + } + + +def _external_providers() -> list[ExternalProvider]: + if not _external_providers_enabled(): + return [] + + providers: list[ExternalProvider] = [] + if os.environ.get("OPENROUTER_API_KEY"): + configured = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS", + "openai/gpt-5.6-luna,anthropic/claude-sonnet-5,google/gemini-3.6-flash", + ) + for model in configured.split(","): + if model.strip(): + providers.append( + ExternalProvider( + name=f"openrouter-{model.strip().replace('/', '-')}", + model=f"openrouter/{model.strip()}", + api_key_name="OPENROUTER_API_KEY", + ) + ) + + if _direct_providers_enabled(): + if os.environ.get("ANTHROPIC_API_KEY"): + providers.append( + ExternalProvider( + name="anthropic", + model="anthropic/" + + os.environ.get( + "OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL", "claude-sonnet-5" + ), + api_key_name="ANTHROPIC_API_KEY", + ) + ) + + gemini_key = "GEMINI_API_KEY" if os.environ.get("GEMINI_API_KEY") else "GOOGLE_API_KEY" + if os.environ.get(gemini_key): + providers.append( + ExternalProvider( + name="gemini", + model="gemini/" + + os.environ.get("OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL", "gemini-3.6-flash"), + api_key_name=gemini_key, + ) + ) + + return providers + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "external_provider" not in metafunc.fixturenames: + return + + providers = _external_providers() + if providers: + metafunc.parametrize("external_provider", providers, ids=[item.name for item in providers]) + return + + metafunc.parametrize("external_provider", [None], ids=["unconfigured"]) + + +def _strict() -> bool: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_STRICT", "").lower() in { + "1", + "true", + "yes", + } + + +def skip_or_fail(reason: str) -> None: + if _strict(): + pytest.fail(reason) + pytest.skip(reason) + + +def _provider_model_credentials(model: str) -> tuple[str, ...]: + provider = model.partition("/")[0] + if provider == "openrouter": + return ("OPENROUTER_API_KEY",) + if provider == "anthropic": + return ("ANTHROPIC_API_KEY",) + if provider in {"gemini", "google"}: + return ("GEMINI_API_KEY", "GOOGLE_API_KEY") + return ("OPENAI_API_KEY",) + + +def _has_provider_credential(credential: str) -> bool: + value = os.environ.get(credential) + if credential == "OPENAI_API_KEY": + return value not in {None, "", "test_key", "fake-for-tests"} + return bool(value) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + if not any(item.get_closest_marker(marker) for marker in LIVE_MARKERS): + return + if item.get_closest_marker("providers"): + fixture_names = getattr(item, "fixturenames", ()) + if "external_provider" in fixture_names: + callspec = getattr(item, "callspec", None) + provider = getattr(callspec, "params", {}).get("external_provider") + if provider is None: + if _external_providers_enabled(): + skip_or_fail( + "External provider coverage requires OPENROUTER_API_KEY or explicitly " + "enabled direct-provider credentials." + ) + pytest.skip( + "Enable external provider coverage and set OPENROUTER_API_KEY, " + "or explicitly include configured direct providers." + ) + return + for fixture_name, environment_name in ( + ("any_llm_models", "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS"), + ("litellm_models", "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS"), + ): + if fixture_name not in fixture_names: + continue + configured_models = os.environ.get(environment_name, "") + if not configured_models.strip(): + break + for model in configured_models.split(","): + if not model.strip(): + continue + credentials = _provider_model_credentials(model.strip()) + if not any(_has_provider_credential(credential) for credential in credentials): + skip_or_fail( + f"Set {' or '.join(credentials)} to execute configured provider " + f"model {model.strip()!r}." + ) + return + if os.environ.get("OPENAI_API_KEY") in {None, "", "test_key", "fake-for-tests"}: + skip_or_fail("Set a real OPENAI_API_KEY to execute live integration tests.") + + +@pytest.fixture(scope="session", autouse=True) +def verify_installed_sdk() -> Iterator[None]: + agents = importlib.import_module("agents") + if agents.__file__ is None: + pytest.fail("agents does not expose an installed module path.") + installed_path = Path(agents.__file__).resolve() + environment = Path(sys.prefix).resolve() + if not installed_path.is_relative_to(environment): + pytest.fail(f"agents resolved outside the isolated environment: {installed_path}") + if "site-packages" not in installed_path.parts: + pytest.fail(f"agents did not resolve from an installed distribution: {installed_path}") + yield + + +@pytest.fixture(scope="session") +def integration_model() -> str: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_MODEL", "gpt-5.6") + + +@pytest.fixture(scope="session") +def integration_realtime_model() -> str: + return os.environ.get("OPENAI_AGENTS_INTEGRATION_REALTIME_MODEL", "gpt-realtime-2.1") + + +@pytest.fixture(scope="session") +def any_llm_models(integration_model: str) -> list[str]: + configured = os.environ.get("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "") + return [model.strip() for model in configured.split(",") if model.strip()] or [ + f"openai/{integration_model}" + ] + + +@pytest.fixture(scope="session") +def litellm_models() -> list[str]: + configured = os.environ.get("OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", "") + return [model.strip() for model in configured.split(",") if model.strip()] or [ + "openai/gpt-4.1-mini" + ] + + +@pytest.fixture(scope="session") +async def integration_pcm_audio() -> bytes: + from openai import AsyncOpenAI + + client = AsyncOpenAI() + audio = bytearray() + request = client.audio.speech.with_streaming_response.create( + model="gpt-4o-mini-tts", + voice="alloy", + input="Please say the words packaged voice ready.", + response_format="pcm", + ) + async with request as response: + async for chunk in response.iter_bytes(): + audio.extend(chunk) + + if len(audio) % 2: + audio.append(0) + return bytes(audio) diff --git a/integration_tests/hosted/test_code_interpreter.py b/integration_tests/hosted/test_code_interpreter.py new file mode 100644 index 0000000000..b102ed891c --- /dev/null +++ b/integration_tests/hosted/test_code_interpreter.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import pytest +from openai.types.responses import ResponseReasoningItem + +from agents import Agent, CodeInterpreterTool, RunConfig, Runner +from agents.items import ToolCallItem + +pytestmark = pytest.mark.hosted + + +async def test_code_interpreter_reasoning_items_survive_follow_up_replay( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged code interpreter agent", + model=integration_model, + instructions=( + "Before using any tools, reason through conditional arithmetic to determine which " + "calculation is required. Then use code interpreter for the calculation and " + "answer with RESULT:." + ), + tools=[ + CodeInterpreterTool( + tool_config={"type": "code_interpreter", "container": {"type": "auto"}} + ) + ], + model_settings={ + "max_tokens": 1024, + "reasoning": {"effort": "medium", "summary": "auto"}, + "response_include": ["reasoning.encrypted_content"], + "store": False, + }, + ) + first = await Runner.run( + agent, + "First determine whether the remainder of 4837 multiplied by 8291 divided by 97 " + "is odd. If it is odd, use the code interpreter to calculate 273 * 312821 + 1782; " + "otherwise calculate 19 * 83. Respond only with RESULT:.", + run_config=RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit"), + ) + expected = str(273 * 312821 + 1782) + assert expected in str(first.final_output) + assert any( + isinstance(item, ToolCallItem) + and getattr(item.raw_item, "type", None) == "code_interpreter_call" + for item in first.new_items + ) + + reasoning_items = [ + output + for response in first.raw_responses + for output in response.output + if isinstance(output, ResponseReasoningItem) + ] + assert reasoning_items, [ + getattr(output, "type", type(output).__name__) + for response in first.raw_responses + for output in response.output + ] + + follow_up = first.to_input_list(mode="normalized") + replayed_reasoning = [item for item in follow_up if item.get("type") == "reasoning"] + assert len(replayed_reasoning) == len(reasoning_items) + assert all(isinstance(item.get("encrypted_content"), str) for item in replayed_reasoning) + follow_up.append({"role": "user", "content": "Repeat the calculated result exactly."}) + second = await Runner.run( + agent, + follow_up, + run_config=RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit"), + ) + + assert expected in str(second.final_output) diff --git a/integration_tests/hosted/test_local_tool_families.py b/integration_tests/hosted/test_local_tool_families.py new file mode 100644 index 0000000000..495e86990d --- /dev/null +++ b/integration_tests/hosted/test_local_tool_families.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import ( + Agent, + CustomTool, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + ShellCommandRequest, + ShellTool, + ToolCallOutputItem, +) +from agents.tool_context import ToolContext + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_custom_tools_preserve_raw_string_inputs_and_outputs( + integration_model: str, + streaming: bool, +) -> None: + raw_inputs: list[str] = [] + + async def format_release_word(_context: ToolContext[Any], raw_input: str) -> str: + raw_inputs.append(raw_input) + return raw_input.strip().upper() + + custom = CustomTool( + name="format_release_word", + description="Convert the raw release word to uppercase.", + on_invoke_tool=format_release_word, + ) + agent = Agent( + name="Packaged raw custom tool agent", + model=integration_model, + instructions=( + "Call format_release_word with exactly the raw string amber, " + "then reply exactly CUSTOM:AMBER." + ), + tools=[custom], + model_settings=ModelSettings(tool_choice="required", max_tokens=256), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Format the release word.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Format the release word.", run_config=config) + + outputs = [item for item in result.new_items if isinstance(item, ToolCallOutputItem)] + assert len(raw_inputs) == 1 + assert raw_inputs[0].strip() == "amber" + assert result.final_output == "CUSTOM:AMBER" + assert len(outputs) == 1 + assert isinstance(outputs[0].raw_item, dict) + assert outputs[0].raw_item["type"] == "custom_tool_call_output" + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_custom_tool_approval_survives_serialized_resume( + integration_model: str, + approved: bool, +) -> None: + calls: list[str] = [] + + async def publish_release(_context: ToolContext[Any], raw_input: str) -> str: + calls.append(raw_input) + return "CUSTOM_APPROVED" + + custom = CustomTool( + name="publish_release_note", + description="Publish the raw release note after operator approval.", + on_invoke_tool=publish_release, + needs_approval=True, + ) + agent = Agent( + name="Packaged approval-gated custom tool agent", + model=integration_model, + instructions=( + "Call publish_release_note with the raw string amber. If approved reply exactly " + "CUSTOM_APPROVED; if rejected reply exactly CUSTOM_REJECTED." + ), + tools=[custom], + model_settings=ModelSettings(tool_choice="required", max_tokens=320), + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run(agent, "Publish the release note.", run_config=config, max_turns=5) + assert len(first.interruptions) == 1 + + state = await RunState.from_json(agent, first.to_state().to_json()) + if approved: + state.approve(state.get_interruptions()[0]) + else: + state.reject(state.get_interruptions()[0], rejection_message="Publication was declined.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=5) + outputs = [item for item in resumed.new_items if isinstance(item, ToolCallOutputItem)] + assert calls == (["amber"] if approved else []) + assert resumed.final_output == ("CUSTOM_APPROVED" if approved else "CUSTOM_REJECTED") + assert any( + isinstance(item.raw_item, dict) and item.raw_item.get("type") == "custom_tool_call_output" + for item in outputs + ) + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_local_shell_tools_execute_only_the_supplied_safe_harness( + integration_model: str, + streaming: bool, +) -> None: + requested_commands: list[list[str]] = [] + + def execute_shell(request: ShellCommandRequest) -> str: + requested_commands.append(request.data.action.commands) + return "SHELL_CHECKPOINT_READY" + + agent = Agent( + name="Packaged local shell tool agent", + model=integration_model, + instructions=( + "Call the shell tool with exactly the command echo release, " + "then reply exactly SHELL_READY." + ), + tools=[ShellTool(executor=execute_shell)], + model_settings=ModelSettings(tool_choice="required", max_tokens=256), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Check the release with shell.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Check the release with shell.", run_config=config) + + outputs = [item for item in result.new_items if isinstance(item, ToolCallOutputItem)] + assert requested_commands == [["echo release"]] + assert result.final_output == "SHELL_READY" + assert len(outputs) == 1 + assert isinstance(outputs[0].raw_item, dict) + assert outputs[0].raw_item["type"] == "shell_call_output" + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_local_shell_approval_survives_serialized_resume( + integration_model: str, + approved: bool, +) -> None: + requested_commands: list[list[str]] = [] + + def execute_shell(request: ShellCommandRequest) -> str: + requested_commands.append(request.data.action.commands) + return "SHELL_APPROVED" + + agent = Agent( + name="Packaged approval-gated shell agent", + model=integration_model, + instructions=( + "Call the shell tool with exactly the command echo release. If approved reply " + "exactly SHELL_APPROVED; if rejected reply exactly SHELL_REJECTED." + ), + tools=[ShellTool(executor=execute_shell, needs_approval=True)], + model_settings=ModelSettings(tool_choice="required", max_tokens=320), + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run(agent, "Check the release with shell.", run_config=config, max_turns=5) + assert len(first.interruptions) == 1 + + state = await RunState.from_json(agent, first.to_state().to_json()) + if approved: + state.approve(state.get_interruptions()[0]) + else: + state.reject(state.get_interruptions()[0], rejection_message="Shell access was declined.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=5) + assert requested_commands == ([["echo release"]] if approved else []) + assert resumed.final_output == ("SHELL_APPROVED" if approved else "SHELL_REJECTED") + assert any( + isinstance(item, ToolCallOutputItem) + and isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "shell_call_output" + for item in resumed.new_items + ) diff --git a/integration_tests/hosted/test_mcp.py b/integration_tests/hosted/test_mcp.py new file mode 100644 index 0000000000..9b14a00ef3 --- /dev/null +++ b/integration_tests/hosted/test_mcp.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import os + +import pytest + +from agents import Agent, HostedMCPTool, ModelSettings, RunConfig, Runner, RunState +from agents.items import ( + MCPApprovalRequestItem, + MCPApprovalResponseItem, + MCPListToolsItem, + ToolCallItem, +) +from agents.model_settings import MCPToolChoice + +pytestmark = pytest.mark.hosted + + +async def test_hosted_mcp_lists_and_calls_a_trusted_remote_server(integration_model: str) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP agent", + model=integration_model, + instructions=( + "Use the DeepWiki MCP server to identify the main programming language of " + "openai/openai-agents-python." + ), + model_settings={"max_tokens": 768}, + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_deepwiki", + "server_url": server_url, + "require_approval": "never", + } + ) + ], + ) + result = await Runner.run( + agent, + "Which language is the openai/openai-agents-python repository mainly written in?", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert "python" in str(result.final_output).lower() + assert any(isinstance(item, MCPListToolsItem) for item in result.new_items) + assert any( + isinstance(item, ToolCallItem) and getattr(item.raw_item, "type", None) == "mcp_call" + for item in result.new_items + ) + + +async def test_hosted_mcp_approval_survives_serialized_pause_and_resume( + integration_model: str, +) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP approval agent", + model=integration_model, + instructions="Use the DeepWiki MCP server to answer the repository language question.", + model_settings=ModelSettings( + max_tokens=768, + tool_choice=MCPToolChoice(server_label="packaged_mcp_approval", name="ask_question"), + ), + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_mcp_approval", + "server_url": server_url, + "require_approval": "always", + } + ) + ], + ) + first = await Runner.run( + agent, + "Which language is the openai/openai-agents-python repository mainly written in?", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + + assert len(first.interruptions) == 1 + assert any(isinstance(item, MCPApprovalRequestItem) for item in first.new_items) + state = await RunState.from_json(agent, first.to_state().to_json()) + state.approve(state.get_interruptions()[0]) + resumed = await Runner.run( + agent, + state, + run_config=RunConfig( + tracing_disabled=True, + model_settings=ModelSettings(tool_choice="auto"), + ), + max_turns=6, + ) + + assert "python" in str(resumed.final_output).lower() + assert any(isinstance(item, MCPApprovalResponseItem) for item in resumed.new_items) + + +@pytest.mark.nightly +async def test_hosted_mcp_rejection_survives_serialized_pause_and_resume( + integration_model: str, +) -> None: + server_url = os.environ.get( + "OPENAI_AGENTS_INTEGRATION_MCP_SERVER_URL", "https://mcp.deepwiki.com/mcp" + ) + agent = Agent( + name="Packaged hosted MCP rejection agent", + model=integration_model, + instructions=( + "Use the DeepWiki MCP server to answer the repository language question. " + "If the request is rejected, reply exactly MCP_REJECTED." + ), + model_settings=ModelSettings( + max_tokens=512, + tool_choice=MCPToolChoice(server_label="packaged_mcp_rejection", name="ask_question"), + ), + tools=[ + HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "packaged_mcp_rejection", + "server_url": server_url, + "require_approval": "always", + "allowed_tools": ["ask_question"], + } + ) + ], + ) + + first = await Runner.run( + agent, + "What is the main repository language?", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + assert len(first.interruptions) == 1 + restored = await RunState.from_json(agent, first.to_state().to_json()) + restored.reject(restored.get_interruptions()[0], rejection_message="Remote access declined.") + resumed = await Runner.run( + agent, + restored, + run_config=RunConfig( + tracing_disabled=True, + model_settings=ModelSettings(tool_choice="auto"), + ), + max_turns=6, + ) + + assert resumed.final_output == "MCP_REJECTED" + assert any(isinstance(item, MCPApprovalResponseItem) for item in resumed.new_items) diff --git a/integration_tests/hosted/test_multi_agent.py b/integration_tests/hosted/test_multi_agent.py new file mode 100644 index 0000000000..a20b6eb7f5 --- /dev/null +++ b/integration_tests/hosted/test_multi_agent.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +from typing import Any + +import pytest + +from agents import Agent, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool +from agents.extensions.experimental.hosted_multi_agent import ( + HostedMultiAgentConfig, + OpenAIHostedMultiAgentModel, + get_hosted_agent_metadata, +) +from agents.tool_context import ToolContext + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_hosted_multi_agent_preserves_subagent_tool_callers(streaming: bool) -> None: + model_name = os.environ.get("OPENAI_AGENTS_INTEGRATION_HOSTED_MODEL", "gpt-5.6-sol") + proposals = {"alpha": 6, "beta": 8} + callers: set[str] = set() + call_ids: set[str] = set() + + @tool + def inspect_proposal(ctx: ToolContext[Any], proposal: str) -> dict[str, object]: + """Return deterministic details for one proposal.""" + metadata = get_hosted_agent_metadata(ctx) + callers.add(metadata.agent_name if metadata else "/root") + call_ids.add(ctx.tool_call_id) + return {"proposal": proposal, "estimated_weeks": proposals[proposal]} + + agent = Agent( + name="Packaged hosted coordinator", + model=OpenAIHostedMultiAgentModel( + model=model_name, + config=HostedMultiAgentConfig(max_concurrent_subagents=2), + ), + instructions=( + "Create two subagents. Have one inspect proposal alpha and the other inspect " + "proposal beta. Each subagent must call inspect_proposal before you compare them." + ), + tools=[inspect_proposal], + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Compare proposal alpha and proposal beta.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + event_types = [event.type async for event in streamed.stream_events()] + assert "raw_response_event" in event_types + result = streamed + else: + result = await Runner.run( + agent, + "Compare proposal alpha and proposal beta.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + + assert result.final_output + assert len(call_ids) == 2 + assert len(callers) >= 2 + assert "/root" not in callers diff --git a/integration_tests/hosted/test_programmatic_tool_calling.py b/integration_tests/hosted/test_programmatic_tool_calling.py new file mode 100644 index 0000000000..48cf76c667 --- /dev/null +++ b/integration_tests/hosted/test_programmatic_tool_calling.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import json +from typing import Any, cast + +import pytest +from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses.response_output_item import Program +from pydantic import BaseModel + +from agents import ( + Agent, + ModelSettings, + ProgrammaticToolCallingTool, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + ToolCallItem, + ToolCallOutputItem, + ToolGuardrailFunctionOutput, + ToolInputGuardrailData, + ToolOutputGuardrailData, + handoff, +) +from agents.decorators import tool, tool_input_guardrail, tool_output_guardrail +from agents.extensions.handoff_filters import remove_all_tools +from agents.handoffs import HandoffInputData + +pytestmark = pytest.mark.hosted + + +class InventoryResult(BaseModel): + units: int + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_programmatic_tool_calling_retains_program_owned_calls_and_output( + integration_model: str, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(allowed_callers=["programmatic"]) + def read_inventory(sku: str) -> InventoryResult: + """Return the deterministic available units for an item.""" + calls.append(sku) + return InventoryResult(units={"alpha": 7, "beta": 11}[sku]) + + agent = Agent( + name="Packaged programmatic tool agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling. Generate a JavaScript program that calls " + "read_inventory('alpha') and read_inventory('beta') with Promise.all, adds the " + "units fields from their returned objects, and returns the result. Then answer " + "exactly TOTAL:18." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[read_inventory, ProgrammaticToolCallingTool()], + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Calculate the total inventory.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run( + agent, + "Calculate the total inventory.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + program_calls = [ + item.raw_item + for item in result.new_items + if isinstance(item, ToolCallItem) + and isinstance(item.raw_item, ResponseFunctionToolCall) + and item.raw_item.caller is not None + and item.raw_item.caller.type == "program" + ] + + assert sorted(calls) == ["alpha", "beta"] + assert len(program_calls) == 2 + assert any( + isinstance(item, ToolCallItem) and isinstance(item.raw_item, Program) + for item in result.new_items + ) + assert any( + isinstance(item, ToolCallOutputItem) + and getattr(item.raw_item, "type", None) == "program_output" + for item in result.new_items + ) + assert result.final_output == "TOTAL:18" + + +async def test_programmatic_tool_history_survives_a_filtered_handoff( + integration_model: str, +) -> None: + calls: list[str] = [] + handoff_filter_inputs: list[tuple[HandoffInputData, HandoffInputData]] = [] + + def capture_filtered_handoff(input_data: HandoffInputData) -> HandoffInputData: + filtered = remove_all_tools(input_data) + handoff_filter_inputs.append((input_data, filtered)) + return filtered + + @tool(allowed_callers=["programmatic"]) + def inspect_inventory(sku: str) -> InventoryResult: + """Return deterministic inventory details to the hosted program.""" + calls.append(sku) + return InventoryResult(units=18) + + specialist = Agent( + name="Packaged program summary specialist", + model=integration_model, + instructions="Reply with exactly FILTERED_PROGRAM_HANDOFF_OK.", + model_settings={"max_tokens": 256}, + ) + coordinator = Agent( + name="Packaged program handoff coordinator", + model=integration_model, + instructions=( + "First use Programmatic Tool Calling to run inspect_inventory('alpha'). " + "After the program returns, immediately transfer to the summary specialist." + ), + tools=[inspect_inventory, ProgrammaticToolCallingTool()], + handoffs=[handoff(specialist, input_filter=capture_filtered_handoff)], + model_settings={"max_tokens": 1024}, + ) + result = await Runner.run( + coordinator, + "Inspect alpha with a program, then transfer the answer.", + run_config=RunConfig(tracing_disabled=True, nest_handoff_history=True), + max_turns=7, + ) + + assert calls == ["alpha"] + assert result.final_output == "FILTERED_PROGRAM_HANDOFF_OK" + assert result.last_agent is specialist + assert len(handoff_filter_inputs) == 1 + original_input, filtered_input = handoff_filter_inputs[0] + assert any( + isinstance(item, ToolCallItem | ToolCallOutputItem) + for item in (*original_input.pre_handoff_items, *original_input.new_items) + ) + assert not any( + isinstance(item, ToolCallItem | ToolCallOutputItem) + for item in (*filtered_input.pre_handoff_items, *filtered_input.new_items) + ) + assert any( + isinstance(output, Program) + for response in result.raw_responses + for output in response.output + ) + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_programmatic_tool_approval_preserves_caller_across_serialized_resume( + integration_model: str, approved: bool +) -> None: + calls: list[str] = [] + + @tool(allowed_callers=["programmatic"], needs_approval=True) + def approve_inventory(sku: str) -> InventoryResult: + """Read inventory only after the program's tool call is approved.""" + calls.append(sku) + return InventoryResult(units=18) + + agent = Agent( + name="Packaged programmatic approval agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling to invoke approve_inventory('alpha'). " + "If it succeeds reply exactly PROGRAM_APPROVED; if it is rejected reply " + "exactly PROGRAM_REJECTED." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[approve_inventory, ProgrammaticToolCallingTool()], + ) + config = RunConfig(tracing_disabled=True) + + first = await Runner.run(agent, "Read the protected inventory.", run_config=config, max_turns=6) + assert len(first.interruptions) == 1 + state = await RunState.from_json(agent, first.to_state().to_json()) + interruption = state.get_interruptions()[0] + if approved: + state.approve(interruption) + else: + state.reject(interruption, rejection_message="Inventory access was rejected.") + + resumed = await Runner.run(agent, state, run_config=config, max_turns=6) + outputs = [item for item in resumed.new_items if isinstance(item, ToolCallOutputItem)] + + assert calls == (["alpha"] if approved else []) + assert outputs + if approved: + assert resumed.final_output == "PROGRAM_APPROVED" + else: + rejected_item = next( + item + for item in outputs + if isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "function_call_output" + ) + assert rejected_item.output == "Inventory access was rejected." + assert json.loads(cast(dict[str, Any], rejected_item.raw_item)["output"]) == { + "error": "Inventory access was rejected." + } + callers = [ + cast(dict[str, Any], item.raw_item).get("caller") + if isinstance(item.raw_item, dict) + else getattr(item.raw_item, "caller", None) + for item in outputs + ] + assert any( + (caller.get("type") if isinstance(caller, dict) else getattr(caller, "type", None)) + == "program" + for caller in callers + ) + + +@pytest.mark.nightly +@pytest.mark.parametrize("rejection_stage", ["input", "output"]) +async def test_programmatic_structured_tool_guardrail_errors_are_valid_json( + integration_model: str, rejection_stage: str +) -> None: + calls: list[str] = [] + rejection_message = f"Inventory {rejection_stage} was rejected." + + @tool_input_guardrail + def inspect_input(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + if rejection_stage == "input": + return ToolGuardrailFunctionOutput.reject_content(rejection_message) + return ToolGuardrailFunctionOutput.allow() + + @tool_output_guardrail + def inspect_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + if rejection_stage == "output": + return ToolGuardrailFunctionOutput.reject_content(rejection_message) + return ToolGuardrailFunctionOutput.allow() + + @tool( + allowed_callers=["programmatic"], + tool_input_guardrails=[inspect_input], + tool_output_guardrails=[inspect_output], + ) + def inspect_inventory(sku: str) -> InventoryResult: + """Read inventory after both programmatic tool guardrails allow the request.""" + calls.append(sku) + return InventoryResult(units=18) + + agent = Agent( + name="Packaged programmatic guardrail rejection agent", + model=integration_model, + instructions=( + "Use Programmatic Tool Calling. Write a JavaScript program that calls " + "inspect_inventory('alpha') and returns the resulting error field when present. " + "After the program finishes, reply exactly PROGRAM_GUARDRAIL_REJECTED." + ), + model_settings=ModelSettings(tool_choice="programmatic_tool_calling", max_tokens=1024), + tools=[inspect_inventory, ProgrammaticToolCallingTool()], + ) + + result = await Runner.run( + agent, + "Inspect the guarded inventory and report its rejection.", + run_config=RunConfig(tracing_disabled=True), + max_turns=6, + ) + rejected_item = next( + item + for item in result.new_items + if isinstance(item, ToolCallOutputItem) + and isinstance(item.raw_item, dict) + and item.raw_item.get("type") == "function_call_output" + ) + + assert calls == ([] if rejection_stage == "input" else ["alpha"]) + assert rejected_item.output == rejection_message + assert json.loads(cast(dict[str, Any], rejected_item.raw_item)["output"]) == { + "error": rejection_message + } + assert result.final_output == "PROGRAM_GUARDRAIL_REJECTED" diff --git a/integration_tests/hosted/test_tool_search.py b/integration_tests/hosted/test_tool_search.py new file mode 100644 index 0000000000..55482c9809 --- /dev/null +++ b/integration_tests/hosted/test_tool_search.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import pytest + +from agents import ( + Agent, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + ToolCallItem, + ToolCallOutputItem, + ToolSearchCallItem, + ToolSearchOutputItem, + ToolSearchTool, + tool_namespace, +) +from agents.decorators import tool + +pytestmark = pytest.mark.hosted + + +@pytest.mark.parametrize( + "streaming", + [False, pytest.param(True, marks=pytest.mark.nightly)], + ids=["nonstreaming", "streaming"], +) +async def test_tool_search_loads_and_executes_a_deferred_namespaced_tool( + integration_model: str, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(defer_loading=True) + def lookup_customer(customer_id: str) -> str: + """Find the customer's release readiness status.""" + calls.append(customer_id) + return "READY" + + namespaced = tool_namespace( + name="customer_support", + description="Look up customer release readiness and support records.", + tools=[lookup_customer], + ) + agent = Agent( + name="Packaged deferred tool search agent", + model=integration_model, + instructions=( + "Find the customer support tool, call lookup_customer with customer_id='customer-42', " + "and then reply with exactly SEARCH_READY." + ), + tools=[*namespaced, ToolSearchTool()], + model_settings=ModelSettings(max_tokens=512, parallel_tool_calls=False), + ) + result: RunResult | RunResultStreaming + if streaming: + streamed = Runner.run_streamed( + agent, + "Find and run the deferred customer lookup.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + events = [event async for event in streamed.stream_events()] + assert any(event.type == "raw_response_event" for event in events) + result = streamed + else: + result = await Runner.run( + agent, + "Find and run the deferred customer lookup.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert calls == ["customer-42"] + assert result.final_output == "SEARCH_READY" + assert any(isinstance(item, ToolSearchCallItem) for item in result.new_items) + assert any(isinstance(item, ToolSearchOutputItem) for item in result.new_items) + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) + + +async def test_tool_search_routes_identically_named_tools_by_namespace( + integration_model: str, +) -> None: + calls: list[str] = [] + + @tool(name_override="lookup", defer_loading=True) + def lookup_billing(customer_id: str) -> str: + """Look up the customer's billing status.""" + calls.append(f"billing:{customer_id}") + return "BILLING_READY" + + @tool(name_override="lookup", defer_loading=True) + def lookup_shipping(customer_id: str) -> str: + """Look up the customer's package shipping status.""" + calls.append(f"shipping:{customer_id}") + return "SHIPPING_READY" + + agent = Agent( + name="Packaged namespaced tool routing agent", + model=integration_model, + instructions=( + "Find the shipping namespace tool named lookup and call it exactly once with " + "customer_id='customer-42'. Do not use billing. Reply exactly SHIPPING_READY." + ), + tools=[ + *tool_namespace(name="billing", description="Billing records", tools=[lookup_billing]), + *tool_namespace( + name="shipping", description="Package shipping records", tools=[lookup_shipping] + ), + ToolSearchTool(), + ], + model_settings=ModelSettings(max_tokens=512, parallel_tool_calls=False), + ) + + result = await Runner.run( + agent, + "Check the customer's shipping status.", + run_config=RunConfig(tracing_disabled=True), + max_turns=5, + ) + + assert calls == ["shipping:customer-42"] + assert result.final_output == "SHIPPING_READY" diff --git a/integration_tests/hosted/test_web_search.py b/integration_tests/hosted/test_web_search.py new file mode 100644 index 0000000000..98ec547b57 --- /dev/null +++ b/integration_tests/hosted/test_web_search.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from agents import Agent, RunConfig, Runner, ToolCallItem, WebSearchTool + +pytestmark = pytest.mark.hosted + + +async def test_web_search_emits_provider_owned_call_items(integration_model: str) -> None: + agent = Agent( + name="Packaged web search agent", + model=integration_model, + instructions=( + "Search the web before answering. Identify the organization that publishes " + "the OpenAI Agents Python SDK, then answer with only OPENAI." + ), + model_settings={"max_tokens": 768}, + tools=[WebSearchTool()], + ) + result = await Runner.run( + agent, + "Search for the official openai-agents-python GitHub repository publisher.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output.strip().upper() == "OPENAI" + assert any( + isinstance(item, ToolCallItem) and getattr(item.raw_item, "type", None) == "web_search_call" + for item in result.new_items + ) diff --git a/integration_tests/openai/test_approval_resume.py b/integration_tests/openai/test_approval_resume.py new file mode 100644 index 0000000000..850fb76d18 --- /dev/null +++ b/integration_tests/openai/test_approval_resume.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + RunState, + SQLiteSession, + ToolCallOutputItem, +) +from agents.decorators import tool + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_tool_approval_survives_serialized_state_and_resume( + integration_model: str, approved: bool, streaming: bool +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def perform_action(action: str) -> str: + """Perform the deterministic action only after explicit approval.""" + calls.append(action) + return "completed" + + agent = Agent( + name="Packaged approval agent", + model=integration_model, + instructions=( + "Call perform_action with action='deploy'. If the tool succeeds, reply exactly " + "APPROVED. If the tool is rejected, reply exactly REJECTED." + ), + tools=[perform_action], + model_settings={"max_tokens": 384}, + ) + config = RunConfig(tracing_disabled=True) + first: RunResult | RunResultStreaming + resumed: RunResult | RunResultStreaming + + if streaming: + first_stream = Runner.run_streamed(agent, "Perform the deployment.", run_config=config) + async for _event in first_stream.stream_events(): + pass + first = first_stream + else: + first = await Runner.run(agent, "Perform the deployment.", run_config=config) + + assert len(first.interruptions) == 1 + interruption = first.interruptions[0] + assert interruption.name == "perform_action" + state_json = first.to_state().to_json() + restored = await RunState.from_json(agent, state_json) + restored_interruption = restored.get_interruptions()[0] + + if approved: + restored.approve(restored_interruption) + else: + restored.reject(restored_interruption, rejection_message="The operator rejected deploy.") + + if streaming: + resumed_stream = Runner.run_streamed(agent, restored, run_config=config) + async for _event in resumed_stream.stream_events(): + pass + resumed = resumed_stream + else: + resumed = await Runner.run(agent, restored, run_config=config) + + assert resumed.final_output == ("APPROVED" if approved else "REJECTED") + assert calls == (["deploy"] if approved else []) + assert any(isinstance(item, ToolCallOutputItem) for item in resumed.new_items) + + +async def test_approval_resume_preserves_durable_sqlite_tool_history( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def confirm_release(version: str) -> str: + """Confirm a release after its approval decision is restored.""" + calls.append(version) + return "approved" + + agent = Agent( + name="Packaged durable approval agent", + model=integration_model, + instructions="Call confirm_release with version='1.0', then reply RELEASE_APPROVED.", + model_settings={"max_tokens": 384}, + tools=[confirm_release], + ) + session = SQLiteSession("packaged-approval", tmp_path / "approval.sqlite3") + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, + "Approve the release.", + session=session, + run_config=config, + ) + restored = await RunState.from_json(agent, first.to_state().to_json()) + restored.approve(restored.get_interruptions()[0]) + resumed = await Runner.run(agent, restored, session=session, run_config=config) + saved_items = await session.get_items() + finally: + session.close() + + assert calls == ["1.0"] + assert resumed.final_output == "RELEASE_APPROVED" + assert sum(item.get("role") == "user" for item in saved_items) == 1 + assert sum(item.get("type") == "function_call_output" for item in saved_items) == 1 + + +async def test_parallel_tool_approvals_preserve_mixed_decisions_after_serialization( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool(needs_approval=True) + def approve_release(version: str) -> str: + """Approve a deterministic release version.""" + calls.append(f"release:{version}") + return "release-approved" + + @tool(needs_approval=True) + def notify_customer(customer: str) -> str: + """Notify a deterministic customer.""" + calls.append(f"customer:{customer}") + return "customer-notified" + + agent = Agent( + name="Packaged mixed approval agent", + model=integration_model, + instructions=( + "In the same turn, call approve_release with version='1.0' and notify_customer " + "with customer='customer-42'. After their approval decisions, reply exactly " + "MIXED_APPROVAL_READY." + ), + model_settings={"max_tokens": 512, "parallel_tool_calls": True}, + tools=[approve_release, notify_customer], + ) + session = SQLiteSession("packaged-mixed-approval", tmp_path / "mixed-approval.sqlite3") + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, "Perform both requested actions.", session=session, run_config=config + ) + assert len(first.interruptions) == 2 + restored = await RunState.from_json(agent, first.to_state().to_json()) + for interruption in restored.get_interruptions(): + if interruption.name == "approve_release": + restored.approve(interruption) + else: + restored.reject(interruption, rejection_message="Customer notification declined.") + resumed = await Runner.run(agent, restored, session=session, run_config=config) + stored = await session.get_items() + finally: + session.close() + + assert calls == ["release:1.0"] + assert resumed.final_output == "MIXED_APPROVAL_READY" + assert sum(item.get("role") == "user" for item in stored) == 1 + outputs = [item for item in stored if item.get("type") == "function_call_output"] + assert len(outputs) == 2 diff --git a/integration_tests/openai/test_chat_completions.py b/integration_tests/openai/test_chat_completions.py new file mode 100644 index 0000000000..8a54eeb561 --- /dev/null +++ b/integration_tests/openai/test_chat_completions.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai import AsyncOpenAI +from pydantic import BaseModel + +from agents import Agent, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool +from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel + +pytestmark = pytest.mark.core + + +class ChatCompletionStatus(BaseModel): + status: str + checkpoints: list[int] + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_chat_completions_tools_settings_and_usage( + integration_model: str, dictionary: bool, streaming: bool +) -> None: + from agents import ModelSettings + + calls: list[str] = [] + + @tool + def package_status(package: str) -> str: + """Return a deterministic package status.""" + calls.append(package) + return "ready" + + values: dict[str, Any] = { + "reasoning": {"effort": "none"}, + "include_usage": True, + "extra_args": {"max_completion_tokens": 512}, + } + settings = values if dictionary else ModelSettings(**values) + agent = Agent( + name="Packaged Chat Completions agent", + model=OpenAIChatCompletionsModel( + model=integration_model, + openai_client=AsyncOpenAI(), + ), + instructions=( + "Call package_status exactly once with package='openai-agents', then reply " + "exactly CHAT_READY." + ), + model_settings=settings, + tools=[package_status], + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + result = Runner.run_streamed(agent, "Check the package.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Check the package.", run_config=config) + + assert calls == ["openai-agents"] + assert result.final_output == "CHAT_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_chat_completions_preserves_typed_structured_output( + integration_model: str, + streaming: bool, +) -> None: + agent = Agent( + name="Packaged structured Chat Completions agent", + model=OpenAIChatCompletionsModel( + model=integration_model, + openai_client=AsyncOpenAI(), + ), + instructions="Return status CHAT_STRUCTURED_READY and checkpoints [2, 4, 8].", + output_type=ChatCompletionStatus, + model_settings={"reasoning": {"effort": "none"}, "include_usage": True}, + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Return the requested typed release status.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Return the requested typed release status.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == ChatCompletionStatus( + status="CHAT_STRUCTURED_READY", checkpoints=[2, 4, 8] + ) + assert result.context_wrapper.usage.total_tokens > 0 diff --git a/integration_tests/openai/test_execution_controls.py b/integration_tests/openai/test_execution_controls.py new file mode 100644 index 0000000000..f0807fa7a5 --- /dev/null +++ b/integration_tests/openai/test_execution_controls.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any, cast + +import pytest +from openai.types.responses import ResponseReasoningItem + +from agents import ( + Agent, + AgentHookContext, + ModelSettings, + RunConfig, + RunContextWrapper, + RunErrorHandlerInput, + RunErrorHandlerResult, + RunHooks, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, + Tool, + ToolCallOutputItem, + ToolExecutionConfig, +) +from agents.decorators import tool +from agents.items import ModelResponse, TResponseInputItem +from agents.run_config import CallModelData, ModelInputData + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_stop_on_first_tool_avoids_a_follow_up_model_request( + integration_model: str, + streaming: bool, +) -> None: + calls: list[str] = [] + + @tool + def resolve_checkpoint(checkpoint: str) -> str: + """Return the requested release checkpoint directly.""" + calls.append(checkpoint) + return "STOP_ON_FIRST_TOOL_READY" + + agent = Agent( + name="Packaged stop-on-tool agent", + model=integration_model, + instructions="Call resolve_checkpoint exactly once with checkpoint='release'.", + tools=[resolve_checkpoint], + tool_use_behavior="stop_on_first_tool", + model_settings={"max_tokens": 256, "tool_choice": "required"}, + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Return the checkpoint through the tool.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Return the checkpoint through the tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["release"] + assert result.final_output == "STOP_ON_FIRST_TOOL_READY" + assert result.context_wrapper.usage.requests == 1 + assert len(result.raw_responses) == 1 + + +@pytest.mark.nightly +@pytest.mark.parametrize("max_concurrency", [1, 2], ids=["sequential", "bounded-parallel"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_parallel_function_tools_preserve_order_and_sdk_concurrency_limits( + integration_model: str, + max_concurrency: int, + streaming: bool, +) -> None: + active_calls = 0 + peak_concurrency = 0 + + async def run_checkpoint(name: str) -> str: + nonlocal active_calls, peak_concurrency + active_calls += 1 + peak_concurrency = max(peak_concurrency, active_calls) + try: + await asyncio.sleep(0.08) + return name.upper() + finally: + active_calls -= 1 + + @tool + async def checkpoint_alpha(checkpoint: str) -> str: + """Return the alpha release checkpoint.""" + return await run_checkpoint(checkpoint) + + @tool + async def checkpoint_beta(checkpoint: str) -> str: + """Return the beta release checkpoint.""" + return await run_checkpoint(checkpoint) + + settings = ModelSettings( + tool_choice="required", + parallel_tool_calls=True, + max_tokens=512, + ) + agent = Agent( + name="Packaged bounded tool concurrency agent", + model=integration_model, + instructions=( + "In the same turn, call checkpoint_alpha with checkpoint='alpha' and " + "checkpoint_beta with checkpoint='beta'. After both tools finish reply exactly " + "CONCURRENCY_READY." + ), + tools=[checkpoint_alpha, checkpoint_beta], + model_settings=settings, + ) + config = RunConfig( + tracing_disabled=True, + tool_execution=ToolExecutionConfig(max_function_tool_concurrency=max_concurrency), + ) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Run both release checkpoints.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Run both release checkpoints.", run_config=config) + + outputs = [item.output for item in result.new_items if isinstance(item, ToolCallOutputItem)] + + assert result.final_output == "CONCURRENCY_READY" + assert outputs == ["ALPHA", "BETA"] + assert peak_concurrency == max_concurrency + assert result.context_wrapper.usage.requests == 2 + assert settings.tool_choice == "required" + + +async def test_session_merge_and_model_input_filter_have_distinct_persistence_boundaries( + integration_model: str, + tmp_path: Path, +) -> None: + callback_inputs: list[tuple[int, str]] = [] + filter_inputs: list[str] = [] + agent = Agent( + name="Packaged session input filtering agent", + model=integration_model, + instructions="Remember user-provided release words and reply exactly as requested.", + model_settings={"max_tokens": 256}, + ) + session = SQLiteSession("packaged-filtered-session", tmp_path / "filtered.sqlite3") + + try: + await Runner.run( + agent, + "Remember the release word JASPER and reply only STORED.", + session=session, + run_config=RunConfig(tracing_disabled=True), + ) + + def merge_session_input( + history: list[TResponseInputItem], + new_input: list[TResponseInputItem], + ) -> list[TResponseInputItem]: + callback_inputs.append((len(history), str(new_input[0].get("content")))) + rewritten = cast( + TResponseInputItem, + { + "role": "user", + "content": "What release word did I provide? Reply only with that word.", + }, + ) + return [*history, rewritten] + + def filter_model_input(data: CallModelData[Any]) -> ModelInputData: + latest = data.model_data.input[-1] + filter_inputs.append(str(latest.get("content"))) + return ModelInputData( + input=data.model_data.input, + instructions=(data.model_data.instructions or "") + + " Prefix the remembered word with FILTERED: and reply with nothing else.", + ) + + result = await Runner.run( + agent, + "PLACEHOLDER_NEW_INPUT", + session=session, + run_config=RunConfig( + tracing_disabled=True, + session_input_callback=merge_session_input, + call_model_input_filter=filter_model_input, + ), + ) + persisted = await session.get_items() + finally: + session.close() + + assert len(callback_inputs) == 1 + assert callback_inputs[0][0] >= 2 + assert callback_inputs[0][1] == "PLACEHOLDER_NEW_INPUT" + assert filter_inputs == ["What release word did I provide? Reply only with that word."] + assert result.final_output == "FILTERED:JASPER" + assert any("What release word" in str(item.get("content", "")) for item in persisted) + assert not any("PLACEHOLDER_NEW_INPUT" in str(item.get("content", "")) for item in persisted) + + +@pytest.mark.nightly +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_stateless_reasoning_replay_preserves_encrypted_content_when_returned( + integration_model: str, + streaming: bool, +) -> None: + stored_words: list[str] = [] + + @tool + def remember_word(word: str) -> str: + """Store the release word and return a deterministic acknowledgement.""" + stored_words.append(word) + return "WORD_STORED" + + agent = Agent( + name="Packaged stateless reasoning replay agent", + model=integration_model, + instructions=( + "Calculate the requested arithmetic before calling remember_word. Use the word " + "AMBER when the result is odd and COBALT when it is even. Once the tool succeeds " + "reply exactly STORED. Answer follow-up questions using the previous tool result." + ), + tools=[remember_word], + model_settings=ModelSettings( + store=False, + reasoning={"effort": "medium", "summary": "auto"}, + response_include=["reasoning.encrypted_content"], + max_tokens=1024, + ), + ) + config = RunConfig(tracing_disabled=True, reasoning_item_id_policy="omit") + first = await Runner.run( + agent, + "What is the remainder when 4837 multiplied by 8291 is divided by 97? " + "Follow the parity rule, call remember_word, and then reply only STORED.", + run_config=config, + ) + reasoning_items = [ + item + for response in first.raw_responses + for item in response.output + if isinstance(item, ResponseReasoningItem) + ] + assert reasoning_items, "The stateless response did not contain any reasoning items." + replay = first.to_input_list(mode="normalized") + replayed_reasoning = [item for item in replay if item.get("type") == "reasoning"] + replay.append( + cast( + TResponseInputItem, + {"role": "user", "content": "What release word did I provide? Reply only AMBER."}, + ) + ) + + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, replay, run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, replay, run_config=config) + + assert all(isinstance(item.encrypted_content, str) for item in reasoning_items) + assert len(replayed_reasoning) == len(reasoning_items) + assert all(isinstance(item.get("encrypted_content"), str) for item in replayed_reasoning) + assert all("id" not in item for item in replayed_reasoning) + assert first.context_wrapper.usage.requests == 2 + assert result.context_wrapper.usage.requests == 1 + assert stored_words == ["AMBER"] + assert result.final_output == "AMBER" + + +@pytest.mark.parametrize("use_session", [False, True], ids=["without-session", "sqlite-session"]) +async def test_cancel_after_turn_resumes_without_repeating_function_tools( + integration_model: str, + use_session: bool, +) -> None: + calls: list[str] = [] + + @tool + def checkpoint(value: str) -> str: + """Record one deterministic cancellation checkpoint.""" + calls.append(value) + return "CANCEL_CHECKPOINT_READY" + + session = SQLiteSession("packaged-cancel-after-turn") if use_session else None + agent = Agent( + name="Packaged streamed cancellation agent", + model=integration_model, + instructions=( + "Call checkpoint with value='release'. After the tool returns, reply exactly " + "CANCEL_RESUMED_READY." + ), + tools=[checkpoint], + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + result = Runner.run_streamed( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + ) + async for event in result.stream_events(): + if getattr(event, "name", None) == "tool_called": + result.cancel(mode="after_turn") + + replay = result.to_input_list(mode="normalized") + resumed = await Runner.run(result.last_agent, replay, run_config=config) + persisted = await session.get_items() if session is not None else [] + finally: + if session is not None: + session.close() + + assert result.final_output is None + assert result.is_complete + assert calls == ["release"] + assert resumed.final_output == "CANCEL_RESUMED_READY" + assert result.context_wrapper.usage.requests == 1 + assert resumed.context_wrapper.usage.requests == 1 + if use_session: + assert any(item.get("type") == "function_call_output" for item in persisted) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_max_turn_error_handler_preserves_tool_side_effects_and_history( + integration_model: str, + streaming: bool, +) -> None: + calls: list[str] = [] + handled: list[str] = [] + + @tool + def release_checkpoint(value: str) -> str: + """Return a release checkpoint before the model exceeds its turn limit.""" + calls.append(value) + return "CHECKPOINT_RECORDED" + + def handle_max_turns(data: RunErrorHandlerInput[Any]) -> RunErrorHandlerResult: + handled.append(type(data.error).__name__) + return RunErrorHandlerResult(final_output="MAX_TURNS_RECOVERED", include_in_history=False) + + session = SQLiteSession(f"packaged-max-turn-recovery-{streaming}") + agent = Agent( + name="Packaged max-turn recovery agent", + model=integration_model, + instructions="Call release_checkpoint with value='release', then explain the result.", + tools=[release_checkpoint], + model_settings={"tool_choice": "required", "max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + max_turns=1, + error_handlers={"max_turns": handle_max_turns}, + ) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run( + agent, + "Run the release checkpoint.", + session=session, + run_config=config, + max_turns=1, + error_handlers={"max_turns": handle_max_turns}, + ) + persisted = await session.get_items() + finally: + session.close() + + assert calls == ["release"] + assert handled == ["MaxTurnsExceeded"] + assert result.final_output == "MAX_TURNS_RECOVERED" + assert any(item.get("type") == "function_call_output" for item in persisted) + assert not any("MAX_TURNS_RECOVERED" in str(item) for item in persisted) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_live_run_hooks_preserve_model_and_function_tool_event_order( + integration_model: str, + streaming: bool, +) -> None: + observed: list[str] = [] + + class RecordingHooks(RunHooks[Any]): + async def on_agent_start(self, context: AgentHookContext[Any], agent: Agent[Any]) -> None: + del context, agent + observed.append("agent_start") + + async def on_agent_end( + self, + context: AgentHookContext[Any], + agent: Agent[Any], + output: Any, + ) -> None: + del context, agent, output + observed.append("agent_end") + + async def on_llm_start( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + system_prompt: str | None, + input_items: list[TResponseInputItem], + ) -> None: + del context, agent, system_prompt, input_items + observed.append("model_start") + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + del context, agent, response + observed.append("model_end") + + async def on_tool_start( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + ) -> None: + del context, agent, tool + observed.append("tool_start") + + async def on_tool_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + result: object, + ) -> None: + del context, agent, tool, result + observed.append("tool_end") + + @tool + def inspect_release(value: str) -> str: + """Inspect a release checkpoint for lifecycle hook ordering.""" + observed.append(f"tool_call:{value}") + return "HOOK_CHECKPOINT_READY" + + agent = Agent( + name="Packaged run hooks agent", + model=integration_model, + instructions=("Call inspect_release with value='release', then reply exactly HOOKS_READY."), + tools=[inspect_release], + model_settings={"max_tokens": 256}, + ) + hooks = RecordingHooks() + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "Inspect the release.", hooks=hooks, run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Inspect the release.", hooks=hooks, run_config=config) + + assert result.final_output == "HOOKS_READY" + assert observed == [ + "agent_start", + "model_start", + "model_end", + "tool_start", + "tool_call:release", + "tool_end", + "model_start", + "model_end", + "agent_end", + ] diff --git a/integration_tests/openai/test_guardrails.py b/integration_tests/openai/test_guardrails.py new file mode 100644 index 0000000000..7d5235e2a4 --- /dev/null +++ b/integration_tests/openai/test_guardrails.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import ( + Agent, + GuardrailFunctionOutput, + InputGuardrailTripwireTriggered, + OutputGuardrailTripwireTriggered, + RunConfig, + RunContextWrapper, + Runner, + ToolExecutionConfig, + ToolGuardrailFunctionOutput, + ToolInputGuardrailData, + ToolOutputGuardrailData, +) +from agents.decorators import ( + input_guardrail, + output_guardrail, + tool, + tool_input_guardrail, + tool_output_guardrail, +) + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_output_guardrails_validate_real_model_results( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def inspect_result( + context: RunContextWrapper[Any], agent: Agent[Any], output: str + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(output) + return GuardrailFunctionOutput( + output_info={"checked": True}, + tripwire_triggered=blocked, + ) + + agent = Agent( + name="Packaged output guardrail agent", + model=integration_model, + instructions="Reply with exactly GUARDED_RESULT.", + output_guardrails=[inspect_result], + model_settings={"max_tokens": 256}, + ) + if blocked: + with pytest.raises(OutputGuardrailTripwireTriggered): + await Runner.run( + agent, + "Return the deterministic guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + else: + result = await Runner.run( + agent, + "Return the deterministic guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + assert result.final_output == "GUARDED_RESULT" + + assert inspected == ["GUARDED_RESULT"] + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_input_guardrails_validate_live_run_requests( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @input_guardrail + async def inspect_input( + context: RunContextWrapper[Any], agent: Agent[Any], input: str | list[Any] + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(str(input)) + return GuardrailFunctionOutput(output_info={"checked": True}, tripwire_triggered=blocked) + + agent = Agent( + name="Packaged input guardrail agent", + model=integration_model, + instructions="Reply with exactly INPUT_GUARDRAIL_READY.", + input_guardrails=[inspect_input], + model_settings={"max_tokens": 256}, + ) + if blocked: + with pytest.raises(InputGuardrailTripwireTriggered): + await Runner.run( + agent, + "Check the input guardrail.", + run_config=RunConfig(tracing_disabled=True), + ) + else: + result = await Runner.run( + agent, + "Check the input guardrail.", + run_config=RunConfig(tracing_disabled=True), + ) + assert result.final_output == "INPUT_GUARDRAIL_READY" + + assert inspected == ["Check the input guardrail."] + + +async def test_tool_input_and_output_guardrails_preserve_live_execution_order( + integration_model: str, +) -> None: + observed: list[str] = [] + + @tool_input_guardrail + def inspect_input(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + observed.append(f"input:{data.context.tool_name}") + return ToolGuardrailFunctionOutput.allow() + + @tool_output_guardrail + def inspect_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + observed.append(f"output:{data.output}") + return ToolGuardrailFunctionOutput.allow() + + @tool( + tool_input_guardrails=[inspect_input], + tool_output_guardrails=[inspect_output], + ) + def guarded_lookup(value: int) -> str: + """Look up a deterministic guarded value.""" + observed.append(f"tool:{value}") + return "guarded-ready" + + agent = Agent( + name="Packaged tool guardrail agent", + model=integration_model, + instructions="Call guarded_lookup with value 42 and then reply TOOL_GUARDRAILS_READY.", + tools=[guarded_lookup], + model_settings={"max_tokens": 384}, + ) + + result = await Runner.run( + agent, + "Use the guarded lookup.", + run_config=RunConfig( + tracing_disabled=True, + tool_execution=ToolExecutionConfig(pre_approval_tool_input_guardrails=True), + ), + ) + + assert result.final_output == "TOOL_GUARDRAILS_READY" + assert observed == ["input:guarded_lookup", "tool:42", "output:guarded-ready"] + + +@pytest.mark.parametrize("blocked", [False, True], ids=["accepted", "blocked"]) +async def test_streaming_output_guardrails_validate_live_model_results( + integration_model: str, blocked: bool +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def inspect_result( + context: RunContextWrapper[Any], agent: Agent[Any], output: str + ) -> GuardrailFunctionOutput: + del context, agent + inspected.append(output) + return GuardrailFunctionOutput(output_info={"checked": True}, tripwire_triggered=blocked) + + agent = Agent( + name="Packaged streamed output guardrail agent", + model=integration_model, + instructions="Reply with exactly STREAM_GUARDED_RESULT.", + output_guardrails=[inspect_result], + model_settings={"max_tokens": 256}, + ) + result = Runner.run_streamed( + agent, + "Return the deterministic streamed guarded result.", + run_config=RunConfig(tracing_disabled=True), + ) + if blocked: + with pytest.raises(OutputGuardrailTripwireTriggered): + async for _event in result.stream_events(): + pass + else: + async for _event in result.stream_events(): + pass + assert result.final_output == "STREAM_GUARDED_RESULT" + + assert inspected == ["STREAM_GUARDED_RESULT"] diff --git a/integration_tests/openai/test_handoffs.py b/integration_tests/openai/test_handoffs.py new file mode 100644 index 0000000000..35758011d2 --- /dev/null +++ b/integration_tests/openai/test_handoffs.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + ToolCallItem, + ToolCallOutputItem, + handoff, +) +from agents.decorators import tool +from agents.extensions.handoff_filters import remove_all_tools + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +@pytest.mark.parametrize("nested", [False, True], ids=["flat-history", "nested-history"]) +async def test_client_side_handoff_preserves_tool_ownership_and_filtered_history( + integration_model: str, streaming: bool, nested: bool +) -> None: + calls: list[str] = [] + + @tool + def lookup_ticket(ticket: str) -> str: + """Return the deterministic status for a support ticket.""" + calls.append(ticket) + return "resolved" + + specialist = Agent( + name="Packaged support specialist", + model=integration_model, + instructions=( + "Call lookup_ticket exactly once with ticket='CASE-42', then answer " + "exactly HANDOFF_RESOLVED." + ), + tools=[lookup_ticket], + model_settings={"max_tokens": 512}, + ) + coordinator = Agent( + name="Packaged handoff coordinator", + model=integration_model, + instructions="Immediately transfer this support ticket to the support specialist.", + handoffs=[handoff(specialist, input_filter=remove_all_tools)], + model_settings={"max_tokens": 512}, + ) + config = RunConfig(tracing_disabled=True, nest_handoff_history=nested) + result: RunResult | RunResultStreaming + + if streaming: + streamed = Runner.run_streamed( + coordinator, "Resolve support ticket CASE-42.", run_config=config + ) + event_types = [event.type async for event in streamed.stream_events()] + assert "agent_updated_stream_event" in event_types + result = streamed + else: + result = await Runner.run(coordinator, "Resolve support ticket CASE-42.", run_config=config) + + assert calls == ["CASE-42"] + assert result.final_output == "HANDOFF_RESOLVED" + assert result.last_agent is specialist + assert any( + isinstance(item, ToolCallItem) and item.agent is specialist for item in result.new_items + ) + assert any( + isinstance(item, ToolCallOutputItem) and item.agent is specialist + for item in result.new_items + ) + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_nested_agent_as_tool_runs_against_the_installed_distribution( + integration_model: str, streaming: bool +) -> None: + worker = Agent( + name="Packaged nested worker", + model=integration_model, + instructions="Reply with exactly INNER:42.", + model_settings={"max_tokens": 256}, + ) + coordinator = Agent( + name="Packaged nested coordinator", + model=integration_model, + instructions="Call ask_worker, then reply exactly OUTER:42.", + model_settings={"max_tokens": 384}, + tools=[ + worker.as_tool( + tool_name="ask_worker", + tool_description="Ask the nested worker for the deterministic answer.", + ) + ], + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + streamed = Runner.run_streamed(coordinator, "Use the nested worker.", run_config=config) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run(coordinator, "Use the nested worker.", run_config=config) + + assert result.final_output == "OUTER:42" + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) diff --git a/integration_tests/openai/test_model_settings.py b/integration_tests/openai/test_model_settings.py new file mode 100644 index 0000000000..0796e3a6e8 --- /dev/null +++ b/integration_tests/openai/test_model_settings.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai.resources.responses import AsyncResponses +from openai.types.shared import Reasoning + +from agents import Agent, ModelSettings, RunConfig, Runner +from agents.retry import ModelRetryBackoffSettings, ModelRetrySettings + +pytestmark = pytest.mark.core + + +@pytest.fixture +def captured_response_requests(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + requests: list[dict[str, Any]] = [] + original_create = AsyncResponses.create + + async def capture_request(responses: AsyncResponses, *args: Any, **kwargs: Any) -> Any: + requests.append(kwargs) + return await original_create(responses, *args, **kwargs) + + monkeypatch.setattr(AsyncResponses, "create", capture_request) + return requests + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_agent_model_settings_reach_the_live_responses_api( + integration_model: str, dictionary: bool, captured_response_requests: list[dict[str, Any]] +) -> None: + settings: ModelSettings | dict[str, Any] + if dictionary: + settings = {"reasoning": {"effort": "low"}, "max_tokens": 256} + else: + settings = ModelSettings(reasoning=Reasoning(effort="low"), max_tokens=256) + + agent = Agent( + name="Packaged settings agent", + model=integration_model, + instructions="Reply with exactly PACKAGED_SETTINGS_OK.", + model_settings=settings, + ) + result = await Runner.run(agent, "Confirm the packaged settings path.") + + assert isinstance(agent.model_settings, ModelSettings) + assert result.final_output == "PACKAGED_SETTINGS_OK" + assert result.context_wrapper.usage.total_tokens > 0 + assert len(captured_response_requests) == 1 + assert captured_response_requests[0]["max_output_tokens"] == 256 + assert captured_response_requests[0]["reasoning"].effort == "low" + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_run_config_model_settings_reach_the_live_responses_api( + integration_model: str, dictionary: bool, captured_response_requests: list[dict[str, Any]] +) -> None: + settings: ModelSettings | dict[str, Any] + if dictionary: + settings = {"reasoning": {"effort": "low"}, "max_tokens": 256} + else: + settings = ModelSettings(reasoning=Reasoning(effort="low"), max_tokens=256) + + config = RunConfig(model_settings=settings, tracing_disabled=True) + agent = Agent( + name="Packaged run configuration agent", + model=integration_model, + instructions="Reply with exactly RUN_CONFIG_OK.", + ) + result = await Runner.run(agent, "Confirm the packaged run configuration.", run_config=config) + + assert isinstance(config.model_settings, ModelSettings) + assert result.final_output == "RUN_CONFIG_OK" + assert len(captured_response_requests) == 1 + assert captured_response_requests[0]["max_output_tokens"] == 256 + assert captured_response_requests[0]["reasoning"].effort == "low" + + +async def test_nested_retry_settings_and_clone_dictionaries_reach_the_api( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged nested settings agent", + model=integration_model, + instructions="Reply with exactly NESTED_SETTINGS_OK.", + model_settings={ + "max_tokens": 256, + "reasoning": {"effort": "low"}, + "retry": { + "max_retries": 0, + "backoff": {"initial_delay": 0.0}, + }, + }, + ) + assert isinstance(agent.model_settings.retry, ModelRetrySettings) + assert isinstance(agent.model_settings.retry.backoff, ModelRetryBackoffSettings) + cloned = agent.clone( + model_settings={ + "max_tokens": 256, + "reasoning": {"effort": "low"}, + "retry": {"max_retries": 0, "backoff": {"initial_delay": 0.0}}, + } + ) + result = await Runner.run(cloned, "Confirm provider-specific settings normalization.") + + assert isinstance(cloned.model_settings, ModelSettings) + assert isinstance(cloned.model_settings.retry, ModelRetrySettings) + assert isinstance(cloned.model_settings.retry.backoff, ModelRetryBackoffSettings) + assert result.final_output == "NESTED_SETTINGS_OK" diff --git a/integration_tests/openai/test_responses.py b/integration_tests/openai/test_responses.py new file mode 100644 index 0000000000..c67effea7b --- /dev/null +++ b/integration_tests/openai/test_responses.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from openai.resources.responses import AsyncResponses +from pydantic import BaseModel + +from agents import ( + Agent, + ModelSettings, + RunConfig, + Runner, + RunResult, + RunResultStreaming, +) +from agents.decorators import tool +from agents.items import ToolCallItem, ToolCallOutputItem + +pytestmark = pytest.mark.core + + +class StructuredStatus(BaseModel): + status: str + value: int + + +class NestedStructuredStatus(BaseModel): + result: StructuredStatus + note: str | None = None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_responses_function_tools_preserve_calls_outputs_and_usage( + integration_model: str, streaming: bool +) -> None: + called: list[int] = [] + + @tool + def double_number(value: int) -> int: + """Double the supplied number.""" + called.append(value) + return value * 2 + + agent = Agent( + name="Packaged Responses tool agent", + model=integration_model, + instructions="Call double_number with value 21, then reply exactly RESULT:42.", + tools=[double_number], + model_settings=ModelSettings(max_tokens=512), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + if streaming: + result = Runner.run_streamed(agent, "Use the tool now.", run_config=config) + events = [event async for event in result.stream_events()] + assert any(event.type == "raw_response_event" for event in events) + else: + result = await Runner.run(agent, "Use the tool now.", run_config=config) + + assert called == [21] + assert result.final_output == "RESULT:42" + assert any(isinstance(item, ToolCallItem) for item in result.new_items) + assert any(isinstance(item, ToolCallOutputItem) for item in result.new_items) + assert result.context_wrapper.usage.total_tokens > 0 + + +async def test_responses_structured_output_is_deserialized_from_the_installed_wheel( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged structured output agent", + model=integration_model, + instructions="Return status READY and value 42.", + output_type=StructuredStatus, + model_settings={"max_tokens": 256}, + ) + result = await Runner.run( + agent, + "Return the requested structured result.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert isinstance(result.final_output, StructuredStatus) + assert result.final_output.status == "READY" + assert result.final_output.value == 42 + + +async def test_previous_response_id_preserves_server_managed_conversation( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged server conversation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + first = await Runner.run( + agent, + "Remember that the secret verification word is ORCHID. Reply only STORED.", + run_config=RunConfig(tracing_disabled=True), + ) + assert first.last_response_id is not None + + second = await Runner.run( + agent, + "What verification word did I ask you to remember? Reply with only that word.", + previous_response_id=first.last_response_id, + run_config=RunConfig(tracing_disabled=True), + ) + + assert second.final_output.strip().upper() == "ORCHID" + assert second.last_response_id != first.last_response_id + + +async def test_streaming_structured_output_preserves_nested_optional_fields( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged streamed structured output agent", + model=integration_model, + instructions="Return result status READY, result value 42, and note null.", + output_type=NestedStructuredStatus, + model_settings={"max_tokens": 384}, + ) + + result = Runner.run_streamed( + agent, + "Return the nested structured status.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert isinstance(result.final_output, NestedStructuredStatus) + assert result.final_output.result == StructuredStatus(status="READY", value=42) + assert result.final_output.note is None + assert "raw_response_event" in event_types + + +async def test_explicit_prompt_cache_settings_reach_the_live_responses_api( + integration_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + captured_requests: list[dict[str, Any]] = [] + original_create = AsyncResponses.create + + async def capture_request(responses: AsyncResponses, *args: Any, **kwargs: Any) -> Any: + captured_requests.append(kwargs) + return await original_create(responses, *args, **kwargs) + + monkeypatch.setattr(AsyncResponses, "create", capture_request) + prefix = " ".join(f"release-checkpoint-{index}" for index in range(1100)) + agent = Agent( + name="Packaged prompt caching agent", + model=integration_model, + instructions="Reply with exactly PROMPT_CACHE_READY.", + model_settings=ModelSettings( + max_tokens=128, + prompt_cache_options={"mode": "explicit", "ttl": "30m"}, + extra_args={"prompt_cache_key": "packaged-integration-explicit-cache"}, + ), + ) + request_input: list[Any] = [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": prefix, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + {"type": "input_text", "text": "Reply with PROMPT_CACHE_READY."}, + ], + } + ] + + result = await Runner.run( + agent, + request_input, + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == "PROMPT_CACHE_READY" + assert result.context_wrapper.usage.input_tokens > 0 + assert len(captured_requests) == 1 + assert captured_requests[0]["prompt_cache_options"] == {"mode": "explicit", "ttl": "30m"} + assert captured_requests[0]["prompt_cache_key"] == "packaged-integration-explicit-cache" + assert captured_requests[0]["input"][0]["content"][0]["prompt_cache_breakpoint"] == { + "mode": "explicit" + } diff --git a/integration_tests/openai/test_retry.py b/integration_tests/openai/test_retry.py new file mode 100644 index 0000000000..85a0c1e480 --- /dev/null +++ b/integration_tests/openai/test_retry.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx +import pytest +from openai import APIConnectionError, AsyncOpenAI + +from agents import ( + Agent, + ModelRetrySettings, + ModelSettings, + OpenAIResponsesModel, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, + retry_policies, +) + +pytestmark = pytest.mark.core + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_retry_reaches_real_api_without_rewinding_session_input( + integration_model: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, streaming: bool +) -> None: + model = OpenAIResponsesModel(model=integration_model, openai_client=AsyncOpenAI()) + original_fetch = model._fetch_response + attempts = 0 + + async def fail_once(*args: Any, **kwargs: Any) -> Any: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise APIConnectionError( + message="Controlled integration-test transport failure.", + request=httpx.Request("POST", "https://api.openai.com/v1/responses"), + ) + return await original_fetch(*args, **kwargs) + + monkeypatch.setattr(model, "_fetch_response", fail_once) + agent = Agent( + name="Packaged real retry agent", + model=model, + instructions="Reply with exactly RETRY_RECOVERED.", + model_settings=ModelSettings( + max_tokens=256, + retry=ModelRetrySettings( + max_retries=1, + backoff={"initial_delay": 0.0}, + policy=retry_policies.network_error(), + ), + ), + ) + session = SQLiteSession("packaged-retry", tmp_path / "retry.sqlite3") + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + + try: + if streaming: + streamed = Runner.run_streamed( + agent, "Recover exactly once.", session=session, run_config=config + ) + async for _event in streamed.stream_events(): + pass + result = streamed + else: + result = await Runner.run( + agent, "Recover exactly once.", session=session, run_config=config + ) + session_items = await session.get_items() + finally: + session.close() + + assert attempts == 2 + assert result.final_output == "RETRY_RECOVERED" + assert [item.get("role") for item in session_items] == ["user", "assistant"] diff --git a/integration_tests/openai/test_sessions.py b/integration_tests/openai/test_sessions.py new file mode 100644 index 0000000000..73d3132815 --- /dev/null +++ b/integration_tests/openai/test_sessions.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import pytest +from openai import AsyncOpenAI + +from agents import ( + Agent, + ModelSettings, + OpenAIConversationsSession, + OpenAIResponsesCompactionSession, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + SQLiteSession, +) +from agents.decorators import tool + +pytestmark = pytest.mark.core + + +async def test_sqlite_session_persists_tool_history_across_reopened_instances( + integration_model: str, tmp_path: Path +) -> None: + calls: list[str] = [] + + @tool + def lookup_codeword(label: str) -> str: + """Look up the deterministic secret codeword.""" + calls.append(label) + return "MARIGOLD" + + agent = Agent( + name="Packaged SQLite session agent", + model=integration_model, + instructions="Use lookup_codeword when requested and remember its result.", + model_settings={"max_tokens": 384}, + tools=[lookup_codeword], + ) + database = tmp_path / "conversation.sqlite3" + session = SQLiteSession("packaged-live-session", database) + config = RunConfig(tracing_disabled=True) + try: + first = await Runner.run( + agent, + "Use lookup_codeword with label='release' and reply only STORED.", + session=session, + run_config=config, + ) + assert first.final_output == "STORED" + assert calls == ["release"] + finally: + session.close() + + reopened = SQLiteSession("packaged-live-session", database) + try: + second = await Runner.run( + agent, + "What exact codeword did the tool return? Answer with that word only.", + session=reopened, + run_config=config, + ) + saved_items = await reopened.get_items() + finally: + reopened.close() + + assert second.final_output.strip().upper() == "MARIGOLD" + assert calls == ["release"] + assert any(item.get("type") == "function_call_output" for item in saved_items) + + +async def test_explicit_input_replay_preserves_a_real_response_history( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged explicit replay agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run( + agent, + "Remember that the verification number is 907. Reply only STORED.", + run_config=config, + ) + replay = first.to_input_list() + assert any( + item.get("role") == "user" and "verification number is 907" in str(item.get("content")) + for item in replay + ) + second = await Runner.run( + agent, + replay + + [ + { + "role": "user", + "content": "What verification number did I provide? Reply with only the number.", + } + ], + run_config=config, + ) + + assert second.final_output.strip() == "907" + + +async def test_streamed_previous_response_id_continues_server_managed_history( + integration_model: str, +) -> None: + agent = Agent( + name="Packaged streamed continuation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + first = await Runner.run( + agent, + "Remember that the state token is IVORY. Reply only STORED.", + run_config=config, + ) + assert first.last_response_id is not None + + second = Runner.run_streamed( + agent, + "What state token did I provide? Answer with only the token.", + previous_response_id=first.last_response_id, + run_config=config, + ) + async for _event in second.stream_events(): + pass + + assert second.final_output.strip().upper() == "IVORY" + assert second.last_response_id != first.last_response_id + + +async def test_openai_conversation_id_preserves_server_owned_state( + integration_model: str, +) -> None: + client = AsyncOpenAI() + conversation = await client.conversations.create() + agent = Agent( + name="Packaged OpenAI conversation agent", + model=integration_model, + model_settings={"max_tokens": 256}, + ) + config = RunConfig(tracing_disabled=True) + + try: + await Runner.run( + agent, + "Remember the project color is CERULEAN. Reply only STORED.", + conversation_id=conversation.id, + run_config=config, + ) + second = await Runner.run( + agent, + "What is the project color? Reply with only the color.", + conversation_id=conversation.id, + run_config=config, + ) + finally: + await client.conversations.delete(conversation.id) + + assert second.final_output.strip().upper() == "CERULEAN" + + +async def test_auto_previous_response_id_preserves_tool_output_across_turns( + integration_model: str, +) -> None: + calls: list[str] = [] + + @tool + def read_checkpoint(name: str) -> str: + """Read a deterministic server-managed continuation checkpoint.""" + calls.append(name) + return "CHECKPOINT:84" + + agent = Agent( + name="Packaged automatic continuation agent", + model=integration_model, + instructions=( + "Call read_checkpoint with name='release', then reply exactly AUTO_CONTINUATION:84." + ), + model_settings={"max_tokens": 384}, + tools=[read_checkpoint], + ) + result = await Runner.run( + agent, + "Read the release checkpoint.", + auto_previous_response_id=True, + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["release"] + assert result.final_output == "AUTO_CONTINUATION:84" + assert result.last_response_id is not None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_openai_conversations_session_preserves_server_managed_history( + integration_model: str, + streaming: bool, +) -> None: + session = OpenAIConversationsSession(session_settings={"limit": 20}) + agent = Agent( + name="Packaged OpenAI Conversations session agent", + model=integration_model, + instructions="Remember user-provided release words and follow exact output instructions.", + model_settings={"max_tokens": 192}, + ) + config = RunConfig(tracing_disabled=True) + + try: + first = await Runner.run( + agent, + "Remember the release word COBALT and reply only STORED.", + session=session, + run_config=config, + ) + second: RunResult | RunResultStreaming + if streaming: + second = Runner.run_streamed( + agent, + "What release word did I give you? Reply with that word only.", + session=session, + run_config=config, + ) + async for _event in second.stream_events(): + pass + else: + second = await Runner.run( + agent, + "What release word did I give you? Reply with that word only.", + session=session, + run_config=config, + ) + items = await session.get_items() + finally: + await session.clear_session() + + assert first.final_output == "STORED" + assert second.final_output == "COBALT" + assert len(items) >= 4 + assert any("COBALT" in str(item) for item in items) + + +@pytest.mark.nightly +@pytest.mark.parametrize( + ("compaction_mode", "store"), + [("auto", False), ("previous_response_id", True)], + ids=["stateless-input", "stored-previous-response"], +) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_responses_compaction_preserves_history_across_owner_modes( + integration_model: str, + compaction_mode: Literal["auto", "previous_response_id"], + store: bool, + streaming: bool, +) -> None: + underlying = SQLiteSession(f"packaged-compaction-{compaction_mode}-{streaming}") + compacted = OpenAIResponsesCompactionSession( + session_id=underlying.session_id, + underlying_session=underlying, + model=integration_model, + compaction_mode=compaction_mode, + should_trigger_compaction=lambda context: bool(context["compaction_candidate_items"]), + ) + agent = Agent( + name="Packaged Responses compaction agent", + model=integration_model, + instructions="Remember user-provided release words and follow exact output instructions.", + model_settings=ModelSettings(store=store, max_tokens=192), + ) + config = RunConfig(tracing_disabled=True) + + try: + first = await Runner.run( + agent, + "Remember the release word JASPER and reply only STORED.", + session=compacted, + run_config=config, + ) + first_items = await underlying.get_items() + second: RunResult | RunResultStreaming + if streaming: + second = Runner.run_streamed( + agent, + "What release word did I give you? Reply with that word only.", + session=compacted, + run_config=config, + ) + async for _event in second.stream_events(): + pass + else: + second = await Runner.run( + agent, + "What release word did I give you? Reply with that word only.", + session=compacted, + run_config=config, + ) + second_items = await underlying.get_items() + finally: + underlying.close() + + assert first.final_output == "STORED" + assert second.final_output == "JASPER" + assert any(item.get("type") == "compaction" for item in first_items) + assert any(item.get("type") == "compaction" for item in second_items) diff --git a/integration_tests/openai/test_tracing.py b/integration_tests/openai/test_tracing.py new file mode 100644 index 0000000000..7212d185d1 --- /dev/null +++ b/integration_tests/openai/test_tracing.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from agents import ( + Agent, + RunConfig, + Runner, + RunResult, + RunResultStreaming, + Span, + Trace, + TracingProcessor, + set_trace_processors, + set_tracing_disabled, +) +from agents.decorators import tool +from agents.tracing import get_trace_provider + +pytestmark = [pytest.mark.core, pytest.mark.nightly] + + +class CollectingTraceProcessor(TracingProcessor): + def __init__(self) -> None: + self.started_traces: list[Trace] = [] + self.finished_traces: list[Trace] = [] + self.started_spans: list[Span[Any]] = [] + self.finished_spans: list[Span[Any]] = [] + + def on_trace_start(self, trace: Trace) -> None: + self.started_traces.append(trace) + + def on_trace_end(self, trace: Trace) -> None: + self.finished_traces.append(trace) + + def on_span_start(self, span: Span[Any]) -> None: + self.started_spans.append(span) + + def on_span_end(self, span: Span[Any]) -> None: + self.finished_spans.append(span) + + def shutdown(self) -> None: + return None + + def force_flush(self) -> None: + return None + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_live_model_and_tool_spans_finish_without_exposing_sensitive_data( + integration_model: str, monkeypatch: pytest.MonkeyPatch, streaming: bool +) -> None: + calls: list[str] = [] + + @tool + def inspect_secret(value: str) -> str: + """Inspect a deterministic sensitive verification value.""" + calls.append(value) + return "TRACE_READY" + + agent = Agent( + name="Packaged traced agent", + model=integration_model, + instructions=( + "Call inspect_secret with value='secret-token-42', then reply with exactly TRACE_READY." + ), + tools=[inspect_secret], + model_settings={"max_tokens": 384}, + ) + processor = CollectingTraceProcessor() + provider = cast(Any, get_trace_provider()) + original_processors = list(provider._multi_processor._processors) + original_env_disabled = provider._env_disabled + original_manual_disabled = provider._manual_disabled + original_disabled = provider._disabled + monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "0") + set_trace_processors([processor]) + set_tracing_disabled(False) + result: RunResult | RunResultStreaming + try: + config = RunConfig( + tracing_disabled=False, + trace_include_sensitive_data=False, + workflow_name="Packaged tracing compatibility", + ) + if streaming: + result = Runner.run_streamed(agent, "Inspect the secret.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "Inspect the secret.", run_config=config) + finally: + set_trace_processors(original_processors) + provider._env_disabled = original_env_disabled + provider._manual_disabled = original_manual_disabled + provider._disabled = original_disabled + + assert calls == ["secret-token-42"] + assert result.final_output == "TRACE_READY" + assert len(processor.started_traces) == len(processor.finished_traces) == 1 + assert len(processor.started_spans) == len(processor.finished_spans) + span_types = {span.span_data.type for span in processor.finished_spans} + assert "agent" in span_types + assert "response" in span_types + assert "function" in span_types + assert all(span.ended_at is not None for span in processor.finished_spans) + assert all("secret-token-42" not in str(span.export()) for span in processor.finished_spans) diff --git a/integration_tests/openai/test_websocket.py b/integration_tests/openai/test_websocket.py new file mode 100644 index 0000000000..949d00bfdd --- /dev/null +++ b/integration_tests/openai/test_websocket.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest + +from agents import ( + Agent, + ModelSettings, + ToolCallOutputItem, + responses_websocket_session, +) +from agents.decorators import tool +from agents.models.openai_responses import OpenAIResponsesWSModel + +pytestmark = [pytest.mark.core, pytest.mark.nightly] + + +async def test_responses_websocket_session_reuses_a_connection_across_tool_turns( + integration_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + opened_connections: list[Any] = [] + original_open = OpenAIResponsesWSModel._open_websocket_connection + + async def capture_connection( + model: OpenAIResponsesWSModel, + url: str, + headers: Mapping[str, str], + *, + connect_timeout: float | None, + ) -> Any: + connection = await original_open(model, url, headers, connect_timeout=connect_timeout) + opened_connections.append(connection) + return connection + + monkeypatch.setattr(OpenAIResponsesWSModel, "_open_websocket_connection", capture_connection) + + @tool + def lookup_checkpoint(name: str) -> str: + """Return a deterministic websocket checkpoint.""" + calls.append(name) + return "WEBSOCKET:42" + + agent = Agent( + name="Packaged Responses websocket agent", + model=integration_model, + instructions=( + "When asked to check a checkpoint, call lookup_checkpoint with name='release'. " + "For a confirmation request, reply exactly WEBSOCKET_CONFIRMED." + ), + tools=[lookup_checkpoint], + model_settings=ModelSettings(max_tokens=384), + ) + + async with responses_websocket_session() as session: + first = await session.run( + agent, + "Check the checkpoint and include WEBSOCKET:42 in your answer.", + ) + second = session.run_streamed(agent, "Reply with exactly WEBSOCKET_CONFIRMED.") + event_types = [event.type async for event in second.stream_events()] + + assert calls == ["release"] + assert "WEBSOCKET:42" in str(first.final_output) + assert any(isinstance(item, ToolCallOutputItem) for item in first.new_items) + assert second.final_output == "WEBSOCKET_CONFIRMED" + assert "raw_response_event" in event_types + assert first.context_wrapper.usage.total_tokens > 0 + assert second.context_wrapper.usage.total_tokens > 0 + assert len(opened_connections) == 1 diff --git a/integration_tests/packaging/test_distribution_contents.py b/integration_tests/packaging/test_distribution_contents.py new file mode 100644 index 0000000000..b965aac5b4 --- /dev/null +++ b/integration_tests/packaging/test_distribution_contents.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import importlib +import importlib.metadata +import importlib.util +import os +import sys +import tarfile +import warnings +import zipfile +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.packaging + + +def test_wheel_excludes_integration_tests_and_contains_runtime_modules() -> None: + wheel = Path(os.environ["OPENAI_AGENTS_INTEGRATION_WHEEL"]) + with zipfile.ZipFile(wheel) as archive: + members = archive.namelist() + + assert not any(Path(member).parts[0] == "integration_tests" for member in members) + assert "agents/py.typed" in members + assert "agents/realtime/session.py" in members + assert "agents/voice/pipeline.py" in members + assert "agents/extensions/models/any_llm_model.py" in members + assert "agents/extensions/models/litellm_model.py" in members + assert "agents/extensions/experimental/hosted_multi_agent/model.py" in members + + +def test_source_distribution_excludes_repository_automation_and_local_caches() -> None: + source_distribution = Path(os.environ["OPENAI_AGENTS_INTEGRATION_SDIST"]) + with tarfile.open(source_distribution, "r:gz") as archive: + members = archive.getnames() + + assert not any( + len(Path(member).parts) > 1 + and ( + Path(member).parts[1] in {".agents", ".github", "integration_tests"} + or Path(member).parts[1].startswith((".tmp", ".uv")) + ) + for member in members + ) + assert any(member.endswith("/src/agents/py.typed") for member in members) + + +def test_installed_distribution_advertises_expected_optional_extras() -> None: + distribution = importlib.metadata.distribution("openai-agents") + extras = set(distribution.metadata.get_all("Provides-Extra") or []) + + assert { + "any-llm", + "encrypt", + "litellm", + "realtime", + "redis", + "s3", + "sqlalchemy", + "viz", + "voice", + }.issubset(extras) + assert distribution.version + + +@pytest.mark.parametrize( + "module_name", + [ + "agents", + "agents.models.openai_responses", + "agents.models.openai_chatcompletions", + "agents.decorators", + "agents.guardrail", + "agents.handoffs", + "agents.memory", + "agents.model_settings", + "agents.realtime", + "agents.responses_websocket_session", + "agents.run", + "agents.run_config", + "agents.tool", + "agents.tool_guardrails", + "agents.tracing", + "agents.extensions.experimental.hosted_multi_agent", + ], +) +def test_public_runtime_modules_import_from_the_distribution(module_name: str) -> None: + module = importlib.import_module(module_name) + + assert module.__file__ is not None + assert "site-packages" in Path(module.__file__).parts + + +@pytest.mark.parametrize( + ("module_name", "export_name", "canonical_module", "canonical_name"), + [ + ("agents.decorators", "function_tool", "agents", "function_tool"), + ("agents.decorators", "tool", "agents", "function_tool"), + ("agents.decorators", "input_guardrail", "agents", "input_guardrail"), + ("agents.decorators", "output_guardrail", "agents", "output_guardrail"), + ("agents.decorators", "tool_input_guardrail", "agents", "tool_input_guardrail"), + ("agents.decorators", "tool_output_guardrail", "agents", "tool_output_guardrail"), + ("agents.agent", "Agent", "agents", "Agent"), + ("agents.run", "Runner", "agents", "Runner"), + ("agents.run_config", "RunConfig", "agents", "RunConfig"), + ("agents.model_settings", "ModelSettings", "agents", "ModelSettings"), + ("agents.guardrail", "input_guardrail", "agents", "input_guardrail"), + ("agents.tool", "function_tool", "agents", "function_tool"), + ("agents.tool_guardrails", "tool_input_guardrail", "agents", "tool_input_guardrail"), + ("agents.memory", "SQLiteSession", "agents", "SQLiteSession"), + ("agents.memory.sqlite_session", "SQLiteSession", "agents", "SQLiteSession"), + ( + "agents.responses_websocket_session", + "ResponsesWebSocketSession", + "agents", + "ResponsesWebSocketSession", + ), + ("agents.tracing", "TracingProcessor", "agents", "TracingProcessor"), + ( + "agents.realtime.model_events", + "RealtimeModelUsageEvent", + "agents.realtime", + "RealtimeModelUsageEvent", + ), + ], +) +def test_supported_import_paths_resolve_to_canonical_runtime_objects( + module_name: str, + export_name: str, + canonical_module: str, + canonical_name: str, +) -> None: + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", DeprecationWarning) + module = importlib.import_module(module_name) + canonical = importlib.import_module(canonical_module) + actual = getattr(module, export_name) + + assert actual is getattr(canonical, canonical_name) + assert not any(isinstance(warning.message, DeprecationWarning) for warning in captured) + + +def test_decorators_module_exports_supported_runtime_aliases() -> None: + decorators = importlib.import_module("agents.decorators") + + assert decorators.__all__ == [ + "function_tool", + "input_guardrail", + "output_guardrail", + "tool", + "tool_input_guardrail", + "tool_output_guardrail", + ] + assert decorators.tool is decorators.function_tool + + def legacy_status() -> str: + """Return the supported legacy decorator status.""" + return "LEGACY_DECORATOR_READY" + + decorated_status = decorators.tool(legacy_status) + assert decorated_status.name == "legacy_status" + + +@pytest.mark.parametrize( + ("module_name", "dependency_name", "expected_extra"), + [ + ("agents.extensions.models.any_llm_model", "any_llm", "any-llm"), + ("agents.extensions.models.litellm_model", "litellm", "litellm"), + ], +) +def test_optional_provider_modules_fail_with_actionable_install_guidance( + module_name: str, dependency_name: str, expected_extra: str +) -> None: + if importlib.util.find_spec(dependency_name) is not None: + pytest.skip(f"{dependency_name} is already installed in this isolated environment.") + + sys.modules.pop(module_name, None) + with pytest.raises(ImportError, match=rf"openai-agents\[{expected_extra}\]"): + importlib.import_module(module_name) diff --git a/integration_tests/packaging/test_optional_extras.py b/integration_tests/packaging/test_optional_extras.py new file mode 100644 index 0000000000..45e586a5d3 --- /dev/null +++ b/integration_tests/packaging/test_optional_extras.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.extras + + +def test_requested_optional_extra_imports_from_its_standalone_environment() -> None: + optional_extra = os.environ["OPENAI_AGENTS_INTEGRATION_EXTRA"] + module_names = { + "any-llm": "agents.extensions.models.any_llm_model", + "encrypt": "agents.extensions.memory.encrypt_session", + "litellm": "agents.extensions.models.litellm_model", + "realtime": "agents.realtime", + "redis": "agents.extensions.memory.redis_session", + "s3": "boto3", + "sqlalchemy": "agents.extensions.memory.sqlalchemy_session", + "viz": "agents.extensions.visualization", + "voice": "agents.voice", + } + module = importlib.import_module(module_names[optional_extra]) + + assert module.__file__ is not None + assert "site-packages" in Path(module.__file__).parts + + +@pytest.mark.parametrize( + ("optional_extra", "package_symbol", "module_name", "module_symbol"), + [ + ( + "encrypt", + "EncryptedSession", + "agents.extensions.memory.encrypt_session", + "EncryptedSession", + ), + ("redis", "RedisSession", "agents.extensions.memory.redis_session", "RedisSession"), + ( + "sqlalchemy", + "SQLAlchemySession", + "agents.extensions.memory.sqlalchemy_session", + "SQLAlchemySession", + ), + ], +) +def test_memory_extra_lazy_exports_resolve_to_the_installed_backend( + optional_extra: str, + package_symbol: str, + module_name: str, + module_symbol: str, +) -> None: + if os.environ["OPENAI_AGENTS_INTEGRATION_EXTRA"] != optional_extra: + pytest.skip(f"This environment does not include the {optional_extra} extra.") + + memory = importlib.import_module("agents.extensions.memory") + module = importlib.import_module(module_name) + + assert getattr(memory, package_symbol) is getattr(module, module_symbol) diff --git a/integration_tests/packaging/test_provider_selection.py b/integration_tests/packaging/test_provider_selection.py new file mode 100644 index 0000000000..496487414a --- /dev/null +++ b/integration_tests/packaging/test_provider_selection.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import runpy +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import pytest +from conftest import _external_providers, pytest_runtest_setup + +pytestmark = pytest.mark.packaging + + +def _configure_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-test-key") + monkeypatch.setenv("GEMINI_API_KEY", "gemini-test-key") + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_OPENROUTER_MODELS", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_ANTHROPIC_MODEL", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_GEMINI_MODEL", raising=False) + + +def test_external_provider_coverage_is_explicitly_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + + assert _external_providers() == [] + + +@pytest.mark.parametrize( + "credential_name", ["OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"] +) +def test_external_provider_tests_do_not_require_an_openai_api_key( + monkeypatch: pytest.MonkeyPatch, + credential_name: str, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv(credential_name, "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["external_provider"], + callspec=SimpleNamespace(params={"external_provider": object()}), + get_closest_marker=lambda name: name == "providers", + ) + + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_openai_backed_provider_tests_require_an_openai_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv("OPENROUTER_API_KEY", "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["integration_model"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set a real OPENAI_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +@pytest.mark.parametrize( + ("fixture_name", "model_environment", "model_name", "credential_name"), + [ + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "openrouter/openai/gpt-5.6-luna", + "OPENROUTER_API_KEY", + ), + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "anthropic/claude-sonnet-5", + "ANTHROPIC_API_KEY", + ), + ( + "any_llm_models", + "OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", + "gemini/gemini-3.6-flash", + "GEMINI_API_KEY", + ), + ( + "litellm_models", + "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", + "openrouter/google/gemini-3.6-flash", + "OPENROUTER_API_KEY", + ), + ( + "litellm_models", + "OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", + "gemini/gemini-3.6-flash", + "GOOGLE_API_KEY", + ), + ], +) +def test_configured_provider_models_use_provider_specific_credentials( + monkeypatch: pytest.MonkeyPatch, + fixture_name: str, + model_environment: str, + model_name: str, + credential_name: str, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv(model_environment, model_name) + monkeypatch.setenv(credential_name, "provider-test-key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=[fixture_name], + get_closest_marker=lambda name: name == "providers", + ) + + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_configured_provider_models_require_their_own_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", "openrouter/openai/gpt-5.6-luna") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["any_llm_models"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set OPENROUTER_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_configured_openai_provider_rejects_placeholder_api_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "test_key") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_LITELLM_MODELS", "openai/gpt-4.1-mini") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + item = SimpleNamespace( + fixturenames=["litellm_models"], + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="Set OPENAI_API_KEY"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +def test_strict_mode_requires_requested_external_provider_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + assert _external_providers() == [] + item = SimpleNamespace( + fixturenames=["external_provider"], + callspec=SimpleNamespace(params={"external_provider": None}), + get_closest_marker=lambda name: name == "providers", + ) + + with pytest.raises(pytest.fail.Exception, match="External provider coverage requires"): + pytest_runtest_setup(cast(pytest.Item, item)) + + +@pytest.mark.parametrize( + ("model", "expected_extra"), + [ + ("anthropic/claude-sonnet-5", "anthropic"), + ("gemini/gemini-3.6-flash", "gemini"), + ("google/gemini-3.6-flash", "gemini"), + ("openrouter/openai/gpt-5.6-luna", "openrouter"), + ], +) +def test_configured_any_llm_models_install_provider_extras_without_external_matrix( + monkeypatch: pytest.MonkeyPatch, model: str, expected_extra: str +) -> None: + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_ANY_LLM_MODELS", model) + runner_path = Path(__file__).resolve().parents[2] / ".github/scripts/run_integration_tests.py" + runner = runpy.run_path(str(runner_path)) + + assert runner["_any_llm_provider_extras"]( + external_providers_enabled=False, direct_providers_enabled=False + ) == [expected_extra] + + +def test_strict_mode_does_not_require_unrequested_external_providers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + assert _external_providers() == [] + + +def test_strict_mode_accepts_explicit_direct_provider_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_STRICT", "1") + + providers = _external_providers() + + assert [provider.name for provider in providers] == ["anthropic"] + + +def test_external_provider_coverage_defaults_to_current_openrouter_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.delenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", raising=False) + + providers = _external_providers() + + assert [provider.name for provider in providers] == [ + "openrouter-openai-gpt-5.6-luna", + "openrouter-anthropic-claude-sonnet-5", + "openrouter-google-gemini-3.6-flash", + ] + assert [provider.model for provider in providers] == [ + "openrouter/openai/gpt-5.6-luna", + "openrouter/anthropic/claude-sonnet-5", + "openrouter/google/gemini-3.6-flash", + ] + + +def test_all_provider_coverage_adds_explicit_direct_provider_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure_provider_credentials(monkeypatch) + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_EXTERNAL_PROVIDERS", "1") + monkeypatch.setenv("OPENAI_AGENTS_INTEGRATION_DIRECT_PROVIDERS", "1") + + providers = _external_providers() + + assert [provider.name for provider in providers] == [ + "openrouter-openai-gpt-5.6-luna", + "openrouter-anthropic-claude-sonnet-5", + "openrouter-google-gemini-3.6-flash", + "anthropic", + "gemini", + ] + assert [provider.model for provider in providers[-2:]] == [ + "anthropic/claude-sonnet-5", + "gemini/gemini-3.6-flash", + ] diff --git a/integration_tests/providers/test_any_llm.py b/integration_tests/providers/test_any_llm.py new file mode 100644 index 0000000000..5291032d29 --- /dev/null +++ b/integration_tests/providers/test_any_llm.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import Agent, ModelSettings, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool + +pytestmark = pytest.mark.providers + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_any_llm_configured_providers_execute_real_function_tools( + any_llm_models: list[str], dictionary: bool +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[int] = [] + + @tool + def lookup_number(value: int) -> int: + """Return the supplied deterministic number.""" + calls.append(value) + return value + + for model_name in any_llm_models: + calls.clear() + settings: ModelSettings | dict[str, Any] + settings = {"max_tokens": 512} if dictionary else ModelSettings(max_tokens=512) + agent = Agent( + name="Packaged AnyLLM agent", + model=AnyLLMModel(model=model_name), + instructions="Call lookup_number with 42 and then reply exactly ANY_LLM:42.", + model_settings=settings, + tools=[lookup_number], + ) + result = await Runner.run( + agent, + "Use the number tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == [42], model_name + assert result.final_output == "ANY_LLM:42", model_name + assert result.context_wrapper.usage.total_tokens > 0, model_name + + +@pytest.mark.parametrize("api", ["responses", "chat_completions"]) +async def test_any_llm_openai_supports_both_api_families(integration_model: str, api: str) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + agent = Agent( + name="Packaged AnyLLM API selector", + model=AnyLLMModel(model=f"openai/{integration_model}", api=api), # type: ignore[arg-type] + instructions="Reply with exactly ANY_LLM_API_OK.", + model_settings={"max_tokens": 256}, + ) + result = await Runner.run( + agent, + "Confirm the selected provider API.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert result.final_output == "ANY_LLM_API_OK" + + +@pytest.mark.filterwarnings( + "ignore:Inheritance class AiohttpClientSession from ClientSession is discouraged:" + r"DeprecationWarning:google\.genai\._api_client" +) +async def test_any_llm_major_external_providers_execute_function_tools( + external_provider: Any, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[str] = [] + + @tool + def provider_status(provider: str) -> str: + """Return the deterministic provider readiness status.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged AnyLLM external provider agent", + model=AnyLLMModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call provider_status exactly once with provider='external', " + "then reply exactly PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[provider_status], + ) + result = await Runner.run( + agent, + "Check the provider with its function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["external"] + assert result.final_output == "PROVIDER_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.nightly +@pytest.mark.filterwarnings( + "ignore:Inheritance class AiohttpClientSession from ClientSession is discouraged:" + r"DeprecationWarning:google\.genai\._api_client" +) +async def test_any_llm_external_provider_streams_function_tool_results( + external_provider: Any, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + calls: list[str] = [] + + @tool + def check_streaming_provider(provider: str) -> str: + """Return the provider's deterministic streaming readiness.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged AnyLLM streaming external provider agent", + model=AnyLLMModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call check_streaming_provider exactly once with provider='external', " + "then reply exactly STREAMING_PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[check_streaming_provider], + ) + result = Runner.run_streamed( + agent, + "Check the streamed external provider function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert calls == ["external"] + assert result.final_output == "STREAMING_PROVIDER_READY" + assert "raw_response_event" in event_types + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_any_llm_chat_completions_preserves_real_token_logprobs( + streaming: bool, +) -> None: + from agents.extensions.models.any_llm_model import AnyLLMModel + + agent = Agent( + name="Packaged AnyLLM token logprob agent", + model=AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions"), + instructions="Reply with exactly BLUE.", + model_settings=ModelSettings(top_logprobs=2, max_tokens=32), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "What color is the sky? Reply BLUE.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "What color is the sky? Reply BLUE.", run_config=config) + + texts = [ + content + for response in result.raw_responses + for item in response.output + for content in getattr(item, "content", []) + if getattr(content, "type", None) == "output_text" + ] + assert result.final_output == "BLUE" + assert texts + assert any(getattr(content, "logprobs", None) for content in texts) diff --git a/integration_tests/providers/test_litellm.py b/integration_tests/providers/test_litellm.py new file mode 100644 index 0000000000..fd1d2d371b --- /dev/null +++ b/integration_tests/providers/test_litellm.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from agents import Agent, ModelSettings, RunConfig, Runner, RunResult, RunResultStreaming +from agents.decorators import tool + +pytestmark = pytest.mark.providers + + +@pytest.mark.parametrize("dictionary", [False, True], ids=["typed", "dictionary"]) +async def test_litellm_configured_providers_execute_real_function_tools( + litellm_models: list[str], dictionary: bool +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def lookup_package(name: str) -> str: + """Return the deterministic package health.""" + calls.append(name) + return "healthy" + + for model_name in litellm_models: + calls.clear() + settings: ModelSettings | dict[str, Any] + values: dict[str, Any] = {"max_tokens": 512} + settings = values if dictionary else ModelSettings(**values) + agent = Agent( + name="Packaged LiteLLM agent", + model=LitellmModel(model=model_name), + instructions=( + "Call lookup_package with name='openai-agents', then reply exactly LITELLM_OK." + ), + model_settings=settings, + tools=[lookup_package], + ) + result = await Runner.run( + agent, + "Check the installed package.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["openai-agents"], model_name + assert result.final_output == "LITELLM_OK", model_name + assert result.context_wrapper.usage.total_tokens > 0, model_name + + +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +async def test_litellm_streaming_preserves_real_provider_usage(integration_model: str) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + agent = Agent( + name="Packaged LiteLLM streaming agent", + model=LitellmModel(model=f"openai/{integration_model}"), + instructions="Reply with exactly LITELLM_STREAM_OK.", + model_settings={"max_tokens": 256}, + ) + result = Runner.run_streamed( + agent, + "Confirm the streaming provider path.", + run_config=RunConfig(tracing_disabled=True), + ) + async for _event in result.stream_events(): + pass + + assert result.final_output == "LITELLM_STREAM_OK" + assert result.context_wrapper.usage.total_tokens > 0 + + +async def test_litellm_major_external_providers_execute_function_tools( + external_provider: Any, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def provider_status(provider: str) -> str: + """Return the deterministic provider readiness status.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged LiteLLM external provider agent", + model=LitellmModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call provider_status exactly once with provider='external', " + "then reply exactly PROVIDER_READY." + ), + model_settings={"max_tokens": 512}, + tools=[provider_status], + ) + result = await Runner.run( + agent, + "Check the provider with its function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + + assert calls == ["external"] + assert result.final_output == "PROVIDER_READY" + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.nightly +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +async def test_litellm_external_provider_streams_function_tool_results( + external_provider: Any, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + calls: list[str] = [] + + @tool + def check_streaming_provider(provider: str) -> str: + """Return the provider's deterministic streaming readiness.""" + calls.append(provider) + return "ready" + + agent = Agent( + name="Packaged LiteLLM streaming external provider agent", + model=LitellmModel(model=external_provider.model, api_key=external_provider.api_key), + instructions=( + "Call check_streaming_provider exactly once with provider='external', " + "then reply exactly STREAMING_PROVIDER_READY." + ), + model_settings={"max_tokens": 512, "include_usage": True}, + tools=[check_streaming_provider], + ) + result = Runner.run_streamed( + agent, + "Check the streamed external provider function tool.", + run_config=RunConfig(tracing_disabled=True), + ) + event_types = [event.type async for event in result.stream_events()] + + assert calls == ["external"] + assert result.final_output == "STREAMING_PROVIDER_READY" + assert "raw_response_event" in event_types + assert result.context_wrapper.usage.total_tokens > 0 + + +@pytest.mark.filterwarnings( + "ignore:Accessing the 'model_(computed_)?fields' attribute on the instance is deprecated:" + "pydantic.warnings.PydanticDeprecatedSince211:" + r"litellm\.litellm_core_utils\.model_response_utils" +) +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstreaming", "streaming"]) +async def test_litellm_preserves_real_token_logprobs( + streaming: bool, +) -> None: + from agents.extensions.models.litellm_model import LitellmModel + + agent = Agent( + name="Packaged LiteLLM token logprob agent", + model=LitellmModel(model="openai/gpt-4.1-mini"), + instructions="Reply with exactly BLUE.", + model_settings=ModelSettings(top_logprobs=2, max_tokens=32), + ) + config = RunConfig(tracing_disabled=True) + result: RunResult | RunResultStreaming + if streaming: + result = Runner.run_streamed(agent, "What color is the sky? Reply BLUE.", run_config=config) + async for _event in result.stream_events(): + pass + else: + result = await Runner.run(agent, "What color is the sky? Reply BLUE.", run_config=config) + + texts = [ + content + for response in result.raw_responses + for item in response.output + for content in getattr(item, "content", []) + if getattr(content, "type", None) == "output_text" + ] + assert result.final_output == "BLUE" + assert texts + assert any(getattr(content, "logprobs", None) for content in texts) diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini new file mode 100644 index 0000000000..ab59a65e6e --- /dev/null +++ b/integration_tests/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +asyncio_mode = auto +asyncio_default_fixture_loop_scope = session +asyncio_default_test_loop_scope = session +timeout = 75 +testpaths = . +markers = + packaging: Distribution contents and installed-package boundaries. + extras: Independently installed optional dependency groups. + core: Live OpenAI Responses and Chat Completions coverage. + providers: Live AnyLLM and LiteLLM provider-adapter coverage. + realtime: Live OpenAI Realtime WebSocket coverage. + voice: Live OpenAI speech-to-text and text-to-speech coverage. + hosted: Live hosted MCP, multi-agent, and programmatic tool coverage. + nightly: Extended integration coverage selected by the nightly and manual profiles. + manual: Expensive or externally provisioned integration coverage selected manually. diff --git a/integration_tests/realtime/test_realtime.py b/integration_tests/realtime/test_realtime.py new file mode 100644 index 0000000000..daa7c4ffb2 --- /dev/null +++ b/integration_tests/realtime/test_realtime.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from agents import GuardrailFunctionOutput, ToolGuardrailFunctionOutput, ToolInputGuardrailData +from agents.decorators import output_guardrail, tool, tool_input_guardrail +from agents.realtime import ( + AssistantMessageItem, + AssistantText, + InputAudio, + RealtimeAgent, + RealtimeGuardrailTripped, + RealtimeHandoffEvent, + RealtimeHistoryAdded, + RealtimeHistoryUpdated, + RealtimeModelUsageEvent, + RealtimeRawModelEvent, + RealtimeRunner, + RealtimeToolApprovalRequired, + UserMessageItem, +) + +pytestmark = pytest.mark.realtime + + +async def test_realtime_text_session_completes_and_updates_history( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime agent", + instructions="Reply with exactly REALTIME_READY.", + ) + runner = RealtimeRunner(agent) + observed_events: list[str] = [] + assistant_text: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Confirm the realtime connection.") + + async def receive() -> None: + async for event in session: + observed_events.append(event.type) + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if not isinstance(item, AssistantMessageItem): + continue + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await asyncio.wait_for(receive(), timeout=45) + + assert "agent_start" in observed_events + assert "agent_end" in observed_events + assert any("REALTIME_READY" in text for text in assistant_text) + + +async def test_realtime_function_tool_emits_start_and_end_events( + integration_realtime_model: str, +) -> None: + calls: list[str] = [] + + @tool + def lookup_city(city: str) -> str: + """Return a deterministic city status.""" + calls.append(city) + return "sunny" + + agent = RealtimeAgent( + name="Packaged realtime tool agent", + instructions="Call lookup_city with Tokyo, then reply with TOKYO_SUNNY.", + tools=[lookup_city], + ) + runner = RealtimeRunner(agent) + observed_events: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("What is the weather in Tokyo? Use the function tool.") + + async def receive() -> None: + async for event in session: + observed_events.append(event.type) + if event.type == "agent_end" and "tool_end" in observed_events: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == ["Tokyo"] + assert "tool_start" in observed_events + assert "tool_end" in observed_events + + +async def test_realtime_session_preserves_history_across_text_turns( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime conversation agent", + instructions="Remember user-provided verification words and answer concisely.", + ) + runner = RealtimeRunner(agent) + assistant_text: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + + async def receive_turn() -> None: + async for event in session: + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if isinstance(item, AssistantMessageItem): + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await session.send_message("Remember the verification word SIERRA. Reply only STORED.") + await asyncio.wait_for(receive_turn(), timeout=45) + await session.send_message("What was the verification word? Reply only with that word.") + await asyncio.wait_for(receive_turn(), timeout=45) + + assert any("SIERRA" in text.upper() for text in assistant_text) + + +async def test_realtime_usage_events_accumulate_once_per_completed_turn( + integration_realtime_model: str, +) -> None: + agent = RealtimeAgent( + name="Packaged realtime usage agent", + instructions="Reply with exactly REALTIME_USAGE_READY.", + ) + runner = RealtimeRunner(agent) + observed_usage: list[int] = [] + completed_totals: list[int] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + + async def receive_turn() -> None: + async for event in session: + if isinstance(event, RealtimeRawModelEvent) and isinstance( + event.data, RealtimeModelUsageEvent + ): + observed_usage.append(event.data.usage.total_tokens) + if event.type == "agent_end": + completed_totals.append(event.info.context.usage.total_tokens) + return + + await session.send_message("Confirm realtime usage for turn one.") + await asyncio.wait_for(receive_turn(), timeout=45) + await session.send_message("Confirm realtime usage for turn two.") + await asyncio.wait_for(receive_turn(), timeout=45) + + assert len(observed_usage) == 2 + assert all(value > 0 for value in observed_usage) + assert completed_totals == [observed_usage[0], sum(observed_usage)] + + +async def test_realtime_handoff_updates_the_active_agent( + integration_realtime_model: str, +) -> None: + specialist = RealtimeAgent( + name="Packaged realtime specialist", + instructions="Reply with exactly REALTIME_HANDOFF_READY.", + ) + coordinator = RealtimeAgent( + name="Packaged realtime coordinator", + instructions="Immediately transfer to the packaged realtime specialist.", + handoffs=[specialist], + ) + runner = RealtimeRunner(coordinator) + handoffs: list[RealtimeHandoffEvent] = [] + ended_agents: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Transfer me to the specialist.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeHandoffEvent): + handoffs.append(event) + if event.type == "agent_end": + ended_agents.append(event.agent.name) + if event.agent is specialist: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert len(handoffs) == 1 + assert handoffs[0].from_agent is coordinator + assert handoffs[0].to_agent is specialist + assert specialist.name in ended_agents + + +async def test_realtime_update_agent_replaces_instructions_and_tool_dispatch( + integration_realtime_model: str, +) -> None: + calls: list[str] = [] + + @tool + def replacement_checkpoint(checkpoint: str) -> str: + """Resolve the replacement agent's release checkpoint.""" + calls.append(checkpoint) + return "REALTIME_UPDATED_READY" + + initial = RealtimeAgent( + name="Packaged initial realtime agent", + instructions="Reply only INITIAL_AGENT_ACTIVE.", + ) + replacement = RealtimeAgent( + name="Packaged replacement realtime agent", + instructions=( + "Call replacement_checkpoint with checkpoint='updated', " + "then reply exactly REALTIME_UPDATED_READY." + ), + tools=[replacement_checkpoint], + ) + runner = RealtimeRunner(initial, config={"async_tool_calls": False}) + ended_agents: list[str] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.update_agent(replacement) + await session.send_message( + "You must call replacement_checkpoint with checkpoint='updated'. " + "Do not reply before calling the function." + ) + + async def receive() -> None: + async for event in session: + if event.type == "agent_end": + ended_agents.append(event.agent.name) + if event.agent is replacement: + if not calls and len(ended_agents) == 1: + await session.send_message( + "Call replacement_checkpoint now with checkpoint='updated'." + ) + continue + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == ["updated"] + assert ended_agents + assert all(name == replacement.name for name in ended_agents) + + +@pytest.mark.nightly +@pytest.mark.parametrize("approved", [False, True], ids=["rejected", "approved"]) +async def test_realtime_function_tool_approval_controls_side_effects( + integration_realtime_model: str, + approved: bool, +) -> None: + calls: list[str] = [] + approvals: list[str] = [] + + @tool(needs_approval=True) + def publish_checkpoint(checkpoint: str) -> str: + """Publish a release checkpoint only after approval.""" + calls.append(checkpoint) + return "REALTIME_APPROVED" + + agent = RealtimeAgent( + name="Packaged realtime approval agent", + instructions=( + "Call publish_checkpoint with checkpoint='release'. If it succeeds reply " + "REALTIME_APPROVED. If it is rejected reply REALTIME_REJECTED." + ), + tools=[publish_checkpoint], + ) + runner = RealtimeRunner(agent) + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Publish the release checkpoint with the tool.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeToolApprovalRequired): + approvals.append(event.call_id) + if approved: + await session.approve_tool_call(event.call_id) + else: + await session.reject_tool_call( + event.call_id, + rejection_message="Publishing the checkpoint was rejected.", + ) + if approved and event.type == "tool_end" and approvals: + return + if not approved and event.type == "agent_end" and approvals: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert len(approvals) == 1 + assert calls == (["release"] if approved else []) + + +@pytest.mark.nightly +async def test_realtime_accepts_committed_pcm_audio_input( + integration_realtime_model: str, integration_pcm_audio: bytes +) -> None: + agent = RealtimeAgent( + name="Packaged realtime audio input agent", + instructions="Respond to the user's speech with exactly REALTIME_AUDIO_READY.", + ) + runner = RealtimeRunner(agent) + assistant_text: list[str] = [] + received_audio = False + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_audio(integration_pcm_audio, commit=True) + await session.send_message("Respond to the committed user audio.") + + async def receive() -> None: + nonlocal received_audio + async for event in session: + if isinstance(event, RealtimeHistoryAdded | RealtimeHistoryUpdated): + items = ( + [event.item] if isinstance(event, RealtimeHistoryAdded) else event.history + ) + for item in items: + if isinstance(item, UserMessageItem): + received_audio = received_audio or any( + isinstance(content, InputAudio) for content in item.content + ) + if isinstance(item, AssistantMessageItem): + assistant_text.extend( + content.text + for content in item.content + if isinstance(content, AssistantText) and content.text + ) + if event.type == "agent_end": + return + + await asyncio.wait_for(receive(), timeout=60) + + assert received_audio + assert any("REALTIME_AUDIO_READY" in text for text in assistant_text) + + +@pytest.mark.nightly +async def test_realtime_output_guardrails_interrupt_audio_transcripts( + integration_realtime_model: str, +) -> None: + inspected: list[str] = [] + + @output_guardrail + async def reject_release_output( + _context: object, _agent: object, text: str + ) -> GuardrailFunctionOutput: + inspected.append(text) + return GuardrailFunctionOutput(output_info={"blocked": True}, tripwire_triggered=True) + + agent = RealtimeAgent( + name="Packaged guarded realtime output agent", + instructions="Reply with exactly BLOCKED_RELEASE_CONTENT.", + output_guardrails=[reject_release_output], + ) + runner = RealtimeRunner( + agent, + config={ + "output_guardrails": [reject_release_output], + "guardrails_settings": {"debounce_text_length": 5}, + }, + ) + tripped: list[RealtimeGuardrailTripped] = [] + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["audio"], + } + } + ) as session: + await session.send_message("Return the blocked release content.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeGuardrailTripped): + tripped.append(event) + return + + await asyncio.wait_for(receive(), timeout=45) + + assert len(tripped) == 1 + assert len(inspected) == 1 + assert tripped[0].message == inspected[0] + assert tripped[0].guardrail_results[0].output.tripwire_triggered + + +@pytest.mark.nightly +@pytest.mark.parametrize("pre_approval", [False, True], ids=["after-approval", "before-approval"]) +async def test_realtime_tool_input_guardrails_control_approval_and_execution( + integration_realtime_model: str, + pre_approval: bool, +) -> None: + approvals: list[str] = [] + calls: list[str] = [] + checks: list[str] = [] + + @tool_input_guardrail + def block_checkpoint(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + checks.append(data.context.tool_name) + return ToolGuardrailFunctionOutput.reject_content("Release execution was blocked.") + + @tool(needs_approval=True, tool_input_guardrails=[block_checkpoint]) + def guarded_checkpoint(value: str) -> str: + """Execute a release checkpoint only when its guardrail allows it.""" + calls.append(value) + return "CHECKPOINT_READY" + + agent = RealtimeAgent( + name="Packaged guarded realtime tool agent", + instructions=( + "Call guarded_checkpoint with value='release'. If blocked, reply exactly " + "REALTIME_TOOL_BLOCKED." + ), + tools=[guarded_checkpoint], + ) + runner = RealtimeRunner( + agent, + config={"tool_execution": {"pre_approval_tool_input_guardrails": pre_approval}}, + ) + + async with await runner.run( + model_config={ + "initial_model_settings": { + "model_name": integration_realtime_model, + "output_modalities": ["text"], + } + } + ) as session: + await session.send_message("Execute the protected release checkpoint.") + + async def receive() -> None: + async for event in session: + if isinstance(event, RealtimeToolApprovalRequired): + approvals.append(event.call_id) + await session.approve_tool_call(event.call_id) + if event.type == "agent_end" and checks: + return + + await asyncio.wait_for(receive(), timeout=60) + + assert calls == [] + assert checks == ["guarded_checkpoint"] + assert len(approvals) == (0 if pre_approval else 1) diff --git a/integration_tests/voice/test_voice_pipeline.py b/integration_tests/voice/test_voice_pipeline.py new file mode 100644 index 0000000000..60b4af9e4d --- /dev/null +++ b/integration_tests/voice/test_voice_pipeline.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from agents import Agent + +pytestmark = pytest.mark.voice + + +@pytest.mark.parametrize("audio_dtype", ["int16", "float32"]) +async def test_static_voice_pipeline_transcribes_and_synthesizes_without_audio_devices( + integration_model: str, + integration_pcm_audio: bytes, + audio_dtype: str, +) -> None: + import numpy as np + + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + SingleAgentWorkflowCallbacks, + VoicePipeline, + VoiceStreamEventAudio, + VoiceStreamEventLifecycle, + ) + + pcm_audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + audio = ( + pcm_audio.astype(np.float32) / np.float32(32767.0) + if audio_dtype == "float32" + else pcm_audio + ) + original_audio = audio.copy() + transcriptions: list[str] = [] + + class RecordingWorkflowCallbacks(SingleAgentWorkflowCallbacks): + def on_run(self, workflow: SingleAgentVoiceWorkflow, transcription: str) -> None: + transcriptions.append(transcription) + + agent: Agent[Any] = Agent( + name="Packaged voice workflow agent", + model=integration_model, + instructions="Reply with exactly VOICE READY.", + model_settings={"max_tokens": 256}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent, callbacks=RecordingWorkflowCallbacks()), + config={ + "tracing_disabled": True, + "stt_settings": {"language": "en"}, + "tts_settings": {"voice": "alloy"}, + }, + ) + result = await pipeline.run(AudioInput(buffer=audio)) + lifecycle: list[str] = [] + audio_chunks = 0 + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + lifecycle.append(event.event) + elif isinstance(event, VoiceStreamEventAudio) and event.data is not None: + audio_chunks += 1 + + assert audio.size > 0 + np.testing.assert_array_equal(audio, original_audio) + assert len(transcriptions) == 1 + assert all(word in transcriptions[0].lower() for word in ("packaged", "voice", "ready")) + assert audio_chunks > 0 + assert lifecycle == ["turn_started", "turn_ended", "session_ended"] + + +@pytest.mark.nightly +@pytest.mark.parametrize("audio_dtype", ["int16", "float32"]) +async def test_streamed_voice_pipeline_transcribes_chunked_input_and_runs_a_function_tool( + integration_model: str, + integration_pcm_audio: bytes, + audio_dtype: str, +) -> None: + import numpy as np + from openai import AsyncOpenAI + + from agents.decorators import tool + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + StreamedAudioInput, + StreamedTranscriptionSession, + STTModel, + STTModelSettings, + VoicePipeline, + VoiceStreamEventAudio, + VoiceStreamEventLifecycle, + ) + + class BoundedTranscriptionSession(StreamedTranscriptionSession): + def __init__(self, audio_input: StreamedAudioInput, client: AsyncOpenAI) -> None: + self.audio_input = audio_input + self.client = client + self.closed = False + + async def transcribe_turns(self) -> AsyncIterator[str]: + buffers: list[Any] = [] + while True: + chunk = await self.audio_input.queue.get() + if chunk is None: + break + buffers.append(chunk) + response = await self.client.audio.transcriptions.create( + model="gpt-4o-mini-transcribe", + file=AudioInput(buffer=np.concatenate(buffers)).to_audio_file(), + ) + yield response.text + + async def close(self) -> None: + self.closed = True + + class BoundedLiveSTTModel(STTModel): + def __init__(self) -> None: + self.client = AsyncOpenAI() + self.session: BoundedTranscriptionSession | None = None + + @property + def model_name(self) -> str: + return "gpt-4o-mini-transcribe" + + async def transcribe( + self, + input: AudioInput, + settings: STTModelSettings, + trace_include_sensitive_data: bool, + trace_include_sensitive_audio_data: bool, + ) -> str: + del settings, trace_include_sensitive_data, trace_include_sensitive_audio_data + response = await self.client.audio.transcriptions.create( + model=self.model_name, + file=input.to_audio_file(), + ) + return response.text + + async def create_session( + self, + input: StreamedAudioInput, + settings: STTModelSettings, + trace_include_sensitive_data: bool, + trace_include_sensitive_audio_data: bool, + ) -> StreamedTranscriptionSession: + del settings, trace_include_sensitive_data, trace_include_sensitive_audio_data + self.session = BoundedTranscriptionSession(input, self.client) + return self.session + + calls: list[str] = [] + + @tool + def voice_status(value: str) -> str: + """Return a deterministic streamed voice readiness status.""" + calls.append(value) + return "ready" + + stt_model = BoundedLiveSTTModel() + agent: Agent[Any] = Agent( + name="Packaged streamed voice workflow agent", + model=integration_model, + instructions=( + "Call voice_status with value='streamed', then reply exactly STREAMED_VOICE_READY." + ), + tools=[voice_status], + model_settings={"max_tokens": 384}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent), + stt_model=stt_model, + config={"tracing_disabled": True, "tts_settings": {"voice": "alloy"}}, + ) + streamed_input = StreamedAudioInput() + pcm_audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + audio = ( + pcm_audio.astype(np.float32) / np.float32(32767.0) + if audio_dtype == "float32" + else pcm_audio + ) + original_audio = audio.copy() + midpoint = len(audio) // 2 + await streamed_input.add_audio(audio[:midpoint]) + await streamed_input.add_audio(audio[midpoint:]) + await streamed_input.add_audio(None) + + result = await pipeline.run(streamed_input) + lifecycle: list[str] = [] + audio_chunks = 0 + + async def consume() -> None: + nonlocal audio_chunks + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + lifecycle.append(event.event) + elif isinstance(event, VoiceStreamEventAudio) and event.data is not None: + audio_chunks += 1 + + await asyncio.wait_for(consume(), timeout=65) + + assert calls == ["streamed"] + np.testing.assert_array_equal(audio, original_audio) + assert audio_chunks > 0 + assert lifecycle == ["turn_started", "turn_ended", "session_ended"] + assert stt_model.session is not None and stt_model.session.closed + + +async def test_voice_pipeline_surfaces_tts_failures_without_hanging( + integration_model: str, + integration_pcm_audio: bytes, +) -> None: + import numpy as np + + from agents.voice import ( + AudioInput, + SingleAgentVoiceWorkflow, + TTSModel, + TTSModelSettings, + VoicePipeline, + VoiceStreamEventLifecycle, + ) + from agents.voice.events import VoiceStreamEventError + + class FailingTTSModel(TTSModel): + @property + def model_name(self) -> str: + return "failing-packaged-tts" + + async def run(self, text: str, settings: TTSModelSettings) -> AsyncIterator[bytes]: + del text, settings + raise RuntimeError("Packaged TTS synthesis failed.") + yield b"" # pragma: no cover + + agent: Agent[Any] = Agent( + name="Packaged failing voice workflow agent", + model=integration_model, + instructions="Reply with exactly VOICE_FAILURE_READY.", + model_settings={"max_tokens": 128}, + ) + pipeline = VoicePipeline( + workflow=SingleAgentVoiceWorkflow(agent), + tts_model=FailingTTSModel(), + config={"tracing_disabled": True}, + ) + audio = np.frombuffer(integration_pcm_audio, dtype=np.int16).copy() + result = await pipeline.run(AudioInput(buffer=audio)) + observed: list[str] = [] + + async def consume() -> None: + async for event in result.stream(): + if isinstance(event, VoiceStreamEventLifecycle): + observed.append(event.event) + elif isinstance(event, VoiceStreamEventError): + observed.append("error") + + with pytest.raises(RuntimeError, match="Packaged TTS synthesis failed"): + await asyncio.wait_for(consume(), timeout=25) + + assert observed[0] == "turn_started" diff --git a/pyproject.toml b/pyproject.toml index cd02c21583..fe22fa070d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,15 @@ agents = { workspace = true } requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build] +exclude = [ + "/.agents", + "/.github", + "/.tmp*", + "/.uv*", + "/integration_tests", +] + [tool.hatch.build.targets.wheel] packages = ["src/agents"]