From 8831881beabbbab95f1ddffbecf1ed6d57120736 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Wed, 2 Sep 2026 15:57:38 -0400 Subject: [PATCH 1/4] [Evaluation] Fix OpenAI httpx2 test proxy compatibility Build proxy URLs with the request's concrete URL type, ignore transport-specific Accept-Encoding values in recordings, and remove the temporary OpenAI test pin now that both httpx generations are supported. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-ai-evaluation/dev_requirements.txt | 9 ---- .../tests/__openai_patcher.py | 10 ++-- .../azure-ai-evaluation/tests/conftest.py | 1 + .../tests/unittests/test_openai_patcher.py | 46 +++++++++++++++++++ 4 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py diff --git a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt index a88287df21e2..527d87a0d912 100644 --- a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt +++ b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt @@ -14,13 +14,4 @@ promptflow-core>=1.17.1 promptflow-devkit>=1.17.1 # Note: redteam extra (pyrit) is installed separately via InjectedPackages in platform-matrix.json # to avoid pillow version conflicts with promptflow-devkit (pillow<11 vs >=12.1) -# Test-only openai pin (see below). Not applied to install_requires so end-user -# installs can still use openai>=3.0 features (Responses API, workload identity, -# etc.) that are runtime-compatible with this SDK. The pin is here because -# tests/__openai_patcher.py constructs httpx.URL objects, but openai 3.x's -# client requires httpx2.URL — a mismatch that only triggers inside the test -# proxy path, never in real production code. Remove this pin once -# tests/__openai_patcher.py is updated for httpx2 compatibility. -openai<3.0 ../azure-ai-evaluation - diff --git a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py index e155b293e4ed..e6f6ed8cd6da 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py @@ -1,4 +1,4 @@ -"""Implementation of an httpx.Client that forwards traffic to the Azure SDK test-proxy. +"""Implementation of an OpenAI HTTP client that forwards traffic to the Azure SDK test-proxy. .. note:: @@ -69,14 +69,16 @@ def _reroute_to_proxy(self, request: httpx.Request) -> Iterator[None]: assert self.is_recording(), f"{self._reroute_to_proxy.__qualname__} should only be called while recording" config = self.recording_config original_url = request.url + url_type = type(original_url) request_path = original_url.copy_with(scheme="", netloc=b"") - request.url = httpx.URL(config.proxy_url).join(request_path) + request.url = url_type(config.proxy_url).join(request_path) original_headers = request.headers request.headers = request.headers.copy() request.headers.setdefault( - "x-recording-upstream-base-uri", str(httpx.URL(scheme=original_url.scheme, netloc=original_url.netloc)) + "x-recording-upstream-base-uri", + str(url_type(scheme=original_url.scheme, netloc=original_url.netloc)), ) request.headers["x-recording-id"] = config.recording_id request.headers["x-recording-mode"] = config.recording_mode @@ -113,6 +115,6 @@ async def send(self, request: httpx.Request, **kwargs) -> httpx.Response: return await super().send(request, **kwargs) -# openai._base_client.{Async,Sync}HttpxClientWrapper are default httpx.Clients instantiated by openai +# These wrappers are the default HTTP clients instantiated by OpenAI. openai._base_client.SyncHttpxClientWrapper = TestProxyHttpxClient openai._base_client.AsyncHttpxClientWrapper = TestProxyAsyncHttpxClient diff --git a/sdk/evaluation/azure-ai-evaluation/tests/conftest.py b/sdk/evaluation/azure-ai-evaluation/tests/conftest.py index 3445a655dbd9..afbacb394df5 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/conftest.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/conftest.py @@ -210,6 +210,7 @@ def evaluatation_run_sanitizer() -> None: # removes some headers since they are causing some unnecessary mismatches in recordings headers_to_ignore = [ + "accept-encoding", "ms-azure-ai-promptflow", "ms-azure-ai-promptflow-called-from", "x-ms-useragent", diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py new file mode 100644 index 000000000000..ec0c08a24cfd --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py @@ -0,0 +1,46 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +from typing import Any + +import httpx +import pytest + +from tests.__openai_patcher import ( + TestProxyConfig as _TestProxyConfig, + TestProxyHttpxClientBase as _TestProxyHttpxClientBase, +) + + +class _DummyClient(_TestProxyHttpxClientBase): + pass + + +def _assert_proxy_reroute(http_module: Any) -> None: + request = http_module.Request("POST", "https://example.test/openai/path?api-version=test") + original_url = request.url + config = _TestProxyConfig( + recording_id="recording", + recording_mode="playback", + proxy_url="http://localhost:5000", + ) + + with _TestProxyHttpxClientBase.record_with_proxy(config): + with _DummyClient()._reroute_to_proxy(request): + assert type(request.url) is type(original_url) + assert str(request.url) == "http://localhost:5000/openai/path?api-version=test" + assert request.headers["x-recording-upstream-base-uri"] == "https://example.test" + + assert request.url == original_url + + +@pytest.mark.unittest +def test_proxy_reroute_with_httpx() -> None: + _assert_proxy_reroute(httpx) + + +@pytest.mark.unittest +def test_proxy_reroute_with_httpx2() -> None: + httpx2 = pytest.importorskip("httpx2") + _assert_proxy_reroute(httpx2) From 164d36d8ae23459462f553486e6703a254995ab9 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Wed, 2 Sep 2026 16:46:03 -0400 Subject: [PATCH 2/4] [Evaluation] Make user-agent tests transport agnostic Patch the evaluation test-proxy wrapper directly so user-agent assertions observe requests from both httpx and httpx2 OpenAI clients. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tests/e2etests/test_builtin_evaluators.py | 5 ++--- .../azure-ai-evaluation/tests/e2etests/test_evaluate.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py index fc9683d2f5a6..0cadc87cf8c5 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py @@ -1782,9 +1782,9 @@ def test_prompty_evaluator( expected_user_agent = f"{base_user_agent} {added_useragent}" - from httpx import AsyncClient, Request + from tests.__openai_patcher import TestProxyAsyncHttpxClient - with self._transparent_mock_method(AsyncClient, "send") as mock: # OpenAI requests sent with httpx + with self._transparent_mock_method(TestProxyAsyncHttpxClient, "send") as mock: evaluator = evaluator_cls(user_agent_model_config) with UserAgentSingleton.add_useragent_product(added_useragent): @@ -1794,7 +1794,6 @@ def test_prompty_evaluator( for call_args in mock.call_args_list: _, request, *_ = call_args.args - request: Request # Not checking for strict equality because some evaluators add to the user agent assert expected_user_agent in request.headers["User-Agent"] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py index 0d19f30b410d..b05b444588f4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py @@ -548,9 +548,9 @@ def test_evaluate_user_agent(self, user_agent_model_config: AzureOpenAIModelConf expected_user_agent = f"{base_user_agent} {added_useragent}" - from httpx import AsyncClient, Request + from tests.__openai_patcher import TestProxyAsyncHttpxClient - with self._transparent_mock_method(AsyncClient, "send") as mock: + with self._transparent_mock_method(TestProxyAsyncHttpxClient, "send") as mock: evaluate( data=data_file, evaluators={"fluency": FluencyEvaluator(user_agent_model_config)}, @@ -561,7 +561,6 @@ def test_evaluate_user_agent(self, user_agent_model_config: AzureOpenAIModelConf for call_args in mock.call_args_list: _, request, *_ = call_args.args - request: Request # Not checking for strict equality because some evaluators add to the user agent assert expected_user_agent in request.headers["User-Agent"] From 0172d060290d72cc17bca8ffbf6cfbb6d3842e2e Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Fri, 4 Sep 2026 09:20:10 -0400 Subject: [PATCH 3/4] [Evaluation] Address OpenAI proxy review feedback Clarify that the import side effect still replaces OpenAI's wrapper aliases, patch those aliases in user-agent tests, and remove redundant tests of the proxy test utility. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 72a6b44e-e340-4c32-b8f7-4a52031ddcbf --- .../tests/__openai_patcher.py | 10 ++-- .../tests/e2etests/test_builtin_evaluators.py | 5 +- .../tests/e2etests/test_evaluate.py | 5 +- .../tests/unittests/test_openai_patcher.py | 46 ------------------- 4 files changed, 10 insertions(+), 56 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py diff --git a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py index e6f6ed8cd6da..2f412fd1d793 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py @@ -2,11 +2,9 @@ .. note:: - This module has side-effects! - - Importing this module will replace the default httpx.Client used - by the openai package with one that can redirect it's traffic - to the Azure SDK test-proxy on demand. + Importing this module replaces OpenAI's default sync and async HTTP + wrapper classes with proxy-aware subclasses. The subclasses preserve + whichever underlying transport OpenAI selected, including httpx2. """ @@ -115,6 +113,6 @@ async def send(self, request: httpx.Request, **kwargs) -> httpx.Response: return await super().send(request, **kwargs) -# These wrappers are the default HTTP clients instantiated by OpenAI. +# OpenAI instantiates these aliases when no custom HTTP client is provided. openai._base_client.SyncHttpxClientWrapper = TestProxyHttpxClient openai._base_client.AsyncHttpxClientWrapper = TestProxyAsyncHttpxClient diff --git a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py index 0cadc87cf8c5..2868929d5c19 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_builtin_evaluators.py @@ -1782,9 +1782,10 @@ def test_prompty_evaluator( expected_user_agent = f"{base_user_agent} {added_useragent}" - from tests.__openai_patcher import TestProxyAsyncHttpxClient + from openai import _base_client as openai_base_client - with self._transparent_mock_method(TestProxyAsyncHttpxClient, "send") as mock: + # __openai_patcher replaces this alias at import time; patching the alias is transport-agnostic. + with self._transparent_mock_method(openai_base_client.AsyncHttpxClientWrapper, "send") as mock: evaluator = evaluator_cls(user_agent_model_config) with UserAgentSingleton.add_useragent_product(added_useragent): diff --git a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py index b05b444588f4..e60013a0b040 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_evaluate.py @@ -548,9 +548,10 @@ def test_evaluate_user_agent(self, user_agent_model_config: AzureOpenAIModelConf expected_user_agent = f"{base_user_agent} {added_useragent}" - from tests.__openai_patcher import TestProxyAsyncHttpxClient + from openai import _base_client as openai_base_client - with self._transparent_mock_method(TestProxyAsyncHttpxClient, "send") as mock: + # __openai_patcher replaces this alias at import time; patching the alias is transport-agnostic. + with self._transparent_mock_method(openai_base_client.AsyncHttpxClientWrapper, "send") as mock: evaluate( data=data_file, evaluators={"fluency": FluencyEvaluator(user_agent_model_config)}, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py deleted file mode 100644 index ec0c08a24cfd..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_openai_patcher.py +++ /dev/null @@ -1,46 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -from typing import Any - -import httpx -import pytest - -from tests.__openai_patcher import ( - TestProxyConfig as _TestProxyConfig, - TestProxyHttpxClientBase as _TestProxyHttpxClientBase, -) - - -class _DummyClient(_TestProxyHttpxClientBase): - pass - - -def _assert_proxy_reroute(http_module: Any) -> None: - request = http_module.Request("POST", "https://example.test/openai/path?api-version=test") - original_url = request.url - config = _TestProxyConfig( - recording_id="recording", - recording_mode="playback", - proxy_url="http://localhost:5000", - ) - - with _TestProxyHttpxClientBase.record_with_proxy(config): - with _DummyClient()._reroute_to_proxy(request): - assert type(request.url) is type(original_url) - assert str(request.url) == "http://localhost:5000/openai/path?api-version=test" - assert request.headers["x-recording-upstream-base-uri"] == "https://example.test" - - assert request.url == original_url - - -@pytest.mark.unittest -def test_proxy_reroute_with_httpx() -> None: - _assert_proxy_reroute(httpx) - - -@pytest.mark.unittest -def test_proxy_reroute_with_httpx2() -> None: - httpx2 = pytest.importorskip("httpx2") - _assert_proxy_reroute(httpx2) From b8893c205560346a0a9750addd51fc59d16a7859 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Fri, 4 Sep 2026 10:11:57 -0400 Subject: [PATCH 4/4] [Evaluation] Route proxy requests after OpenAI auth Move test-proxy URL rewriting to the final transport boundary so OpenAI 3.5 redirect-safety hooks evaluate the original Azure origin and retain authentication headers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 72a6b44e-e340-4c32-b8f7-4a52031ddcbf --- .../tests/__openai_patcher.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py index 2f412fd1d793..e94bdaa05ace 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/__openai_patcher.py @@ -89,28 +89,22 @@ def _reroute_to_proxy(self, request: httpx.Request) -> Iterator[None]: class TestProxyHttpxClient(TestProxyHttpxClientBase, openai._base_client.SyncHttpxClientWrapper): @override - def send(self, request: httpx.Request, **kwargs) -> httpx.Response: + def _send_single_request(self, request: httpx.Request) -> httpx.Response: + # OpenAI auth and redirect-safety hooks must evaluate the original Azure URL before proxy routing. if self.is_recording(): with self._reroute_to_proxy(request): - response = super().send(request, **kwargs) - - response.request.url = request.url - return response - else: - return super().send(request, **kwargs) + return super()._send_single_request(request) + return super()._send_single_request(request) class TestProxyAsyncHttpxClient(TestProxyHttpxClientBase, openai._base_client.AsyncHttpxClientWrapper): @override - async def send(self, request: httpx.Request, **kwargs) -> httpx.Response: + async def _send_single_request(self, request: httpx.Request) -> httpx.Response: + # OpenAI auth and redirect-safety hooks must evaluate the original Azure URL before proxy routing. if self.is_recording(): with self._reroute_to_proxy(request): - response = await super().send(request, **kwargs) - - response.request.url = request.url - return response - else: - return await super().send(request, **kwargs) + return await super()._send_single_request(request) + return await super()._send_single_request(request) # OpenAI instantiates these aliases when no custom HTTP client is provided.