Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/workflows/pr-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (hash-verified from uv.lock)
Expand Down Expand Up @@ -297,7 +297,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project without [uipath] extra
Expand Down Expand Up @@ -381,7 +381,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (hash-verified from uv.lock)
Expand Down Expand Up @@ -527,7 +527,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (hash-verified from uv.lock)
Expand Down Expand Up @@ -697,7 +697,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (hash-verified from uv.lock)
Expand Down Expand Up @@ -840,7 +840,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (with codex extra)
Expand Down Expand Up @@ -919,7 +919,7 @@ jobs:

- name: Install uv
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade "pip>=26.2" # PYSEC-2026-3721 fix floor
pip install uv

- name: Install project dependencies (hash-verified from uv.lock)
Expand Down
9 changes: 8 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,14 @@ dependencies = [
"click>=8.3.3",
"rich>=14.3.3",
"python-dotenv>=1.2.2",
"anthropic>=0.86.0",
# Cap the major: 1.0.0 already proved a major can drop call kwargs
# (temperature/top_p/top_k off messages.create — see judge_anthropic.py).
"anthropic>=1.0.0,<2.0.0",
# anthropic>=1.0.0 migrated its HTTP layer from httpx to httpx2 (its own
# exceptions, e.g. APIConnectionError, now carry an httpx2.Request). Declared
# explicitly since our code/tests construct httpx2 types directly. Capped to
# mirror anthropic's own bound on it (`httpx2<3,>=2.0.0`).
"httpx2>=2.12.0,<3.0.0",
"claude-agent-sdk>=0.2.124",
"anyio>=4.13.0",
"radon>=6.0.1",
Expand Down
18 changes: 15 additions & 3 deletions src/coder_eval/evaluation/judge_anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,14 @@ async def invoke_anthropic_judge_async(
Returns the SDK response converted to a dict via ``model_dump`` so the
caller can reuse ``extract_verdict_from_anthropic_response`` — Anthropic's
native shape (content blocks with ``type: tool_use``) is identical
between this SDK call and the Bedrock httpx-direct call.
between this SDK call and the Bedrock httpx2-direct call.

``temperature`` is forwarded via ``extra_body`` rather than as a top-level
kwarg: anthropic 1.0.0 dropped ``temperature``/``top_p``/``top_k`` from
``AsyncMessages.create``'s typed signature, but the Messages API itself
still accepts ``temperature`` in the raw JSON body — the same body shape
the Bedrock path already sends it in — so ``extra_body`` keeps both judge
backends honoring ``LLMJudgeCriterion.temperature`` identically.
"""
alias = to_anthropic_alias(model)
client = AsyncAnthropic(timeout=timeout_seconds)
Expand All @@ -51,12 +58,17 @@ async def invoke_anthropic_judge_async(
system=system,
messages=[{"role": "user", "content": user}],
max_tokens=max_tokens,
temperature=temperature,
tools=[tool_spec], # type: ignore[arg-type]
tools=[tool_spec], # pyright: ignore[reportArgumentType]
tool_choice={"type": "tool", "name": tool_spec["name"]},
extra_body={"temperature": temperature},
)
except APIError as e:
# The SDK already retries transient failures internally (2 attempts
# by default) — do not add another retry loop here.
raise JudgeInfrastructureError(f"Anthropic judge API error: {e}") from e
except Exception as e:
# A signature/contract break (e.g. a removed or renamed kwarg after an
# SDK bump) must not be scored as an agent failure — see CLAUDE.md's
# CE039 rationale: an eval-infra fault is not the agent's fault.
raise JudgeInfrastructureError(f"Anthropic judge call failed: {e}") from e
return response.model_dump()
8 changes: 4 additions & 4 deletions src/coder_eval/evaluation/judge_bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
intentionally do not share an HTTP client.

Async on purpose: this is llm_judge's only implementation of the network
call (there is no sync twin) — ``httpx.AsyncClient`` lets the call yield the
call (there is no sync twin) — ``httpx2.AsyncClient`` lets the call yield the
event loop instead of blocking a thread-pool thread for the wait, so
``SuccessChecker.check_all_async`` awaits it directly without pinning a
thread. (``check_all_async`` currently runs criteria sequentially; running
Expand All @@ -27,7 +27,7 @@
import logging
from typing import Any

import httpx
import httpx2

from coder_eval.errors import JudgeInfrastructureError
from coder_eval.errors.categories import RetryConfig
Expand Down Expand Up @@ -90,13 +90,13 @@ async def invoke_bedrock_judge_async(
attempts = _JUDGE_RETRY.max_retries + 1
last_failure = ""
last_exc: Exception | None = None
async with httpx.AsyncClient() as client:
async with httpx2.AsyncClient() as client:
for attempt in range(attempts):
if attempt:
await asyncio.sleep(compute_backoff(_JUDGE_RETRY, attempt - 1))
try:
response = await client.post(url, headers=headers, json=body, timeout=timeout_seconds)
except httpx.HTTPError as e:
except httpx2.HTTPError as e:
last_failure = f"Bedrock invoke transport error: {e}"
last_exc = e
logger.warning("Bedrock judge attempt %d/%d failed: %s", attempt + 1, attempts, last_failure)
Expand Down
4 changes: 2 additions & 2 deletions src/coder_eval/evaluation/verdict_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"description": _SUBMIT_VERDICT_DESCRIPTION,
"input_schema": JudgeVerdict.model_json_schema(),
}
"""Anthropic-native tool spec for the Bedrock httpx-direct path and Anthropic SDK calls.
"""Anthropic-native tool spec for the Bedrock httpx2-direct path and Anthropic SDK calls.

Note: Anthropic uses ``input_schema``, not OpenAI's ``parameters``.
"""
Expand Down Expand Up @@ -132,7 +132,7 @@ def extract_verdict_from_anthropic_response(
``{"type": "tool_use", "name": "submit_verdict"}`` and validates the last
one's ``input`` against ``JudgeVerdict``. Used by:

* The Bedrock httpx path (``invoke_bedrock_judge_async``) — raw JSON dict.
* The Bedrock httpx2 path (``invoke_bedrock_judge_async``) — raw JSON dict.
* The Anthropic SDK Direct path (``invoke_anthropic_judge_async``) —
response converted via ``Message.model_dump()``.

Expand Down
41 changes: 37 additions & 4 deletions tests/test_judge_anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from anthropic.resources.messages import AsyncMessages

from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge_async
from coder_eval.evaluation.verdict_tool import SUBMIT_VERDICT_ANTHROPIC_TOOL
Expand All @@ -29,12 +30,25 @@ def _make_response(*, score: float = 0.5, rationale: str = "ok") -> MagicMock:

def _make_client(response: MagicMock | None = None) -> MagicMock:
client = MagicMock()
client.messages.create = AsyncMock(return_value=response if response is not None else _make_response())
# spec-bound to the real ``AsyncMessages.create`` signature so a kwarg the
# installed SDK no longer accepts (e.g. a removed ``temperature``) fails
# here instead of silently passing against an unconstrained MagicMock.
client.messages = MagicMock(spec=AsyncMessages)
client.messages.create = AsyncMock(
wraps=lambda **kwargs: _bind_and_return(response if response is not None else _make_response(), **kwargs)
)
client.__aenter__ = AsyncMock(return_value=client)
client.__aexit__ = AsyncMock(return_value=None)
return client


def _bind_and_return(response: MagicMock, **kwargs: Any) -> MagicMock:
import inspect

inspect.signature(AsyncMessages.create).bind(MagicMock(), **kwargs)
return response


async def _invoke(**overrides):
defaults = {
"model": "anthropic.claude-sonnet-4-6",
Expand Down Expand Up @@ -79,24 +93,43 @@ async def test_invoke_anthropic_judge_raises_on_empty_model() -> None:


async def test_invoke_anthropic_judge_passes_temperature_and_max_tokens() -> None:
"""``temperature`` travels via ``extra_body`` — anthropic 1.0.0 dropped it as a
top-level ``messages.create`` kwarg, but the Messages API still accepts it in
the raw JSON body (the same body shape the Bedrock path sends it in)."""
client = _make_client()
with patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client):
await _invoke(temperature=0.7, max_tokens=321, system="sys", user="usr")
kwargs: dict[str, Any] = client.messages.create.call_args.kwargs
assert kwargs["temperature"] == 0.7
assert "temperature" not in kwargs
assert kwargs["extra_body"] == {"temperature": 0.7}
assert kwargs["max_tokens"] == 321
assert kwargs["system"] == "sys"
assert kwargs["messages"] == [{"role": "user", "content": "usr"}]


async def test_invoke_anthropic_judge_escalates_on_signature_break() -> None:
"""A kwarg the installed SDK no longer accepts must escalate as infra, not
silently score the row 0.0 (see judge_bedrock.py's parallel retry/escalation
contract and CLAUDE.md's CE039 rationale)."""
from coder_eval.errors import JudgeInfrastructureError

client = _make_client()
client.messages.create.side_effect = TypeError("create() got an unexpected keyword argument 'temperature'")
with (
patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client),
pytest.raises(JudgeInfrastructureError, match="Anthropic judge call failed"),
):
await _invoke()


async def test_invoke_anthropic_judge_wraps_api_error() -> None:
import httpx
import httpx2
from anthropic import APIConnectionError

from coder_eval.errors import JudgeInfrastructureError

client = _make_client()
sdk_error = APIConnectionError(request=httpx.Request("POST", "https://api.anthropic.com"))
sdk_error = APIConnectionError(request=httpx2.Request("POST", "https://api.anthropic.com"))
client.messages.create.side_effect = sdk_error
with (
patch("coder_eval.evaluation.judge_anthropic.AsyncAnthropic", return_value=client),
Expand Down
33 changes: 15 additions & 18 deletions tests/test_judge_bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import Any
from unittest.mock import AsyncMock, MagicMock

import httpx2
import pytest

from coder_eval.errors import JudgeInfrastructureError
Expand Down Expand Up @@ -51,7 +52,7 @@ def _tool_use_response(score: float = 0.5, rationale: str = "ok") -> dict[str, A


def _make_async_client(post_side_effect) -> MagicMock:
"""Mock ``httpx.AsyncClient`` — supports the ``async with`` + repeated ``.post(...)`` shape."""
"""Mock ``httpx2.AsyncClient`` — supports the ``async with`` + repeated ``.post(...)`` shape."""
client = MagicMock()
client.__aenter__ = AsyncMock(return_value=client)
client.__aexit__ = AsyncMock(return_value=None)
Expand Down Expand Up @@ -83,7 +84,7 @@ def fake_post(url: str, *, headers: dict[str, str], json: dict[str, Any], timeou
captured["timeout"] = timeout
return _make_response(status_code=200, json_data=_tool_use_response(score=0.5))

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(fake_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(fake_post))
result = await _invoke(max_tokens=42)

assert result["content"][0]["type"] == "tool_use"
Expand All @@ -99,7 +100,7 @@ def fake_post(url: str, *, headers: dict[str, str], json: dict[str, Any], timeou

async def test_invoke_bedrock_judge_raises_on_4xx(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
judge_bedrock.httpx,
judge_bedrock.httpx2,
"AsyncClient",
lambda: _make_async_client(lambda *a, **kw: _make_response(status_code=400, text='{"message":"bad model"}')),
)
Expand All @@ -115,7 +116,7 @@ def counting_post(*a: Any, **kw: Any) -> MagicMock:
calls.append(1)
return _make_response(status_code=500, text="upstream error")

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(counting_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(counting_post))
with pytest.raises(JudgeInfrastructureError, match="Bedrock invoke failed: 500"):
await _invoke()
# Exactly 1 initial call + max_retries retries — read from the constant, don't hardcode.
Expand All @@ -125,7 +126,7 @@ def counting_post(*a: Any, **kw: Any) -> MagicMock:

async def test_invoke_bedrock_judge_raises_on_non_dict_response(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
judge_bedrock.httpx,
judge_bedrock.httpx2,
"AsyncClient",
lambda: _make_async_client(lambda *a, **kw: _make_response(json_data=["not a dict"])),
)
Expand All @@ -149,16 +150,14 @@ async def test_invoke_bedrock_judge_raises_on_empty_model() -> None:
async def test_invoke_bedrock_judge_wraps_transport_error(
monkeypatch: pytest.MonkeyPatch, no_sleep: list[float]
) -> None:
import httpx as _httpx

def raising_post(*a: Any, **kw: Any) -> MagicMock:
raise _httpx.ConnectTimeout("connection timed out")
raise httpx2.ConnectTimeout("connection timed out")

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(raising_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(raising_post))
with pytest.raises(JudgeInfrastructureError, match="Bedrock invoke transport error") as excinfo:
await _invoke()
assert "connection timed out" in str(excinfo.value)
assert isinstance(excinfo.value.__cause__, _httpx.ConnectTimeout)
assert isinstance(excinfo.value.__cause__, httpx2.ConnectTimeout)


async def test_invoke_bedrock_judge_strips_v1_suffix_in_url(monkeypatch: pytest.MonkeyPatch) -> None:
Expand All @@ -168,7 +167,7 @@ def fake_post(url: str, **kw: Any) -> MagicMock:
captured["url"] = url
return _make_response(json_data=_tool_use_response())

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(fake_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(fake_post))
await _invoke(model="anthropic.claude-opus-4-6-v1")
assert "/model/eu.anthropic.claude-opus-4-6/invoke" in captured["url"]

Expand All @@ -189,7 +188,7 @@ def sequenced_post(*a: Any, **kw: Any) -> MagicMock:
calls.append(1)
return next(responses)

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(sequenced_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(sequenced_post))
result = await _invoke()
assert result["content"][0]["input"]["score"] == 0.9
assert len(calls) == 3
Expand All @@ -205,7 +204,7 @@ def counting_post(*a: Any, **kw: Any) -> MagicMock:
calls.append(1)
return _make_response(status_code=403, text="forbidden")

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(counting_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(counting_post))
with pytest.raises(JudgeInfrastructureError, match="Bedrock invoke failed: 403"):
await _invoke()
assert len(calls) == 1
Expand All @@ -215,17 +214,15 @@ def counting_post(*a: Any, **kw: Any) -> MagicMock:
async def test_invoke_bedrock_judge_retries_connect_error_then_succeeds(
monkeypatch: pytest.MonkeyPatch, no_sleep: list[float]
) -> None:
import httpx as _httpx

calls: list[int] = []

def flaky_post(*a: Any, **kw: Any) -> MagicMock:
calls.append(1)
if len(calls) == 1:
raise _httpx.ConnectError("connection refused")
raise httpx2.ConnectError("connection refused")
return _make_response(status_code=200, json_data=_tool_use_response())

monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(flaky_post))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(flaky_post))
result = await _invoke()
assert result["content"][0]["type"] == "tool_use"
assert len(calls) == 2
Expand All @@ -237,6 +234,6 @@ async def test_invoke_bedrock_judge_malformed_json_body_escalates(monkeypatch: p

response = _make_response(status_code=200)
response.json.side_effect = _json.JSONDecodeError("Expecting value", doc="", pos=0)
monkeypatch.setattr(judge_bedrock.httpx, "AsyncClient", lambda: _make_async_client(lambda *a, **kw: response))
monkeypatch.setattr(judge_bedrock.httpx2, "AsyncClient", lambda: _make_async_client(lambda *a, **kw: response))
with pytest.raises(JudgeInfrastructureError, match="not valid JSON"):
await _invoke()
4 changes: 2 additions & 2 deletions tests/test_judge_burn_in_live.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Live burn-in tests for the typed verdict tool channel.

Exercises the ``submit_verdict`` channel against the three real backends:
Anthropic-direct (Anthropic SDK + native ``tools``), Bedrock (httpx +
Anthropic-direct (Anthropic SDK + native ``tools``), Bedrock (httpx2 +
Anthropic-native tools), and the Claude Code SDK (in-process MCP server).
Each test ``pytest.skip``s when the required credentials are not present, so
the file is safe to run in CI without a credential set.
Expand Down Expand Up @@ -93,7 +93,7 @@ def test_llm_judge_anthropic_direct_tool_channel(hello_sandbox: Sandbox) -> None


def test_llm_judge_bedrock_tool_channel(hello_sandbox: Sandbox) -> None:
"""Bedrock route: httpx POST with Anthropic-native ``tools`` + ``tool_choice``."""
"""Bedrock route: httpx2 POST with Anthropic-native ``tools`` + ``tool_choice``."""
bearer = os.environ.get("AWS_BEARER_TOKEN_BEDROCK")
region = os.environ.get("AWS_REGION")
if not bearer or not region:
Expand Down
Loading
Loading