From a3da2c2c2673f7afd991c85db506f32c627d7058 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 4 Aug 2026 16:36:28 +0000 Subject: [PATCH] fix(test): repair eval adapter integ fixture and wire LLM judge key The evaluation group of Secure Integration test has been failing on main since #568 with all six third-party adapter tests red. Two distinct causes, both of which had to be fixed: 1. Span fixture shape. strands-evals' CloudWatchSessionMapper reads message text via content.content / content.message, where the inner value is a JSON-encoded list of content blocks, and keys off scope.name to select the CloudWatch format. The fixture passed a bare string and set no scope, so _extract_content_field returned None, the mapper produced zero traces, no AgentInvocationSpan was found, and the adapters returned FIELD_EXTRACTION_ERROR with value=None. The repo's own unit fixtures already use the correct nested shape; only the integ fixture drifted. test_factuality_scorer additionally overwrote output.messages inline with the old bare-string shape, which would have defeated the fix for that one test, so that mutation is gone. 2. LLM judge credentials. Both DeepEval and Autoevals judge with an LLM (GPT by default) and raise before returning when no key is present. Fixing the fixture alone does not turn the job green: it moves the Autoevals failures from FIELD_EXTRACTION_ERROR to METRIC_ERROR (missing credentials). OPENAI_API_KEY is now fetched as a repo-specific workflow secret and exported to the test step. The secret must exist in the central DevX Secrets Manager account and be readable by this repository's reader role before this merges. The fetch-secrets step is shared by all nine matrix groups, so an unfetchable secret id fails every group, not just evaluation. These tests require OPENAI_API_KEY and fail without it, so local runs and any environment lacking the key will report six failures. Verified: with the fixture fixed and the judge stubbed, both adapters extract input/actual_output correctly and return a score. --- .github/workflows/integration-testing.yml | 3 +- .../evaluation/test_third_party_adapters.py | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 3e65b7ed..44d29e76 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -159,7 +159,7 @@ jobs: with: role-arn: ${{ secrets.WORKFLOW_SECRETS_READER_ROLE_ARN }} shared: AGENTCORE_INTEG_TEST_ROLE - repo: MEMORY_KINESIS_ARN MEMORY_ROLE_ARN MEMORY_PREPOPULATED_ID RESOURCE_POLICY_TEST_ARN RESOURCE_POLICY_TEST_PRINCIPAL GATEWAY_ROLE_ARN GATEWAY_LAMBDA_ARN KB_ROLE_ARN EVAL_ROLE_ARN EVAL_LOG_GROUP RUNTIME_ROLE_ARN RUNTIME_S3_CODE_URI COGNITO_POOL_ID COGNITO_CLIENT_ID COGNITO_CLIENT_SECRET + repo: MEMORY_KINESIS_ARN MEMORY_ROLE_ARN MEMORY_PREPOPULATED_ID RESOURCE_POLICY_TEST_ARN RESOURCE_POLICY_TEST_PRINCIPAL GATEWAY_ROLE_ARN GATEWAY_LAMBDA_ARN KB_ROLE_ARN EVAL_ROLE_ARN EVAL_LOG_GROUP RUNTIME_ROLE_ARN RUNTIME_S3_CODE_URI COGNITO_POOL_ID COGNITO_CLIENT_ID COGNITO_CLIENT_SECRET OPENAI_API_KEY - name: Configure Credentials uses: aws-actions/configure-aws-credentials@v6 @@ -207,6 +207,7 @@ jobs: COGNITO_POOL_ID: ${{ env.COGNITO_POOL_ID }} COGNITO_CLIENT_ID: ${{ env.COGNITO_CLIENT_ID }} COGNITO_CLIENT_SECRET: ${{ env.COGNITO_CLIENT_SECRET }} + OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }} PYTEST_PATH: ${{ matrix.path }} PYTEST_IGNORE: ${{ matrix.ignore }} id: tests diff --git a/tests_integ/evaluation/test_third_party_adapters.py b/tests_integ/evaluation/test_third_party_adapters.py index 9ae5dd8b..4b766692 100644 --- a/tests_integ/evaluation/test_third_party_adapters.py +++ b/tests_integ/evaluation/test_third_party_adapters.py @@ -4,18 +4,35 @@ They verify the full adapter flow from EvaluatorInput through span parsing to metric execution, using real library metrics (not mocks). +Both libraries judge with an LLM, so every test here requires OPENAI_API_KEY. +In CI the key is fetched from the central DevX Secrets Manager account as a +repo-specific workflow secret. + SETUP: pip install deepeval autoevals + export OPENAI_API_KEY=... RUN: pytest tests_integ/evaluation/test_third_party_adapters.py -v """ +import json + import pytest from bedrock_agentcore.evaluation.custom_code_based_evaluators.models import EvaluatorInput, EvaluatorOutput +def _text_content(text): + """Wrap text in the nested, double-encoded shape the CloudWatch mapper expects. + + strands-evals reads message content via `content.content` / `content.message`, + where the inner value is a JSON-encoded list of content blocks. Passing a bare + string yields no AgentInvocationSpan and the adapters fail extraction. + """ + return {"content": json.dumps([{"text": text}])} + + def _make_agent_evaluator_input( user_prompt="What is the capital of France?", agent_response="The capital of France is Paris.", @@ -25,18 +42,20 @@ def _make_agent_evaluator_input( output_messages = [] if tool_messages: for msg in tool_messages: - output_messages.append({"role": "tool", "content": msg}) - output_messages.append({"role": "assistant", "content": agent_response}) + output_messages.append({"role": "tool", "content": _text_content(msg)}) + output_messages.append({"role": "assistant", "content": _text_content(agent_response)}) spans = [ { "traceId": "integ-trace-1", "spanId": "integ-span-1", + # The mapper keys off scope.name to pick the CloudWatch span format. + "scope": {"name": "strands.telemetry.tracer"}, "attributes": {"gen_ai.operation.name": "invoke_agent"}, "span_events": [ { "body": { - "input": {"messages": [{"role": "user", "content": user_prompt}]}, + "input": {"messages": [{"role": "user", "content": _text_content(user_prompt)}]}, "output": {"messages": output_messages}, } } @@ -127,10 +146,9 @@ def test_factuality_scorer(self): scorer = Factuality() adapter = AutoEvalsAdapter(metric=scorer) - evaluator_input = _make_agent_evaluator_input() - evaluator_input.session_spans[0]["span_events"][0]["body"]["output"]["messages"] = [ - {"role": "assistant", "content": "The capital of France is Paris."} - ] + # Assistant-only output (no tool messages), which is what the helper + # already builds; keep it explicit here to document the intent. + evaluator_input = _make_agent_evaluator_input(agent_response="The capital of France is Paris.") result = adapter(evaluator_input)