diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index bccd2b9df..698b9b82f 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.14.44" +version = "2.14.45" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/_helpers/agent_output.py b/packages/uipath/src/uipath/eval/_helpers/agent_output.py new file mode 100644 index 000000000..1c16825bf --- /dev/null +++ b/packages/uipath/src/uipath/eval/_helpers/agent_output.py @@ -0,0 +1,71 @@ +"""Render a workload's final output for the trajectory judge prompts.""" + +import json +from typing import Any + +from pydantic import TypeAdapter, ValidationError + +from uipath.core.chat import ( + UiPathConversationContentPartData, + UiPathConversationMessageData, + UiPathExternalValue, +) + +AGENT_RESPONSE_MESSAGES_KEY = "uipath__agent_response_messages" +FINAL_RESPONSE_HEADER = "Agent Final Response:" + +_messages_adapter = TypeAdapter(list[UiPathConversationMessageData]) + + +def render_agent_output(output: Any) -> str: + if isinstance(output, str): + return output + if isinstance(output, dict) and AGENT_RESPONSE_MESSAGES_KEY in output: + rendered = _render_conversational_output(output) + if rendered: + return rendered + return _to_json(output) + + +def append_agent_output(history: str, output: Any) -> str: + if output is None: + return history + section = f"{FINAL_RESPONSE_HEADER}\n{render_agent_output(output)}" + return f"{history}\n\n{section}" if history else section + + +def _render_conversational_output(output: dict[str, Any]) -> str: + try: + messages = _messages_adapter.validate_python( + output[AGENT_RESPONSE_MESSAGES_KEY] + ) + except ValidationError: + return "" + + # toolCalls are skipped: trace_to_str already renders the executed calls from spans. + lines = [ + f"[{message.role}] {rendered}" + for message in messages + for part in message.content_parts + if (rendered := _render_content_part(part)) + ] + + structured_fields = { + k: v for k, v in output.items() if k != AGENT_RESPONSE_MESSAGES_KEY + } + if structured_fields: + lines.append(_to_json(structured_fields)) + return "\n".join(lines) + + +def _render_content_part(part: UiPathConversationContentPartData) -> str: + if isinstance(part.data, UiPathExternalValue): + return f"Attachment ({part.mime_type}): {part.data.uri}" + inline = part.data.inline + if inline is None or inline == "": + return "" + return inline if isinstance(inline, str) else _to_json(inline) + + +def _to_json(value: Any) -> str: + return json.dumps(value, indent=2, default=str) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 1368d9339..e22ae6410 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -11,6 +11,7 @@ from uipath.platform.chat.llm_gateway import RequiredToolChoice from .._execution_context import eval_set_run_id_context +from .._helpers.agent_output import append_agent_output from .._helpers.evaluators_helpers import trace_to_str from .._helpers.helpers import is_empty_value from ..models import EvaluationResult @@ -101,6 +102,7 @@ async def evaluate( evaluation_prompt = self._create_evaluation_prompt( expected_agent_behavior=workload_execution.expected_agent_behavior, agent_run_history=workload_execution.workload_trace, + agent_output=self._get_actual_output(workload_execution), ) llm_response = await self._get_llm_response(evaluation_prompt) @@ -113,6 +115,7 @@ def _create_evaluation_prompt( self, expected_agent_behavior: Any, agent_run_history: Any, + agent_output: Any = None, ) -> str: """Create the evaluation prompt for the LLM.""" # Validate that expected agent behavior is not empty @@ -134,18 +137,17 @@ def _create_evaluation_prompt( ) # Trim extra properties from the spans (such as timestamps which are not relevant to the eval) - if ( - isinstance(agent_run_history, list) - and agent_run_history - and isinstance(agent_run_history[0], ReadableSpan) + if isinstance(agent_run_history, list) and all( + isinstance(span, ReadableSpan) for span in agent_run_history ): agent_run_history = trace_to_str(agent_run_history) else: agent_run_history = str(agent_run_history) + # trace_to_str only renders tool spans; the agent's own answer lives in the output. formatted_prompt = formatted_prompt.replace( self.agent_run_history_placeholder, - agent_run_history, + append_agent_output(agent_run_history, agent_output), ) return formatted_prompt diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py index d8b16ea90..1ae30b2fb 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, Field +from .._helpers.agent_output import append_agent_output from .._helpers.evaluators_helpers import trace_to_str from ..models import ( EvaluationResult, @@ -14,6 +15,11 @@ LLMJudgePromptTemplates, LLMJudgeTrajectoryOutputSchema, ) +from .attachment_utils import ( + download_attachment_as_string, + extract_attachment_id, + is_job_attachment_uri, +) from .base_evaluator import BaseEvaluationCriteria from .llm_as_judge_evaluator import ( BaseLLMJudgeEvaluatorConfig, @@ -80,7 +86,13 @@ async def evaluate( def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: """Get the actual output from the workload execution.""" - return trace_to_str(workload_execution.workload_trace) + output = workload_execution.workload_output + if isinstance(output, str) and is_job_attachment_uri(output): + output = download_attachment_as_string(extract_attachment_id(output)) + # trace_to_str only renders tool spans; the agent's own answer lives in the output. + return append_agent_output( + trace_to_str(workload_execution.workload_trace), output + ) def _get_expected_output( self, evaluation_criteria: TrajectoryEvaluationCriteria diff --git a/packages/uipath/tests/evaluators/test_trajectory_agent_output.py b/packages/uipath/tests/evaluators/test_trajectory_agent_output.py new file mode 100644 index 000000000..53d66f640 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_trajectory_agent_output.py @@ -0,0 +1,264 @@ +import uuid +from typing import Any + +import pytest +from opentelemetry.sdk.trace import ReadableSpan + +from uipath.eval._helpers.agent_output import ( + FINAL_RESPONSE_HEADER, + append_agent_output, + render_agent_output, +) +from uipath.eval.evaluators import LegacyTrajectoryEvaluator +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.evaluators.llm_judge_trajectory_evaluator import ( + LLMJudgeTrajectoryEvaluator, +) +from uipath.eval.models.models import ( + LegacyEvaluatorCategory, + LegacyEvaluatorType, + LLMResponse, + WorkloadExecution, +) + +ANSWER = "I can only share your own mentorship records, not other people's." + +# Shape a low-code conversational agent returns, copied from a cloud eval trace. +CONVERSATIONAL_OUTPUT = { + "uipath__agent_response_messages": [ + { + "role": "assistant", + "contentParts": [ + { + "mimeType": "text/markdown", + "data": {"inline": ANSWER}, + "citations": [], + } + ], + "toolCalls": [], + } + ] +} + + +def _tool_span() -> ReadableSpan: + return ReadableSpan( + name="Tool call - GetUserContext", + start_time=0, + end_time=1, + attributes={ + "tool.name": "GetUserContext", + "input.value": '{"userSlackId": "U1"}', + "output.value": '{"matches": []}', + }, + ) + + +def _legacy_evaluator(target_output_key: str = "*") -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + targetOutputKey=target_output_key, + ) + + +def _llm_judge_evaluator() -> LLMJudgeTrajectoryEvaluator: + return LLMJudgeTrajectoryEvaluator.model_validate( + { + "evaluatorConfig": { + "name": "LlmTrajectoryTest", + "prompt": "History: {{AgentRunHistory}} Expected: {{ExpectedAgentBehavior}}", + "model": "gpt-4", + }, + "id": str(uuid.uuid4()), + } + ) + + +class TestRenderAgentOutput: + def test_conversational_output_renders_message_text(self) -> None: + rendered = render_agent_output(CONVERSATIONAL_OUTPUT) + + assert rendered == f"[assistant] {ANSWER}" + + def test_snake_case_output_renders_text_without_tool_calls(self) -> None: + output = { + "uipath__agent_response_messages": [ + { + "role": "assistant", + "content_parts": [ + {"mime_type": "text/plain", "data": {"inline": ANSWER}} + ], + "tool_calls": [{"name": "Escalate", "input": {"reason": "x"}}], + } + ] + } + + assert render_agent_output(output) == f"[assistant] {ANSWER}" + + def test_external_and_falsy_parts_are_kept(self) -> None: + output = { + "uipath__agent_response_messages": [ + { + "role": "assistant", + "contentParts": [ + { + "mimeType": "text/plain", + "data": {"inline": "Report attached."}, + }, + { + "mimeType": "application/pdf", + "data": {"uri": "urn:uipath:cas:file:orchestrator:abc"}, + }, + {"mimeType": "application/json", "data": {"inline": False}}, + {"mimeType": "application/json", "data": {"inline": 0}}, + {"mimeType": "text/plain", "data": {"inline": ""}}, + ], + } + ] + } + + assert render_agent_output(output) == ( + "[assistant] Report attached.\n" + "[assistant] Attachment (application/pdf): " + "urn:uipath:cas:file:orchestrator:abc\n" + "[assistant] false\n" + "[assistant] 0" + ) + + def test_structured_fields_next_to_messages_are_kept(self) -> None: + output = {**CONVERSATIONAL_OUTPUT, "category": "privacy"} + + rendered = render_agent_output(output) + + assert ANSWER in rendered + assert '"category": "privacy"' in rendered + assert "uipath__agent_response_messages" not in rendered + + def test_malformed_messages_fall_back_to_json(self) -> None: + output = {"uipath__agent_response_messages": [{"unexpected": True}]} + + assert render_agent_output(output) == ( + '{\n "uipath__agent_response_messages": [\n' + ' {\n "unexpected": true\n }\n ]\n}' + ) + + @pytest.mark.parametrize( + ("output", "expected"), + [ + ("plain answer", "plain answer"), + ({"result": 42}, '{\n "result": 42\n}'), + ({}, "{}"), + ], + ) + def test_non_conversational_output(self, output: Any, expected: str) -> None: + assert render_agent_output(output) == expected + + +class TestAppendAgentOutput: + def test_empty_history_becomes_final_response_section(self) -> None: + assert append_agent_output("", "answer") == f"{FINAL_RESPONSE_HEADER}\nanswer" + + def test_tool_history_is_kept_ahead_of_final_response(self) -> None: + assert append_agent_output("tool history", "answer") == ( + f"tool history\n\n{FINAL_RESPONSE_HEADER}\nanswer" + ) + + def test_missing_output_leaves_history_untouched(self) -> None: + assert append_agent_output("tool history", None) == "tool history" + + +class TestLLMJudgeTrajectoryHistory: + def test_zero_tool_run_shows_agent_answer(self) -> None: + execution = WorkloadExecution( + agent_input={}, + workload_output=CONVERSATIONAL_OUTPUT, + workload_trace=[], + ) + + history = _llm_judge_evaluator()._get_actual_output(execution) + + assert history == f"{FINAL_RESPONSE_HEADER}\n[assistant] {ANSWER}" + + def test_tool_run_shows_tool_calls_and_agent_answer(self) -> None: + execution = WorkloadExecution( + agent_input={}, + workload_output=CONVERSATIONAL_OUTPUT, + workload_trace=[_tool_span()], + ) + + history = _llm_judge_evaluator()._get_actual_output(execution) + + assert "Tool: GetUserContext" in history + assert history.endswith(f"{FINAL_RESPONSE_HEADER}\n[assistant] {ANSWER}") + + def test_attachment_output_is_downloaded( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + attachment_id = "0b9e4f3a-1c2d-4e5f-8a9b-0c1d2e3f4a5b" + monkeypatch.setattr( + "uipath.eval.evaluators.llm_judge_trajectory_evaluator.download_attachment_as_string", + lambda key: ANSWER if key == attachment_id else "wrong id", + ) + execution = WorkloadExecution( + agent_input={}, + workload_output=f"urn:uipath:cas:file:orchestrator:{attachment_id}", + workload_trace=[], + ) + + history = _llm_judge_evaluator()._get_actual_output(execution) + + assert history == f"{FINAL_RESPONSE_HEADER}\n{ANSWER}" + + +class TestLegacyTrajectoryHistory: + def test_zero_tool_run_shows_agent_answer_instead_of_empty_list(self) -> None: + prompt = _legacy_evaluator()._create_evaluation_prompt( + expected_agent_behavior="Refuse to share other people's data.", + agent_run_history=[], + agent_output=CONVERSATIONAL_OUTPUT, + ) + + assert prompt.startswith( + f"History:\n{FINAL_RESPONSE_HEADER}\n[assistant] {ANSWER}\nExpected:" + ) + + async def test_evaluate_resolves_target_output_key(self) -> None: + evaluator = _legacy_evaluator(target_output_key="result") + evaluator._initialize_llm = lambda: None # type: ignore[method-assign] + prompts: list[str] = [] + + async def fake_llm_response(prompt: str) -> LLMResponse: + prompts.append(prompt) + return LLMResponse(score=100.0, justification="ok") + + evaluator._get_llm_response = fake_llm_response # type: ignore[method-assign, assignment] + execution = WorkloadExecution( + agent_input={}, + workload_output={"result": ANSWER, "debug": "must not reach the judge"}, + workload_trace=[_tool_span()], + expected_agent_behavior="Refuse to share other people's data.", + ) + + await evaluator.evaluate( + execution, + LegacyEvaluationCriteria( + expected_output=None, + expected_agent_behavior="Refuse to share other people's data.", + ), + ) + + assert "Tool: GetUserContext" in prompts[0] + assert f"{FINAL_RESPONSE_HEADER}\n{ANSWER}" in prompts[0] + assert "must not reach the judge" not in prompts[0] diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index e6ae1076e..c463c717e 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.44" +version = "2.14.45" source = { editable = "." } dependencies = [ { name = "applicationinsights" },