fix(eval): include agent final response in trajectory AgentRunHistory [FOR-1629]

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.14.44"
+version = "2.14.45"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/eval/_helpers/agent_output.py

@@ -0,0 +1,71 @@
+"""Render a workload's final output for the trajectory judge prompts."""
+
+import json
+from typing import Any
+
+from pydantic import TypeAdapter, ValidationError
+
+from uipath.core.chat import (
+    UiPathConversationContentPartData,
+    UiPathConversationMessageData,
+    UiPathExternalValue,
+)
+
+AGENT_RESPONSE_MESSAGES_KEY = "uipath__agent_response_messages"
+FINAL_RESPONSE_HEADER = "Agent Final Response:"
+
+_messages_adapter = TypeAdapter(list[UiPathConversationMessageData])
+
+
+def render_agent_output(output: Any) -> str:
+    if isinstance(output, str):
+        return output
+    if isinstance(output, dict) and AGENT_RESPONSE_MESSAGES_KEY in output:
+        rendered = _render_conversational_output(output)
+        if rendered:
+            return rendered
+    return _to_json(output)
+
+
+def append_agent_output(history: str, output: Any) -> str:
+    if output is None:
+        return history
+    section = f"{FINAL_RESPONSE_HEADER}\n{render_agent_output(output)}"
+    return f"{history}\n\n{section}" if history else section
+
+
+def _render_conversational_output(output: dict[str, Any]) -> str:
+    try:
+        messages = _messages_adapter.validate_python(
+            output[AGENT_RESPONSE_MESSAGES_KEY]
+        )
+    except ValidationError:
+        return ""
+
+    # toolCalls are skipped: trace_to_str already renders the executed calls from spans.
+    lines = [
+        f"[{message.role}] {rendered}"
+        for message in messages
+        for part in message.content_parts
+        if (rendered := _render_content_part(part))
+    ]
+
+    structured_fields = {
+        k: v for k, v in output.items() if k != AGENT_RESPONSE_MESSAGES_KEY
+    }
+    if structured_fields:
+        lines.append(_to_json(structured_fields))
+    return "\n".join(lines)
+
+
+def _render_content_part(part: UiPathConversationContentPartData) -> str:
+    if isinstance(part.data, UiPathExternalValue):
+        return f"Attachment ({part.mime_type}): {part.data.uri}"
+    inline = part.data.inline
+    if inline is None or inline == "":
+        return ""
+    return inline if isinstance(inline, str) else _to_json(inline)
+
+
+def _to_json(value: Any) -> str:
+    return json.dumps(value, indent=2, default=str)

packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py

@@ -11,6 +11,7 @@
 from uipath.platform.chat.llm_gateway import RequiredToolChoice
 
 from .._execution_context import eval_set_run_id_context
+from .._helpers.agent_output import append_agent_output
 from .._helpers.evaluators_helpers import trace_to_str
 from .._helpers.helpers import is_empty_value
 from ..models import EvaluationResult
@@ -101,6 +102,7 @@ async def evaluate(
         evaluation_prompt = self._create_evaluation_prompt(
             expected_agent_behavior=workload_execution.expected_agent_behavior,
             agent_run_history=workload_execution.workload_trace,
+            agent_output=self._get_actual_output(workload_execution),
         )
         llm_response = await self._get_llm_response(evaluation_prompt)
 
@@ -113,6 +115,7 @@ def _create_evaluation_prompt(
         self,
         expected_agent_behavior: Any,
         agent_run_history: Any,
+        agent_output: Any = None,
     ) -> str:
         """Create the evaluation prompt for the LLM."""
         # Validate that expected agent behavior is not empty
@@ -134,18 +137,17 @@ def _create_evaluation_prompt(
         )
 
         # Trim extra properties from the spans (such as timestamps which are not relevant to the eval)
-        if (
-            isinstance(agent_run_history, list)
-            and agent_run_history
-            and isinstance(agent_run_history[0], ReadableSpan)
+        if isinstance(agent_run_history, list) and all(
+            isinstance(span, ReadableSpan) for span in agent_run_history
         ):
             agent_run_history = trace_to_str(agent_run_history)
         else:
             agent_run_history = str(agent_run_history)
 
+        # trace_to_str only renders tool spans; the agent's own answer lives in the output.
         formatted_prompt = formatted_prompt.replace(
             self.agent_run_history_placeholder,
-            agent_run_history,
+            append_agent_output(agent_run_history, agent_output),
         )
 
         return formatted_prompt

packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py

@@ -4,6 +4,7 @@
 
 from pydantic import BaseModel, Field
 
+from .._helpers.agent_output import append_agent_output
 from .._helpers.evaluators_helpers import trace_to_str
 from ..models import (
     EvaluationResult,
@@ -14,6 +15,11 @@
     LLMJudgePromptTemplates,
     LLMJudgeTrajectoryOutputSchema,
 )
+from .attachment_utils import (
+    download_attachment_as_string,
+    extract_attachment_id,
+    is_job_attachment_uri,
+)
 from .base_evaluator import BaseEvaluationCriteria
 from .llm_as_judge_evaluator import (
     BaseLLMJudgeEvaluatorConfig,
@@ -80,7 +86,13 @@ async def evaluate(
 
     def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any:
         """Get the actual output from the workload execution."""
-        return trace_to_str(workload_execution.workload_trace)
+        output = workload_execution.workload_output
+        if isinstance(output, str) and is_job_attachment_uri(output):
+            output = download_attachment_as_string(extract_attachment_id(output))
+        # trace_to_str only renders tool spans; the agent's own answer lives in the output.
+        return append_agent_output(
+            trace_to_str(workload_execution.workload_trace), output
+        )
 
     def _get_expected_output(
         self, evaluation_criteria: TrajectoryEvaluationCriteria