From 4dd9052d2e5f1ebd1f8dfd2a665a4f25cefee9c9 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:08:25 +0000 Subject: [PATCH 1/7] Fix array input in Chat Completions format losing user message text (#5112) When input was sent as an array without an explicit "type": "message" field (Chat Completions format), _is_openai_multimodal_format returned False and _convert_openai_input_to_chat_message silently skipped the item. This caused the user's query to be replaced with an empty string, making the agent return unfiltered tool results. Broaden both _is_openai_multimodal_format and the item-level check in _convert_openai_input_to_chat_message to also accept items that have a "role" field but no explicit "type" field. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- REPRODUCTION_REPORT.md | 44 ++++++++++++++ .../devui/agent_framework_devui/_executor.py | 12 +++- .../tests/devui/test_multimodal_workflow.py | 57 +++++++++++++++++++ 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 REPRODUCTION_REPORT.md diff --git a/REPRODUCTION_REPORT.md b/REPRODUCTION_REPORT.md new file mode 100644 index 0000000000..2429605ab0 --- /dev/null +++ b/REPRODUCTION_REPORT.md @@ -0,0 +1,44 @@ +# Reproduction Report: Issue #5112 + +**Issue**: [Python: [Bug]: Agent behaves differently with input when sent as string vs array - Difference is consistent](https://github.com/microsoft/agent-framework/issues/5112) +**Repository**: microsoft/agent-framework +**Investigated**: 2026-04-06 23:05:09 UTC +**Worktree Path**: `/repos/agent-framework/.worktrees/agent/fix-5112-1` +**Gate Result**: ✅ PASSED + +--- + +## Reproduction Verdict + +| Field | Value | +|-------|-------| +| Status | `reproduced` | +| Failure observed | `yes` | +| Evidence strength | `high` | +| Confidence | `high` | +| Failing test | `python/packages/devui/tests/devui/test_issue_5112_chat_completions_format.py` | +| Gate passed | ✅ PASSED | + +## Finding + +Bug reproduced with 2 failing tests. When input is sent as array in Chat Completions format (without "type": "message"), _convert_openai_input_to_chat_message silently skips the item at line 634, falling back to an empty text message at line 779. This causes the agent to receive no user query, explaining why it returns unfiltered tool results. + +## Files Examined + +python/packages/devui/agent_framework_devui/_executor.py, python/packages/devui/tests/devui/test_multimodal_workflow.py + +## Tests Run + +python/packages/devui/tests/devui/test_issue_5112_chat_completions_format.py + +## Full Analysis + +Now let me look at `_is_openai_multimodal_format` to understand the full path, and also check `_convert_input_to_chat_message`:Now I have clear evidence of the bug. Let me write a reproduction test.Bug is fully reproduced. Now let me write the report.Bug **confirmed and reproduced** with 2 failing tests. The root cause is in `_executor.py` line 634: `if item_type == "message"` only handles Responses API format, silently dropping Chat Completions format items (which lack `"type": "message"`), resulting in an empty user message. + +--- + +## Next Steps + +- Reproduction confirmed — proceed to investigation and fix phase +- Review the failing test or evidence above to understand the root cause +- The issue branch is checked out in the worktree: `/repos/agent-framework/.worktrees/agent/fix-5112-1` \ No newline at end of file diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index 530695ce20..4192f90763 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -631,7 +631,7 @@ def _convert_openai_input_to_chat_message(self, input_items: list[Any], Message: if isinstance(item, dict): item_dict = cast(dict[str, Any], item) item_type = item_dict.get("type") - if item_type == "message": + if item_type == "message" or (item_type is None and "role" in item_dict): # Extract content from OpenAI message message_content = item_dict.get("content", []) @@ -828,8 +828,14 @@ def _is_openai_multimodal_format(self, input_data: Any) -> bool: first_item = input_data_items[0] if not isinstance(first_item, dict): return False - first_type = cast(dict[str, Any], first_item).get("type") - return isinstance(first_type, str) and first_type == "message" + first_dict = cast(dict[str, Any], first_item) + first_type = first_dict.get("type") + if isinstance(first_type, str) and first_type == "message": + return True + # Also accept Chat Completions format: {"role": "...", "content": "..."} + if first_type is None and "role" in first_dict: + return True + return False async def _parse_workflow_input(self, workflow: Any, raw_input: Any) -> Any: """Parse input based on workflow's expected input type. diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index 7af7f3f308..a9f8c90353 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -150,3 +150,60 @@ async def test_parse_workflow_input_still_handles_simple_dict(self): # Result should be Message (from _parse_structured_workflow_input) assert isinstance(result, Message), f"Expected Message, got {type(result)}" + + def test_is_openai_multimodal_format_detects_chat_completions_format(self): + """Test that _is_openai_multimodal_format detects Chat Completions format (no type field).""" + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Chat Completions format: role + content, no type field + chat_completions_format = [ + {"role": "user", "content": "Describe this image"} + ] + assert executor._is_openai_multimodal_format(chat_completions_format) is True + + def test_convert_chat_completions_format_with_string_content(self): + """Test that Chat Completions format with string content is converted correctly.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Chat Completions format (no type field, string content) + input_data = [ + {"role": "user", "content": "Which Google phones are allowed?"} + ] + + result = executor._convert_input_to_chat_message(input_data) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 1 + assert result.contents[0].text == "Which Google phones are allowed?" + + def test_convert_chat_completions_format_with_list_content(self): + """Test that Chat Completions format with list content is converted correctly.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Chat Completions format with list content (input_text items) + input_data = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this image"}, + {"type": "input_image", "image_url": TEST_IMAGE_DATA_URI}, + ], + } + ] + + result = executor._convert_input_to_chat_message(input_data) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 2 + assert result.contents[0].text == "Describe this image" + assert result.contents[1].type == "data" From 16005e76ec911c0cb7ae3a03afcbad907e0df097 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:09:51 +0000 Subject: [PATCH 2/7] Fix #5112: Handle Chat Completions format input (no type field) Fix ruff SIM103 lint error by replacing if/return True/return False pattern with direct return of bool expression in _is_openai_multimodal_format. Also accept formatter-applied style changes in tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- python/packages/devui/agent_framework_devui/_executor.py | 4 +--- .../devui/tests/devui/test_multimodal_workflow.py | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index 4192f90763..8739340ce4 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -833,9 +833,7 @@ def _is_openai_multimodal_format(self, input_data: Any) -> bool: if isinstance(first_type, str) and first_type == "message": return True # Also accept Chat Completions format: {"role": "...", "content": "..."} - if first_type is None and "role" in first_dict: - return True - return False + return bool(first_type is None and "role" in first_dict) async def _parse_workflow_input(self, workflow: Any, raw_input: Any) -> Any: """Parse input based on workflow's expected input type. diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index a9f8c90353..64780bccf9 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -158,9 +158,7 @@ def test_is_openai_multimodal_format_detects_chat_completions_format(self): executor = AgentFrameworkExecutor(discovery, mapper) # Chat Completions format: role + content, no type field - chat_completions_format = [ - {"role": "user", "content": "Describe this image"} - ] + chat_completions_format = [{"role": "user", "content": "Describe this image"}] assert executor._is_openai_multimodal_format(chat_completions_format) is True def test_convert_chat_completions_format_with_string_content(self): @@ -172,9 +170,7 @@ def test_convert_chat_completions_format_with_string_content(self): executor = AgentFrameworkExecutor(discovery, mapper) # Chat Completions format (no type field, string content) - input_data = [ - {"role": "user", "content": "Which Google phones are allowed?"} - ] + input_data = [{"role": "user", "content": "Which Google phones are allowed?"}] result = executor._convert_input_to_chat_message(input_data) From c68af40bfe5c95b384e5abbaf0e020ebf8decb35 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:11:59 +0000 Subject: [PATCH 3/7] Remove reproduction report artifact Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- REPRODUCTION_REPORT.md | 44 ------------------------------------------ 1 file changed, 44 deletions(-) delete mode 100644 REPRODUCTION_REPORT.md diff --git a/REPRODUCTION_REPORT.md b/REPRODUCTION_REPORT.md deleted file mode 100644 index 2429605ab0..0000000000 --- a/REPRODUCTION_REPORT.md +++ /dev/null @@ -1,44 +0,0 @@ -# Reproduction Report: Issue #5112 - -**Issue**: [Python: [Bug]: Agent behaves differently with input when sent as string vs array - Difference is consistent](https://github.com/microsoft/agent-framework/issues/5112) -**Repository**: microsoft/agent-framework -**Investigated**: 2026-04-06 23:05:09 UTC -**Worktree Path**: `/repos/agent-framework/.worktrees/agent/fix-5112-1` -**Gate Result**: ✅ PASSED - ---- - -## Reproduction Verdict - -| Field | Value | -|-------|-------| -| Status | `reproduced` | -| Failure observed | `yes` | -| Evidence strength | `high` | -| Confidence | `high` | -| Failing test | `python/packages/devui/tests/devui/test_issue_5112_chat_completions_format.py` | -| Gate passed | ✅ PASSED | - -## Finding - -Bug reproduced with 2 failing tests. When input is sent as array in Chat Completions format (without "type": "message"), _convert_openai_input_to_chat_message silently skips the item at line 634, falling back to an empty text message at line 779. This causes the agent to receive no user query, explaining why it returns unfiltered tool results. - -## Files Examined - -python/packages/devui/agent_framework_devui/_executor.py, python/packages/devui/tests/devui/test_multimodal_workflow.py - -## Tests Run - -python/packages/devui/tests/devui/test_issue_5112_chat_completions_format.py - -## Full Analysis - -Now let me look at `_is_openai_multimodal_format` to understand the full path, and also check `_convert_input_to_chat_message`:Now I have clear evidence of the bug. Let me write a reproduction test.Bug is fully reproduced. Now let me write the report.Bug **confirmed and reproduced** with 2 failing tests. The root cause is in `_executor.py` line 634: `if item_type == "message"` only handles Responses API format, silently dropping Chat Completions format items (which lack `"type": "message"`), resulting in an empty user message. - ---- - -## Next Steps - -- Reproduction confirmed — proceed to investigation and fix phase -- Review the failing test or evidence above to understand the root cause -- The issue branch is checked out in the worktree: `/repos/agent-framework/.worktrees/agent/fix-5112-1` \ No newline at end of file From 6842d1671e1118937d56b35fbc0801874e8266b9 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:20:34 +0000 Subject: [PATCH 4/7] fix(devui): address review feedback for #5112 Chat Completions parsing - Tighten _is_openai_multimodal_format to validate role is a known string value and content is str|list, reducing false positives - Skip non-user messages (system/assistant/tool/developer) during conversion to prevent them being treated as user text - Rename test to clarify it covers Chat Completions envelope with Responses API content parts - Add end-to-end regression test through _parse_workflow_input with JSON-stringified Chat Completions array - Add tests for non-user message skipping and malformed input rejection Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../devui/agent_framework_devui/_executor.py | 18 +++++- .../tests/devui/test_multimodal_workflow.py | 61 ++++++++++++++++++- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index 8739340ce4..88722c9053 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -632,6 +632,10 @@ def _convert_openai_input_to_chat_message(self, input_items: list[Any], Message: item_dict = cast(dict[str, Any], item) item_type = item_dict.get("type") if item_type == "message" or (item_type is None and "role" in item_dict): + message_role = item_dict.get("role") + if message_role is not None and message_role != "user": + logger.debug("Skipping non-user OpenAI message item with role %r", message_role) + continue # Extract content from OpenAI message message_content = item_dict.get("content", []) @@ -833,7 +837,19 @@ def _is_openai_multimodal_format(self, input_data: Any) -> bool: if isinstance(first_type, str) and first_type == "message": return True # Also accept Chat Completions format: {"role": "...", "content": "..."} - return bool(first_type is None and "role" in first_dict) + # but require the minimum expected shape to avoid misclassifying + # unrelated or malformed list inputs as chat messages. + if first_type is not None: + return False + role = first_dict.get("role") + content = first_dict.get("content") + valid_roles = {"system", "user", "assistant", "tool", "developer"} + return bool( + isinstance(role, str) + and role in valid_roles + and "content" in first_dict + and isinstance(content, str | list) + ) async def _parse_workflow_input(self, workflow: Any, raw_input: Any) -> Any: """Parse input based on workflow's expected input type. diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index 64780bccf9..6c0dd6e7de 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -178,8 +178,8 @@ def test_convert_chat_completions_format_with_string_content(self): assert len(result.contents) == 1 assert result.contents[0].text == "Which Google phones are allowed?" - def test_convert_chat_completions_format_with_list_content(self): - """Test that Chat Completions format with list content is converted correctly.""" + def test_convert_chat_completions_envelope_with_responses_api_content(self): + """Test Chat Completions-style envelope (no type field) with Responses API content parts.""" from agent_framework import Message discovery = MagicMock(spec=EntityDiscovery) @@ -203,3 +203,60 @@ def test_convert_chat_completions_format_with_list_content(self): assert len(result.contents) == 2 assert result.contents[0].text == "Describe this image" assert result.contents[1].type == "data" + + async def test_parse_workflow_input_chat_completions_json_string(self): + """Regression test: JSON-stringified Chat Completions array goes through _parse_workflow_input.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # JSON-stringified Chat Completions format (the path DevUI/frontend commonly uses) + chat_input = json.dumps([{"role": "user", "content": "Which Google phones are allowed?"}]) + + mock_workflow = MagicMock() + mock_executor = MagicMock() + mock_executor.input_types = [Message] + mock_workflow.get_start_executor.return_value = mock_executor + + result = await executor._parse_workflow_input(mock_workflow, chat_input) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 1 + assert result.contents[0].text == "Which Google phones are allowed?" + + def test_convert_skips_non_user_messages(self): + """Test that non-user messages (system, assistant) are skipped during conversion.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Mix of system and user messages - only user content should be kept + input_data = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + ] + + result = executor._convert_input_to_chat_message(input_data) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 1 + assert result.contents[0].text == "Hello!" + + def test_is_openai_multimodal_format_rejects_malformed_input(self): + """Test that _is_openai_multimodal_format rejects inputs missing content or with invalid roles.""" + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Missing content key + assert executor._is_openai_multimodal_format([{"role": "user"}]) is False + # Invalid role value + assert executor._is_openai_multimodal_format([{"role": "unknown", "content": "hi"}]) is False + # Role is not a string + assert executor._is_openai_multimodal_format([{"role": 123, "content": "hi"}]) is False + # Content is neither str nor list + assert executor._is_openai_multimodal_format([{"role": "user", "content": 42}]) is False From 786d9591e14ffc67fb9dc4673d542b2c1d12880f Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:26:38 +0000 Subject: [PATCH 5/7] Address review feedback for #5112: add warning log and edge-case tests - Add logger.warning when all input items are non-user and the empty content fallback is triggered, making silent data loss visible in production logs. - Add test for all-non-user messages (Chat Completions format) verifying the empty-text fallback behavior. - Add test for non-user messages in Responses API format (type: message envelope) verifying they are skipped correctly. - Add test verifying all valid roles (user, system, assistant, tool, developer) are accepted by _is_openai_multimodal_format. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../devui/agent_framework_devui/_executor.py | 1 + .../tests/devui/test_multimodal_workflow.py | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index 88722c9053..cc13a038d5 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -781,6 +781,7 @@ def _convert_openai_input_to_chat_message(self, input_items: list[Any], Message: # If no contents found, create a simple text message if not contents: + logger.warning("All input items were non-user; no user content extracted") contents.append(Content.from_text(text="")) chat_message = Message(role="user", contents=contents) diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index 6c0dd6e7de..51920a835a 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -246,6 +246,56 @@ def test_convert_skips_non_user_messages(self): assert len(result.contents) == 1 assert result.contents[0].text == "Hello!" + def test_convert_skips_all_non_user_messages_chat_completions(self): + """When ALL messages are non-user (Chat Completions format), the result is a Message with empty text.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Only non-user messages, no user content at all + input_data = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "assistant", "content": "How can I help?"}, + ] + + result = executor._convert_input_to_chat_message(input_data) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 1 + assert result.contents[0].text == "" + + def test_convert_skips_non_user_messages_responses_api_format(self): + """Non-user messages in Responses API format (with type: message) are also skipped.""" + from agent_framework import Message + + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + input_data = [ + {"type": "message", "role": "system", "content": "You are a helpful assistant."}, + {"type": "message", "role": "user", "content": "Hello!"}, + ] + + result = executor._convert_input_to_chat_message(input_data) + + assert isinstance(result, Message), f"Expected Message, got {type(result)}" + assert len(result.contents) == 1 + assert result.contents[0].text == "Hello!" + + def test_is_openai_multimodal_format_accepts_all_valid_roles(self): + """All valid roles (user, system, assistant, tool, developer) are accepted by format detection.""" + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + for role in ("user", "system", "assistant", "tool", "developer"): + assert executor._is_openai_multimodal_format([{"role": role, "content": "hi"}]) is True, ( + f"Expected role {role!r} to be accepted" + ) + def test_is_openai_multimodal_format_rejects_malformed_input(self): """Test that _is_openai_multimodal_format rejects inputs missing content or with invalid roles.""" discovery = MagicMock(spec=EntityDiscovery) From 195d4925d82afc8b34bb15be7972b727e28e5248 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:29:40 +0000 Subject: [PATCH 6/7] Restrict format detection to require user-role item (#5112) Address review feedback: _is_openai_multimodal_format now requires at least one item with role='user' before classifying input as multimodal format. This prevents non-user-only arrays (e.g., system-only) from being routed into the conversion path where all items would be skipped, silently producing an empty message. Changes: - Restructure _is_openai_multimodal_format to check both Responses API and Chat Completions paths, then verify user-role presence - Update test_is_openai_multimodal_format_accepts_all_valid_roles to test non-user roles alongside a user companion item - Add test_is_openai_multimodal_format_rejects_non_user_only for both Chat Completions and Responses API format arrays Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../devui/agent_framework_devui/_executor.py | 37 ++++++++++++------- .../tests/devui/test_multimodal_workflow.py | 33 ++++++++++++++--- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index cc13a038d5..8c226a68d4 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -835,21 +835,32 @@ def _is_openai_multimodal_format(self, input_data: Any) -> bool: return False first_dict = cast(dict[str, Any], first_item) first_type = first_dict.get("type") + is_chat_format = False if isinstance(first_type, str) and first_type == "message": - return True - # Also accept Chat Completions format: {"role": "...", "content": "..."} - # but require the minimum expected shape to avoid misclassifying - # unrelated or malformed list inputs as chat messages. - if first_type is not None: + is_chat_format = True + elif first_type is None: + # Also accept Chat Completions format: {"role": "...", "content": "..."} + # but require the minimum expected shape to avoid misclassifying + # unrelated or malformed list inputs as chat messages. + role = first_dict.get("role") + content = first_dict.get("content") + valid_roles = {"system", "user", "assistant", "tool", "developer"} + is_chat_format = bool( + isinstance(role, str) + and role in valid_roles + and "content" in first_dict + and isinstance(content, str | list) + ) + + if not is_chat_format: return False - role = first_dict.get("role") - content = first_dict.get("content") - valid_roles = {"system", "user", "assistant", "tool", "developer"} - return bool( - isinstance(role, str) - and role in valid_roles - and "content" in first_dict - and isinstance(content, str | list) + + # Require at least one user-role item to avoid routing non-user-only + # arrays into the conversion path where all items would be skipped, + # silently producing an empty message. + return any( + isinstance(item, dict) and item.get("role") == "user" + for item in input_data_items ) async def _parse_workflow_input(self, workflow: Any, raw_input: Any) -> Any: diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index 51920a835a..427f4faf76 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -286,15 +286,38 @@ def test_convert_skips_non_user_messages_responses_api_format(self): assert result.contents[0].text == "Hello!" def test_is_openai_multimodal_format_accepts_all_valid_roles(self): - """All valid roles (user, system, assistant, tool, developer) are accepted by format detection.""" + """All valid roles are accepted when accompanied by a user-role message.""" discovery = MagicMock(spec=EntityDiscovery) mapper = MagicMock(spec=MessageMapper) executor = AgentFrameworkExecutor(discovery, mapper) - for role in ("user", "system", "assistant", "tool", "developer"): - assert executor._is_openai_multimodal_format([{"role": role, "content": "hi"}]) is True, ( - f"Expected role {role!r} to be accepted" - ) + # Single user message is accepted + assert executor._is_openai_multimodal_format([{"role": "user", "content": "hi"}]) is True + + # Non-user roles are accepted when a user-role item is also present + for role in ("system", "assistant", "tool", "developer"): + assert executor._is_openai_multimodal_format([ + {"role": role, "content": "hi"}, + {"role": "user", "content": "hello"}, + ]) is True, f"Expected role {role!r} to be accepted alongside user" + + def test_is_openai_multimodal_format_rejects_non_user_only(self): + """Arrays with no user-role message are rejected to prevent silent empty message fallback.""" + discovery = MagicMock(spec=EntityDiscovery) + mapper = MagicMock(spec=MessageMapper) + executor = AgentFrameworkExecutor(discovery, mapper) + + # Non-user-only Chat Completions format + assert executor._is_openai_multimodal_format([{"role": "system", "content": "hi"}]) is False + assert executor._is_openai_multimodal_format([ + {"role": "system", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ]) is False + + # Non-user-only Responses API format + assert executor._is_openai_multimodal_format([ + {"type": "message", "role": "system", "content": "hi"}, + ]) is False def test_is_openai_multimodal_format_rejects_malformed_input(self): """Test that _is_openai_multimodal_format rejects inputs missing content or with invalid roles.""" From 5a726daca4b7eacd6b6665bf45ed43b2f479c904 Mon Sep 17 00:00:00 2001 From: Copilot Date: Mon, 6 Apr 2026 23:31:53 +0000 Subject: [PATCH 7/7] Address review feedback for #5112: Python: [Bug]: Agent behaves differently with input when sent as string vs array - Difference is consistent --- .../devui/agent_framework_devui/_executor.py | 3 +- .../tests/devui/test_multimodal_workflow.py | 31 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/python/packages/devui/agent_framework_devui/_executor.py b/python/packages/devui/agent_framework_devui/_executor.py index 8c226a68d4..5c9d214e37 100644 --- a/python/packages/devui/agent_framework_devui/_executor.py +++ b/python/packages/devui/agent_framework_devui/_executor.py @@ -859,8 +859,7 @@ def _is_openai_multimodal_format(self, input_data: Any) -> bool: # arrays into the conversion path where all items would be skipped, # silently producing an empty message. return any( - isinstance(item, dict) and item.get("role") == "user" - for item in input_data_items + isinstance(item, dict) and cast(dict[str, Any], item).get("role") == "user" for item in input_data_items ) async def _parse_workflow_input(self, workflow: Any, raw_input: Any) -> Any: diff --git a/python/packages/devui/tests/devui/test_multimodal_workflow.py b/python/packages/devui/tests/devui/test_multimodal_workflow.py index 427f4faf76..709fd5d5cd 100644 --- a/python/packages/devui/tests/devui/test_multimodal_workflow.py +++ b/python/packages/devui/tests/devui/test_multimodal_workflow.py @@ -296,10 +296,13 @@ def test_is_openai_multimodal_format_accepts_all_valid_roles(self): # Non-user roles are accepted when a user-role item is also present for role in ("system", "assistant", "tool", "developer"): - assert executor._is_openai_multimodal_format([ - {"role": role, "content": "hi"}, - {"role": "user", "content": "hello"}, - ]) is True, f"Expected role {role!r} to be accepted alongside user" + assert ( + executor._is_openai_multimodal_format([ + {"role": role, "content": "hi"}, + {"role": "user", "content": "hello"}, + ]) + is True + ), f"Expected role {role!r} to be accepted alongside user" def test_is_openai_multimodal_format_rejects_non_user_only(self): """Arrays with no user-role message are rejected to prevent silent empty message fallback.""" @@ -309,15 +312,21 @@ def test_is_openai_multimodal_format_rejects_non_user_only(self): # Non-user-only Chat Completions format assert executor._is_openai_multimodal_format([{"role": "system", "content": "hi"}]) is False - assert executor._is_openai_multimodal_format([ - {"role": "system", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - ]) is False + assert ( + executor._is_openai_multimodal_format([ + {"role": "system", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ]) + is False + ) # Non-user-only Responses API format - assert executor._is_openai_multimodal_format([ - {"type": "message", "role": "system", "content": "hi"}, - ]) is False + assert ( + executor._is_openai_multimodal_format([ + {"type": "message", "role": "system", "content": "hi"}, + ]) + is False + ) def test_is_openai_multimodal_format_rejects_malformed_input(self): """Test that _is_openai_multimodal_format rejects inputs missing content or with invalid roles."""