Skip to content

Fix：robot image issues and add test cases #1815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lichunn merged 3 commits into from
Jun 30, 2026
Merged

Fix：robot image issues and add test cases #1815

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
01104b9
fix：调整RobotChat逻辑和修复图片上传问题
lichunn Jun 26, 2026
3b65f7d
feat：添加测试用例
lichunn Jun 26, 2026
6d31656
fix:fix review
lichunn Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 75 additions & 6 deletions packages/plugins/robot/src/Main.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@
v-model:show="robotVisible"
v-model:input="inputMessage"
:status="mappedStatus"
:chat-mode="robotSettingState.chatMode"
:prompt-items="promptItems"
:bubble-renderers="bubbleRenderers"
:allowFiles="isVisualModel && robotSettingState.chatMode === ChatMode.Agent"
:show-aborted="robotSettingState.chatMode !== ChatMode.Agent"
:message-content-resolver="resolveChatMessageContent"
:beforeSubmit="checkApiKey"
:promptClickHandler="promptClickHandler"
@fileSelected="handleFileSelected"
Expand Down Expand Up @@ -93,6 +93,16 @@ import { AgentRenderer } from './components/renderers'
import useChat from './composables/useChat'
import useModelConfig from './composables/core/useConfig'
import { ChatMode } from './types/mode.types'
import { STATUS } from './constants/status'
import {
AgentMessageStatus,
RobotMessageContentType,
RobotMessageRole,
isAgentFinalStatus,
type MessageResolverContext,
type RobotMessage,
type RobotRenderContentItem
} from './types'
import apiService from './services/api'

const props = defineProps({
Expand All @@ -106,6 +116,7 @@ const { robotSettingState, getModelCapabilities, updateThinkingState, getSelecte

const robotVisible = ref(false)
const fullscreen = ref(false)
const inputMessage = ref('')

watch(robotVisible, (visible) => {
useLayout().layoutState.toolbars.render = visible ? META_APP.Robot : ''
Expand Down Expand Up @@ -150,7 +161,6 @@ const showSetting = ref(false)

const {
mappedStatus,
inputMessage,
messages,
changeChatMode,
abortRequest,
Expand Down Expand Up @@ -224,9 +234,9 @@ const promptClickHandler = (item: PromptProps & { mode?: 'chat' | 'agent' }) =>
changeChatMode(item.mode)
}
messages.value.push({
role: 'user',
role: RobotMessageRole.User,
content: item.description || '',
renderContent: [{ type: 'text', content: item.description }]
renderContent: [{ type: RobotMessageContentType.Text, content: item.description }]
})
sendUserMessage()
}
Expand All @@ -245,8 +255,67 @@ const openAIRobot = () => {
useLayout().closeSetting(true)
}

// 当前Robot的bubbleRenderers无法做到响应式更新，因此Agent模式的type要与Chat模式不同
const bubbleRenderers = { 'agent-content': AgentRenderer, 'agent-loading': AgentRenderer }
// 当前 Robot 的 bubbleRenderers 无法做到响应式更新，因此 Agent 模式需要独立的内容类型。
// `agent-content` 表示 Agent 的最终内容片段；`agent-loading` 表示 Agent 处理中间态占位片段。
const bubbleRenderers = {
[RobotMessageContentType.AgentContent]: AgentRenderer,
[RobotMessageContentType.AgentLoading]: AgentRenderer
}

const resolveChatMessageContent = (message: RobotMessage, context: MessageResolverContext) => {
const renderContent = Array.isArray(message.renderContent) ? message.renderContent : []
const hasAgentContent = renderContent.some((item) => {
return item.type === RobotMessageContentType.AgentContent || item.type === RobotMessageContentType.AgentLoading
})
const isAgentMessage = message.metadata?.chatMode === ChatMode.Agent || hasAgentContent

if (!isAgentMessage || message.role !== RobotMessageRole.Assistant) {
return Array.isArray(message.renderContent) && message.renderContent.length > 0
? message.renderContent
: message.content
}

// context.status 是当前整轮对话请求的运行状态，不是单条渲染片段状态。
const isLastMessage = context.messages.at(-1) === message
const isGenerating = Boolean(message.loading) || (isLastMessage && context.status !== STATUS.FINISHED)
const resolvedRenderContent = isGenerating
? renderContent
: renderContent.filter((item) => item.type !== RobotMessageContentType.AgentLoading)
Comment thread
lichunn marked this conversation as resolved.
const agentContents = resolvedRenderContent.filter(
(item): item is RobotRenderContentItem =>
item.type === RobotMessageContentType.AgentContent || item.type === RobotMessageContentType.AgentLoading
)
// item.status 是单个 agent 片段的业务结果，例如 success/failed/fix/loading。
const finalStatus = agentContents.findLast((item) => isAgentFinalStatus(item.status))?.status

if (!Array.isArray(message.renderContent) || message.renderContent.length === 0) {
const agentStatus = isAgentFinalStatus(message.metadata?.agentStatus)
? message.metadata.agentStatus
: AgentMessageStatus.Failed
return [
{
type: RobotMessageContentType.AgentContent,
status: agentStatus,
content: message.content
}
]
}

return resolvedRenderContent.map((item) => {
if (item.type !== RobotMessageContentType.AgentContent || isGenerating) {
return item
}

if (!item.status || item.status === AgentMessageStatus.Loading) {
return {
...item,
status: finalStatus || message.metadata?.agentStatus || AgentMessageStatus.Failed
}
}

return item
})
}

const handleFileSelected = async (formData: FormData, updateAttachment: (resourceUrl: string) => void) => {
try {
Expand Down
161 changes: 79 additions & 82 deletions packages/plugins/robot/src/components/chat/RobotChat.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</template>

<div class="robot-chat-container-content" ref="chatContainerRef">
<div v-if="messages.filter((item) => item.role !== 'system').length === 0">
<div v-if="messages.filter((item) => item.role !== RobotMessageRole.System).length === 0">
<tr-welcome title="AI助手" description="您好，我是您的开发小助手" :icon="welcomeIcon" class="robot-welcome">
</tr-welcome>
<tr-prompts
Expand Down Expand Up @@ -99,10 +99,18 @@ import {
type RawFileAttachment,
type BubbleContentRendererMatch
} from '@opentiny/tiny-robot'
import { type ChatMessage } from '@opentiny/tiny-robot-kit'
import { GeneratingStatus } from '../../constants/status'
import { LoadingRenderer, MarkdownRenderer, ImgRenderer } from '../renderers'
import { useNotify } from '@opentiny/tiny-engine-meta-register'
import {
RobotMessageContentType,
RobotMessageRole,
type MessageContentResolver,
type RobotInputContentPart,
type RobotMessage,
type RobotRenderContentItem
} from '../../types'
import { extractMessageText } from '../../utils'

const props = defineProps({
promptItems: {
Expand All @@ -113,7 +121,9 @@ const props = defineProps({
type: Function
},
status: { type: String },
chatMode: { type: String },
messageContentResolver: {
type: Function as PropType<MessageContentResolver>
},
allowFiles: {
type: Boolean,
default: false
Expand All @@ -139,7 +149,7 @@ const selectedAttachments = ref([])
const robotVisible = defineModel<boolean>('show', { required: true })
const fullscreen = defineModel<boolean>('fullscreen')
const inputMessage = defineModel<string>('input', { required: true })
const messages = defineModel<ChatMessage[]>('messages', { required: true })
const messages = defineModel<RobotMessage[]>('messages', { required: true })
const senderRef = ref<InstanceType<typeof TrSender> | null>(null)

watch(
Expand All @@ -159,63 +169,29 @@ const contentRendererMatches = computed<BubbleContentRendererMatch[]>(() => [
},
...Object.entries(props.bubbleRenderers).map(([type, renderer]) => ({
priority: BubbleRendererMatchPriority.NORMAL,
find: (_message: any, content: any) => content?.type === type,
find: (_message: RobotMessage, content: RobotRenderContentItem) => content?.type === type,
renderer
})),
{
priority: BubbleRendererMatchPriority.NORMAL,
find: (message: any, content: any) => content?.type === 'tool' && message.tool_calls?.length,
find: (message: RobotMessage, content: RobotRenderContentItem) =>
content?.type === RobotMessageContentType.Tool && Boolean(message.tool_calls?.length),
renderer: BubbleRenderers.Tools
},
{
priority: BubbleRendererMatchPriority.NORMAL,
find: (message: any, content: any) =>
!message.loading && message.content && (!content?.type || ['markdown', 'text'].includes(content.type)),
find: (_message: RobotMessage, content: RobotRenderContentItem) =>
!content?.type || [RobotMessageContentType.Markdown, RobotMessageContentType.Text].includes(content.type as any),
renderer: MarkdownRenderer
},
{
priority: BubbleRendererMatchPriority.NORMAL,
find: (message: any) => message?.content?.[0]?.type === 'img' || message?.content?.[0]?.type === 'image',
find: (_message: RobotMessage, content: RobotRenderContentItem) =>
[RobotMessageContentType.Img, RobotMessageContentType.Image].includes(content?.type as any),
Comment thread
chilingling marked this conversation as resolved.
renderer: ImgRenderer
}
])

const isAgentMessage = (message: any) => {
const hasAgentContent = message.renderContent?.some((item: any) => {
return item.type === 'agent-content' || item.type === 'agent-loading'
})
return message.metadata?.chatMode === 'agent' || hasAgentContent
}

const resolveAgentRenderContent = (message: any) => {
if (!isAgentMessage(message) || message.role !== 'assistant') {
return message.renderContent
}

const isLastMessage = messages.value.at(-1) === message
const isGenerating = Boolean(message.loading) || (isLastMessage && GeneratingStatus.includes(props.status as any))
const renderContent = isGenerating
? message.renderContent
: message.renderContent.filter((item: any) => item.type !== 'agent-loading')
const agentContents = renderContent.filter((item: any) => item.type === 'agent-content')
const finalStatus = agentContents.findLast((item: any) => ['success', 'failed', 'fix'].includes(item.status))?.status

return renderContent.map((item: any) => {
if (item.type !== 'agent-content' || isGenerating) {
return item
}

if (!item.status || item.status === 'loading') {
return {
...item,
status: finalStatus || message.metadata?.agentStatus || 'failed'
}
}

return item
})
}

// 处理文件选择事件
const handleSingleFilesSelected = (files: File[] | null, retry = false) => {
if (!files?.length) return
Expand Down Expand Up @@ -271,36 +247,56 @@ const getSvgIcon = (name: string, style?: CSSProperties) => {
const aiAvatar = getSvgIcon('AI')
const welcomeIcon = getSvgIcon('AI', { fontSize: '44px' })

const resolveMessageContent = (message: any) => {
if (Array.isArray(message.renderContent) && message.renderContent.length > 0) {
return resolveAgentRenderContent(message)
const resolveMessageContent = (message: RobotMessage) => {
if (props.messageContentResolver) {
return props.messageContentResolver(message, {
messages: messages.value,
status: props.status
})
}

if (isAgentMessage(message) && message.role === 'assistant' && message.content) {
const agentStatus = ['success', 'failed', 'fix'].includes(message.metadata?.agentStatus)
? message.metadata.agentStatus
: 'failed'
return [
{
type: 'agent-content',
status: agentStatus,
content: message.content
if (Array.isArray(message.renderContent) && message.renderContent.length > 0) {
return message.renderContent.map((item) => {
if (item?.type === RobotMessageContentType.Img || item?.type === RobotMessageContentType.Image) {
return {
type: RobotMessageContentType.Img,
content: item.content || item.url || item.image_url?.url || ''
}
Comment thread
lichunn marked this conversation as resolved.
}
]
if (item?.type === RobotMessageContentType.Text) {
return {
type: RobotMessageContentType.Text,
content: item.content ?? item.text ?? ''
}
}
return item
})
}

if (Array.isArray(message.content) && message.content.length > 0) {
const textContent = extractMessageText(message.content)
if (textContent) {
return textContent
}
}

const textContent = extractMessageText(message.content)
if (textContent) {
return textContent
}

return message.content
}

const roleConfigs: Record<string, BubbleRoleConfig> = {
assistant: {
[RobotMessageRole.Assistant]: {
placement: 'start',
avatar: aiAvatar
},
user: {
[RobotMessageRole.User]: {
placement: 'end'
},
system: {
[RobotMessageRole.System]: {
hidden: true
}
}
Expand All @@ -320,37 +316,38 @@ const handleSendMessage = async (content: string) => {
return
}

const userMessage: ChatMessage = {
role: 'user',
const userMessage: RobotMessage = {
role: RobotMessageRole.User,
content: messageContent
}
const files = selectedAttachments.value.filter((item) => item.status === 'success')
if (files.length > 0) {
const fileMessages: ChatMessage[] = files.map((file) => ({
role: 'user',
content: '',
renderContent: [
{
type: 'img',
content: file.url
}
]
}))
messages.value.push(...fileMessages)
userMessage.content = files
.map((item) => ({
type: 'image_url',
userMessage.content = [
{
type: RobotMessageContentType.Text,
text: messageContent
},
...files.map((item) => ({
type: RobotMessageContentType.ImageUrl,
image_url: {
url: item.url
}
}))
.concat({
type: 'text',
text: messageContent
})
] as RobotInputContentPart[]
userMessage.renderContent = [
{
type: RobotMessageContentType.Text,
content: messageContent
},
...files.map((item) => ({
type: RobotMessageContentType.Img,
content: item.url
}))
]
} else {
userMessage.renderContent = [
{
type: 'text',
type: RobotMessageContentType.Text,
content: messageContent
}
]
Expand Down
Loading
Loading