Skip to content

Commit e422905

Browse files
pythonHuangclaude
andcommitted
feat: 智能体对话支持生图/生视频模型-模型名关键词识别类型(chat/image/video,前后端一致),后端generate-media端点:通义万相走DashScope异步任务+轮询(生图text2image/生视频text2video,视频约3分钟超时),OpenAI兼容走images/generations(支持url与b64_json),conversation/media端点追加媒体消息到会话历史;前端助手运行页模型下拉显示🎨/🎬标记,生图/生视频模式下发送自动走媒体生成(自动开启会话记录历史),气泡内渲染图片/视频并带下载/打开按钮,生成中显示加载态,媒体模式隐藏图片上传与结束对话按钮,输入框占位提示动态切换,MdContent支持video标签样式
Co-Authored-By: Claude Code <noreply@anthropic.com>
1 parent 9b12e36 commit e422905

6 files changed

Lines changed: 286 additions & 10 deletions

File tree

‎Juggle.Api/Controllers/Api/AiController.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,33 @@ public async Task<ApiResult> ConversationChat([FromBody] AiConversationChatReque
362362
catch (Exception ex) { return ApiResult.Fail(ex.Message); }
363363
}
364364

365+
/// <summary>生成图片/视频(生图/生视频模型)。</summary>
366+
[HttpPost("generate-media")]
367+
public async Task<ApiResult> GenerateMedia([FromBody] AiGenerateMediaRequest req)
368+
{
369+
try
370+
{
371+
var result = await _aiService.GenerateMediaAsync(req.ProviderId, req.Model, req.Prompt ?? "");
372+
return ApiResult.Success(result);
373+
}
374+
catch (Exception ex) { return ApiResult.Fail(ex.Message); }
375+
}
376+
377+
/// <summary>追加媒体消息(生图/生视频提问与结果)到会话历史。</summary>
378+
[HttpPost("conversation/media")]
379+
public async Task<ApiResult> ConversationMedia([FromBody] AiConversationMediaRequest req)
380+
{
381+
try
382+
{
383+
var conv = await _db.AiConversations.FirstOrDefaultAsync(c => c.Id == req.ConversationId && c.Deleted == 0)
384+
?? throw new Exception("会话不存在");
385+
if (conv.Status == 1) throw new Exception("会话已结束,请开启新对话");
386+
await _aiService.AppendMediaMessageAsync(conv, req.UserContent ?? "", req.AssistantContent ?? "");
387+
return ApiResult.Success();
388+
}
389+
catch (Exception ex) { return ApiResult.Fail(ex.Message); }
390+
}
391+
365392
/// <summary>结束对话:有输出参数时生成最终结果,会话标记已结束</summary>
366393
[HttpPost("conversation/end")]
367394
public async Task<ApiResult> EndConversation([FromBody] AiConversationEndRequest req)
@@ -509,6 +536,20 @@ public class AiOptimizePromptRequest
509536
public string? Purpose { get; set; }
510537
}
511538

539+
public class AiGenerateMediaRequest
540+
{
541+
public long ProviderId { get; set; }
542+
public string? Model { get; set; }
543+
public string? Prompt { get; set; }
544+
}
545+
546+
public class AiConversationMediaRequest
547+
{
548+
public long ConversationId { get; set; }
549+
public string? UserContent { get; set; }
550+
public string? AssistantContent { get; set; }
551+
}
552+
512553
public class AiGenerateIconRequest
513554
{
514555
public string? Name { get; set; }

‎Juggle.Application/Services/Impl/AiService.cs‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Text;
22
using System.Text.Json;
3+
using System.Text.RegularExpressions;
34
using Juggle.Domain.Engine.NodeExecutors;
45
using Juggle.Domain.Entities;
56
using Juggle.Infrastructure.Persistence;
@@ -272,6 +273,128 @@ public async Task<AiChatResult> ChatRequestAsync(AiChatRequest req)
272273
return result;
273274
}
274275

276+
/// <summary>模型类型识别(chat/image/video,按模型名关键词,与前端 utils/aiModel.ts 保持一致)。</summary>
277+
public static string DetectModelKind(string? model)
278+
{
279+
var m = (model ?? "").ToLower();
280+
if (Regex.IsMatch(m, @"(t2v|wanx[^a-z0-9]*video|kling|hailuo|pixverse|pika|cogvideo|minimax[^a-z0-9]*video|sora|veo|mochi|ltx|video)"))
281+
return "video";
282+
if (Regex.IsMatch(m, @"(wanx|t2i|dall|gpt-image|flux|stable|midjourney|kandinsky|playground|image)"))
283+
return "image";
284+
return "chat";
285+
}
286+
287+
/// <summary>生成图片/视频(生图/生视频模型):通义万相走 DashScope 异步任务+轮询,OpenAI 兼容走 images/generations。返回 {type,url}。</summary>
288+
public async Task<object> GenerateMediaAsync(long providerId, string? model, string prompt)
289+
{
290+
var kind = DetectModelKind(model);
291+
if (kind == "chat")
292+
throw new Exception($"模型「{model}」不是生图/生视频模型");
293+
294+
var provider = providerId > 0
295+
? await _db.AiProviders.FirstOrDefaultAsync(p => p.Id == providerId && p.Deleted == 0)
296+
: null;
297+
provider ??= await _db.AiProviders.FirstOrDefaultAsync(p => p.Deleted == 0 && p.Enabled == 1)
298+
?? throw new Exception("未配置 AI 大模型:请在系统设置 → 大模型设置中添加并启用供应商");
299+
300+
var apiKey = provider.ApiKey ?? "";
301+
if (string.IsNullOrEmpty(apiKey))
302+
throw new Exception("供应商未配置 API Key");
303+
var modelName = string.IsNullOrWhiteSpace(model) ? provider.Model ?? "" : model;
304+
305+
var client = _httpClientFactory.CreateClient();
306+
client.Timeout = TimeSpan.FromSeconds(60);
307+
308+
// 通义万相(DashScope):异步任务提交 + 轮询结果
309+
if ((provider.BaseUrl ?? "").Contains("dashscope", StringComparison.OrdinalIgnoreCase))
310+
{
311+
var isVideo = kind == "video";
312+
var api = isVideo
313+
? "https://dashscope.aliyuncs.com/api/v1/services/aigc/text2video/video-synthesis"
314+
: "https://dashscope.aliyuncs.com/api/v1/services/aigc/text2image/image-synthesis";
315+
var body = JsonSerializer.Serialize(new
316+
{
317+
model = modelName,
318+
input = new { prompt },
319+
parameters = isVideo ? (object)new { size = "1280*720" } : new { size = "1024*1024", n = 1 }
320+
});
321+
using var reqMsg = new HttpRequestMessage(HttpMethod.Post, api);
322+
reqMsg.Headers.TryAddWithoutValidation("Authorization", $"Bearer {apiKey}");
323+
reqMsg.Headers.TryAddWithoutValidation("X-DashScope-Async", "enable");
324+
reqMsg.Content = new StringContent(body, Encoding.UTF8, "application/json");
325+
using var resp = await client.SendAsync(reqMsg);
326+
var respBody = await resp.Content.ReadAsStringAsync();
327+
if (!resp.IsSuccessStatusCode)
328+
throw new Exception($"{(isVideo ? "视频" : "图片")}生成提交失败({(int)resp.StatusCode}): {Truncate(respBody, 200)}");
329+
using var doc = JsonDocument.Parse(respBody);
330+
var taskId = doc.RootElement.GetProperty("output").GetProperty("task_id").GetString()
331+
?? throw new Exception("未获取到生成任务 ID");
332+
333+
// 轮询任务结果(最多约 3 分钟,视频生成较慢)
334+
for (var i = 0; i < 90; i++)
335+
{
336+
await Task.Delay(2000);
337+
var taskResp = await client.GetAsync($"https://dashscope.aliyuncs.com/api/v1/tasks/{taskId}");
338+
var taskBody = await taskResp.Content.ReadAsStringAsync();
339+
using var tdoc = JsonDocument.Parse(taskBody);
340+
var root = tdoc.RootElement;
341+
var output = root.TryGetProperty("output", out var outEl) ? outEl : default;
342+
var status = output.ValueKind == JsonValueKind.Object && output.TryGetProperty("task_status", out var st)
343+
? st.GetString() : "";
344+
if (status == "SUCCEEDED")
345+
{
346+
var results = output.GetProperty("results");
347+
var url = results[0].GetProperty("url").GetString();
348+
return new { type = kind, url };
349+
}
350+
if (status == "FAILED")
351+
{
352+
var msg = output.TryGetProperty("message", out var me) ? me.GetString() : "未知错误";
353+
throw new Exception($"生成任务失败: {msg}");
354+
}
355+
}
356+
throw new Exception("生成超时(约 3 分钟),请稍后重试");
357+
}
358+
359+
// OpenAI 兼容 images/generations(仅生图)
360+
if (kind == "image")
361+
{
362+
var baseUrl = provider.BaseUrl!.TrimEnd('/');
363+
var url = baseUrl.EndsWith("/images/generations", StringComparison.OrdinalIgnoreCase)
364+
? baseUrl : baseUrl + "/images/generations";
365+
var body = JsonSerializer.Serialize(new { model = modelName, prompt, n = 1, size = "1024x1024" });
366+
using var reqMsg = new HttpRequestMessage(HttpMethod.Post, url);
367+
reqMsg.Headers.TryAddWithoutValidation("Authorization", $"Bearer {apiKey}");
368+
reqMsg.Content = new StringContent(body, Encoding.UTF8, "application/json");
369+
using var resp = await client.SendAsync(reqMsg);
370+
var respBody = await resp.Content.ReadAsStringAsync();
371+
if (!resp.IsSuccessStatusCode)
372+
throw new Exception($"图片生成失败({(int)resp.StatusCode}): {Truncate(respBody, 200)}");
373+
using var doc = JsonDocument.Parse(respBody);
374+
var data = doc.RootElement.GetProperty("data")[0];
375+
if (data.TryGetProperty("url", out var u) && u.ValueKind == JsonValueKind.String)
376+
return new { type = "image", url = u.GetString() };
377+
if (data.TryGetProperty("b64_json", out var b64) && b64.ValueKind == JsonValueKind.String)
378+
return new { type = "image", url = "data:image/png;base64," + b64.GetString() };
379+
throw new Exception("图片生成响应缺少 url/b64_json");
380+
}
381+
382+
throw new Exception("视频生成仅支持通义万相(DashScope)供应商,请添加 dashscope 供应商或将模型切换为通义视频模型");
383+
}
384+
385+
/// <summary>追加媒体消息(生图/生视频的提问与结果)到会话历史。</summary>
386+
public async Task AppendMediaMessageAsync(AiConversationEntity conv, string userContent, string assistantContent)
387+
{
388+
var history = ParseMessages(conv.Messages);
389+
history.Add(("user", userContent));
390+
history.Add(("assistant", assistantContent));
391+
conv.Messages = JsonSerializer.Serialize(history.Select(m => new { role = m.Role, content = m.Content }));
392+
if (string.IsNullOrEmpty(conv.Title))
393+
conv.Title = userContent.Length > 30 ? userContent[..30] : userContent;
394+
conv.UpdatedAt = DateTime.Now.ToString("o");
395+
await _db.SaveChangesAsync();
396+
}
397+
275398
private static string Truncate(string s, int len) => s.Length <= len ? s : s[..len] + "...";
276399

277400
// ==================== 流程生成 ====================

‎JuggleNet6.Frontend/src/components/MdContent.vue‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ onMounted(render)
138138
.md-content :deep(p) { margin: 4px 0; }
139139
.md-content :deep(ul), .md-content :deep(ol) { padding-left: 20px; margin: 4px 0; }
140140
.md-content :deep(img) { max-width: 100%; border-radius: 6px; }
141+
.md-content :deep(video) { max-width: 100%; border-radius: 6px; display: block; margin: 6px 0; }
141142
.md-content :deep(table) { border-collapse: collapse; margin: 6px 0; }
142143
.md-content :deep(th), .md-content :deep(td) { border: 1px solid #e4e7ed; padding: 4px 8px; font-size: 12px; }
143144
.md-content :deep(blockquote) { border-left: 3px solid #dcdfe6; margin: 6px 0; padding: 2px 10px; color: #666; }
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* 模型类型识别(按模型名关键词):chat 对话 / image 生图 / video 生视频。
3+
* 与后端 AiService.DetectModelKind 保持一致。
4+
*/
5+
export function detectModelKind(model?: string): 'chat' | 'image' | 'video' {
6+
const m = (model || '').toLowerCase()
7+
if (/(t2v|wanx[^a-z0-9]*video|kling|hailuo|pixverse|pika|cogvideo|minimax[^a-z0-9]*video|sora|veo|mochi|ltx|video)/.test(m)) return 'video'
8+
if (/(wanx|t2i|dall|gpt-image|flux|stable|midjourney|kandinsky|playground|image)/.test(m)) return 'image'
9+
return 'chat'
10+
}
11+
12+
/** 模型类型标记 emoji(下拉/标签展示用) */
13+
export function modelKindEmoji(model?: string): string {
14+
const kind = detectModelKind(model)
15+
return kind === 'image' ? '🎨' : kind === 'video' ? '🎬' : ''
16+
}

‎JuggleNet6.Frontend/src/views/ai/AiAssistantList.vue‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,9 @@
111111
<el-option v-for="p in aiProviders" :key="p.id" :label="p.providerName" :value="p.id" />
112112
</el-select>
113113
<el-select v-model="form.model" size="small" style="flex:1" clearable filterable allow-create default-first-option placeholder="模型(空=供应商默认)">
114-
<el-option v-for="m in assistantModelOptions" :key="m" :label="m" :value="m" />
114+
<el-option v-for="m in assistantModelOptions" :key="m" :value="m">
115+
<span v-if="modelKindEmoji(m)" style="margin-right:4px">{{ modelKindEmoji(m) }}</span>{{ m }}
116+
</el-option>
115117
</el-select>
116118
</div>
117119
<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px">
@@ -189,6 +191,7 @@ import { ElMessage, ElMessageBox } from 'element-plus'
189191
import request from '../../utils/request'
190192
import MarketPublishDialog from '../../components/MarketPublishDialog.vue'
191193
import MarketShareDialog from '../../components/MarketShareDialog.vue'
194+
import { modelKindEmoji } from '../../utils/aiModel'
192195
193196
// 发布到市场
194197
const publishVisible = ref(false)

0 commit comments

Comments
 (0)