Skip to content

Commit 023e7a3

Browse files
pythonHuangclaude
pythonHuang
and
claude
committed
feat: 大模型节点增强-可设置供应商与模型/输入变量可选流程入出参及中间变量/图片输入(多模态视觉识别,多选data URL变量)/输出目标可选变量出参入参;文件解析节点支持图片类型(视觉模型识别描述);AiService增加OpenAI兼容多模态对话接口
Co-Authored-By: Claude Code <noreply@anthropic.com>
1 parent d15f35a commit 023e7a3

7 files changed

Lines changed: 203 additions & 31 deletions

File tree

‎Juggle.Application/Services/Flow/FlowExecutionService.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,13 @@ public FlowExecutionService(JuggleDbContext db, IHttpClientFactory httpClientFac
4545
return $"{host}:{port},password={password},defaultDatabase={dbIndex},abortConnect=false,connectTimeout=5000";
4646
}
4747

48-
/// <summary>构建引擎（注入大模型对话函数、Redis 连接与知识库检索，供 AI/Redis/KB_SEARCH 节点使用）。</summary>
48+
/// <summary>构建引擎（注入大模型对话/视觉函数、Redis 连接与知识库检索，供 AI/文件解析/Redis/KB_SEARCH 节点使用）。</summary>
4949
private async Task<FlowEngine> BuildEngineAsync(Dictionary<string, DataSourceInfo> dsInfos,
5050
Dictionary<string, string?> staticVars, Func<string, Task<string?>> flowContentLoader)
5151
=> new FlowEngine(_httpClientFactory, dsInfos, staticVars, flowContentLoader,
52-
aiChatFunc: (systemPrompt, userInput) => _aiService.ChatAsync(systemPrompt, userInput),
52+
aiChatFunc: req => req.Images.Count > 0
53+
? _aiService.ChatWithImagesAsync(req.SystemPrompt, req.UserInput, req.Images, req.ProviderId, req.Model)
54+
: _aiService.ChatAsync(req.SystemPrompt, req.UserInput, req.ProviderId, modelOverride: req.Model),
5355
redisConnStr: await GetRedisConnStrAsync(),
5456
kbSearchFunc: (kbId, query, topK) => _kbService.SearchAsContextAsync(kbId, query, topK));
5557

‎Juggle.Application/Services/Impl/AiService.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,48 @@ public async Task<string> ChatAsync(string systemPrompt, string userPrompt, long
106106
return await ChatMessagesAsync(messages, providerId, config, modelOverride);
107107
}
108108

109+
/// <summary>视觉对话：文本 + 图片（OpenAI 兼容多模态 content 数组）。</summary>
110+
public async Task<string> ChatWithImagesAsync(string systemPrompt, string userText, List<string> imageUrls,
111+
long providerId = 0, string? modelOverride = null)
112+
{
113+
var cfg = await ResolveConfigAsync(providerId > 0 ? providerId : null);
114+
if (!string.IsNullOrEmpty(modelOverride)) cfg.Model = modelOverride;
115+
if (string.IsNullOrEmpty(cfg.BaseUrl) || string.IsNullOrEmpty(cfg.ApiKey))
116+
throw new Exception("未配置 AI 大模型：请在系统设置 → 大模型设置中添加并启用供应商");
117+
118+
var url = cfg.BaseUrl.TrimEnd('/');
119+
if (!url.EndsWith("/chat/completions", StringComparison.OrdinalIgnoreCase))
120+
url += "/chat/completions";
121+
122+
// 多模态 content：[{type:text,text},{type:image_url,image_url:{url}}]
123+
var content = new List<object>();
124+
if (!string.IsNullOrWhiteSpace(userText))
125+
content.Add(new { type = "text", text = userText });
126+
foreach (var img in imageUrls)
127+
content.Add(new { type = "image_url", image_url = new { url = img } });
128+
129+
var client = _httpClientFactory.CreateClient();
130+
client.Timeout = TimeSpan.FromSeconds(180);
131+
using var reqMsg = new HttpRequestMessage(HttpMethod.Post, url);
132+
reqMsg.Headers.TryAddWithoutValidation("Authorization", $"Bearer {cfg.ApiKey}");
133+
reqMsg.Content = new StringContent(JsonSerializer.Serialize(new
134+
{
135+
model = cfg.Model,
136+
messages = new object[]
137+
{
138+
new { role = "system", content = systemPrompt },
139+
new { role = "user", content = content.ToArray() }
140+
},
141+
temperature = 0.2
142+
}), Encoding.UTF8, "application/json");
143+
using var resp = await client.SendAsync(reqMsg);
144+
var body = await resp.Content.ReadAsStringAsync();
145+
if (!resp.IsSuccessStatusCode)
146+
throw new Exception($"AI 调用失败({(int)resp.StatusCode}): {Truncate(body, 300)}");
147+
using var doc = JsonDocument.Parse(body);
148+
return doc.RootElement.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()?.Trim() ?? "";
149+
}
150+
109151
/// <summary>多轮对话：完整消息列表（system + 历史消息）一次性发送。</summary>
110152
public async Task<string> ChatMessagesAsync(List<(string Role, string Content)> messages, long providerId = 0, AiConfig? config = null, string? modelOverride = null)
111153
{

‎Juggle.Domain/Engine/FlowEngine.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ public class FlowEngine
1717
private readonly Dictionary<string, string?> _staticVarSnapshot;
1818
/// <summary>根据 flowKey 加载最新已发布流程内容（供 SUB_FLOW 节点调用），可为 null（不支持子流程）</summary>
1919
private readonly Func<string, Task<string?>>? _flowContentLoader;
20-
/// <summary>大模型对话函数（systemPrompt, userInput → reply），供 AI 节点调用，可为 null（未接入大模型）</summary>
21-
private readonly Func<string, string, Task<string>>? _aiChatFunc;
20+
/// <summary>大模型对话函数（AiChatRequest → reply），供 AI 节点与文件解析(图片)调用，可为 null（未接入大模型）</summary>
21+
private readonly Func<AiChatRequest, Task<string>>? _aiChatFunc;
2222
/// <summary>Redis 连接串（供 REDIS_GET/REDIS_SET 节点使用），可为 null（未配置 Redis）</summary>
2323
private readonly string? _redisConnStr;
2424
/// <summary>知识库检索函数（kbId, query, topK → 上下文文本），供 KB_SEARCH 节点使用</summary>
@@ -28,7 +28,7 @@ public FlowEngine(IHttpClientFactory httpClientFactory,
2828
Dictionary<string, DataSourceInfo>? dataSources = null,
2929
Dictionary<string, string?>? staticVariables = null,
3030
Func<string, Task<string?>>? flowContentLoader = null,
31-
Func<string, string, Task<string>>? aiChatFunc = null,
31+
Func<AiChatRequest, Task<string>>? aiChatFunc = null,
3232
string? redisConnStr = null,
3333
Func<long, string, int, Task<string>>? kbSearchFunc = null)
3434
{
@@ -162,7 +162,7 @@ public async Task<FlowResult> ExecuteAsync(
162162
"TRANSFORM" => new TransformNodeExecutor(),
163163
"AI" => new AiNodeExecutor(
164164
_aiChatFunc ?? throw new InvalidOperationException("流程引擎未接入大模型，无法执行 AI 节点")),
165-
"FILE_PARSE" => new FileParseNodeExecutor(),
165+
"FILE_PARSE" => new FileParseNodeExecutor(_aiChatFunc),
166166
"EXCEL_READ" => new ExcelReadNodeExecutor(),
167167
"FILE_WRITE" => new FileWriteNodeExecutor(),
168168
"REDIS_GET" => new RedisGetNodeExecutor(

‎Juggle.Domain/Engine/FlowModels.cs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,9 +278,25 @@ public class AiConfig
278278
[JsonPropertyName("systemPrompt")]
279279
public string SystemPrompt { get; set; } = "";
280280

281-
/// <summary>输出变量名（模型回复写入）</summary>
281+
/// <summary>输出目标名（模型回复写入）</summary>
282282
[JsonPropertyName("output")]
283283
public string Output { get; set; } = "";
284+
285+
/// <summary>输出目标类型：VARIABLE（中间变量）/ OUTPUT（出参）/ INPUT（入参）</summary>
286+
[JsonPropertyName("outputTargetType")]
287+
public string OutputTargetType { get; set; } = "VARIABLE";
288+
289+
/// <summary>图片输入变量（逗号分隔多个，data URL / http 地址，供视觉模型识别）</summary>
290+
[JsonPropertyName("inputImages")]
291+
public string InputImages { get; set; } = "";
292+
293+
/// <summary>供应商 ID（0=第一个启用供应商）</summary>
294+
[JsonPropertyName("providerId")]
295+
public long ProviderId { get; set; }
296+
297+
/// <summary>模型名（空=供应商默认模型）</summary>
298+
[JsonPropertyName("model")]
299+
public string Model { get; set; } = "";
284300
}
285301

286302
/// <summary>文件解析节点配置（FILE_PARSE 节点）</summary>

‎Juggle.Domain/Engine/NodeExecutors/AiNodeExecutor.cs‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
namespace Juggle.Domain.Engine.NodeExecutors;
22

3+
/// <summary>大模型调用请求（AI 节点与文件解析图片识别共用）。</summary>
4+
public record AiChatRequest(string SystemPrompt, string UserInput, List<string> Images, long ProviderId, string? Model);
5+
36
/// <summary>
47
/// AI 大模型节点执行器：把输入变量内容作为用户消息、按系统提示词调用大模型，
5-
/// 回复写入输出变量。对话函数由应用层注入（供应商/模型配置见系统设置 → 大模型设置）。
8+
/// 回复写入输出变量（支持选择供应商/模型、图片输入、输出到变量/入参/出参）。
9+
/// 对话函数由应用层注入（供应商/模型配置见系统设置 → 大模型设置）。
610
/// </summary>
711
public class AiNodeExecutor : INodeExecutor
812
{
9-
private readonly Func<string, string, Task<string>> _chat;
13+
private readonly Func<AiChatRequest, Task<string>> _chat;
1014

11-
public AiNodeExecutor(Func<string, string, Task<string>> chat) => _chat = chat;
15+
public AiNodeExecutor(Func<AiChatRequest, Task<string>> chat) => _chat = chat;
1216

1317
public async Task<string?> ExecuteAsync(FlowNode node, FlowContext context)
1418
{
@@ -18,10 +22,39 @@ public class AiNodeExecutor : INodeExecutor
1822
var input = string.IsNullOrWhiteSpace(cfg.Input) ? "" : context.GetVariable(cfg.Input)?.ToString() ?? "";
1923
var systemPrompt = string.IsNullOrWhiteSpace(cfg.SystemPrompt) ? "你是一个智能助手。" : cfg.SystemPrompt;
2024

21-
var reply = await _chat(systemPrompt, input);
25+
// 图片输入（视觉模型识别）
26+
var images = new List<string>();
27+
if (!string.IsNullOrWhiteSpace(cfg.InputImages))
28+
{
29+
foreach (var name in cfg.InputImages.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
30+
{
31+
var v = context.GetVariable(name);
32+
if (v != null)
33+
{
34+
var str = v.ToString() ?? "";
35+
if (str.StartsWith("data:image") || str.StartsWith("http"))
36+
images.Add(str);
37+
}
38+
}
39+
}
40+
41+
var reply = await _chat(new AiChatRequest(systemPrompt, input, images, cfg.ProviderId, string.IsNullOrWhiteSpace(cfg.Model) ? null : cfg.Model));
2242

2343
if (!string.IsNullOrWhiteSpace(cfg.Output))
24-
context.SetVariable(cfg.Output, reply);
44+
{
45+
switch ((cfg.OutputTargetType ?? "VARIABLE").ToUpper())
46+
{
47+
case "OUTPUT":
48+
context.SetOutputParameter(cfg.Output, reply);
49+
break;
50+
case "INPUT":
51+
context.SetVariable(cfg.Output, reply);
52+
break;
53+
default:
54+
context.SetVariable(cfg.Output, reply);
55+
break;
56+
}
57+
}
2558

2659
return node.Outgoings.FirstOrDefault();
2760
}

‎Juggle.Domain/Engine/NodeExecutors/FileNodeExecutors.cs‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,51 @@
44

55
namespace Juggle.Domain.Engine.NodeExecutors;
66

7-
/// <summary>文件解析节点：解析文本/JSON/XML/CSV 文件内容（支持 data URL 或 base64 输入）。</summary>
7+
/// <summary>文件解析节点：解析文本/JSON/XML/CSV/图片 文件内容（支持 data URL 或 base64 输入，图片用视觉模型识别）。</summary>
88
public class FileParseNodeExecutor : INodeExecutor
99
{
10-
public Task<string?> ExecuteAsync(FlowNode node, FlowContext context)
10+
private readonly Func<AiChatRequest, Task<string>>? _chat;
11+
12+
public FileParseNodeExecutor(Func<AiChatRequest, Task<string>>? chat = null) => _chat = chat;
13+
14+
public async Task<string?> ExecuteAsync(FlowNode node, FlowContext context)
1115
{
1216
var cfg = node.FileParseConfig
1317
?? throw new InvalidOperationException($"文件解析节点 [{node.Key}] 未配置 fileParseConfig。");
1418

1519
var raw = string.IsNullOrWhiteSpace(cfg.Input) ? "" : context.GetVariable(cfg.Input)?.ToString() ?? "";
16-
var content = DecodeContent(raw);
1720
var fileType = (cfg.FileType ?? "auto").ToLower();
1821

19-
object? result = fileType switch
22+
object? result;
23+
if (fileType == "image")
2024
{
21-
"json" => ParseJson(content),
22-
"csv" => ParseCsv(content),
23-
"text" => content,
24-
"xml" => content,
25-
_ => TryParseJson(content, out var obj) ? obj : content
26-
};
25+
// 图片：视觉模型识别（未接入大模型时报错）
26+
if (_chat == null)
27+
throw new InvalidOperationException("流程引擎未接入大模型，无法识别图片（请先在系统设置 → 大模型设置中配置供应商）");
28+
var imageUrl = raw.Trim();
29+
if (!imageUrl.StartsWith("data:image") && !imageUrl.StartsWith("http"))
30+
imageUrl = "data:image/png;base64," + imageUrl;
31+
result = await _chat(new AiChatRequest(
32+
"你是一个专业的图片识别助手，请详细、准确地描述图片内容（文字、对象、场景等）。",
33+
"请识别这张图片", new List<string> { imageUrl }, 0, null));
34+
}
35+
else
36+
{
37+
var content = DecodeContent(raw);
38+
result = fileType switch
39+
{
40+
"json" => ParseJson(content),
41+
"csv" => ParseCsv(content),
42+
"text" => content,
43+
"xml" => content,
44+
_ => TryParseJson(content, out var obj) ? obj : content
45+
};
46+
}
2747

2848
if (!string.IsNullOrWhiteSpace(cfg.Output))
2949
context.SetVariable(cfg.Output, result);
3050

31-
return Task.FromResult(node.Outgoings.FirstOrDefault());
51+
return node.Outgoings.FirstOrDefault();
3252
}
3353

3454
/// <summary>解析 JSON 为对象/数组（失败返回原文）。</summary>

0 commit comments

Comments
 (0)