From 0764891ffe0db3fbbc83617456ae96704f3e6197 Mon Sep 17 00:00:00 2001 From: hongwei Date: Wed, 10 Jun 2026 16:32:48 +0800 Subject: [PATCH 1/2] =?UTF-8?q?[0253]=20=E5=A2=9E=E5=8A=A0=E5=AF=B9?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plugins/llm/progs/llm/chat-persist.scm | 13 ++- .../plugins/llm/progs/llm/chat-protocol.scm | 50 +++++++---- .../plugins/llm/progs/llm/chat-tree-ops.scm | 8 ++ devel/0253.md | 84 +++++++++++++++++++ 4 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 devel/0253.md diff --git a/TeXmacs/plugins/llm/progs/llm/chat-persist.scm b/TeXmacs/plugins/llm/progs/llm/chat-persist.scm index 54483109e2..07af1d2eed 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-persist.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-persist.scm @@ -86,7 +86,11 @@ (verbatim-text (texmacs->verbatim body)) (title (string-replace verbatim-text "\n" " ")) ) ; - title + (if (and (string-null? (string-trim-spaces title)) + (chat-tab-tree-has-image? body)) + "[image]" + title + ) ;if ) ;let* ) ;tm-define @@ -146,9 +150,12 @@ ;; 避免 buffer-set-body 对已有嵌入式 editor 触发 assign 导致 crash (let* ((doc (tree-import (system->url msg-path) "generic")) (body (tmfile-extract doc 'body)) + ;; tmfile-extract 可能返回 stree 而非 tree， + ;; 显式转换以确保 raw-data 经 decode_base64 正确还原为二进制 + (body-tree (if (tree? body) body (stree->tree body))) ) ; - (when body - (buffer-set-body msg-buf body) + (when body-tree + (buffer-set-body msg-buf body-tree) (with-buffer msg-buf (session-unfold-last-n n) (chat-tab-add-default-style-packages! "llm") diff --git a/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm b/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm index a099d04691..96022c3a46 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm @@ -25,7 +25,7 @@ ) ;:use ) ;texmacs-module -(import (liii njson)) +(import (liii njson) (liii base64)) ;;; ---------- 全局常量 ---------- @@ -298,10 +298,24 @@ ((or (== suffix "jpg") (== suffix "jpeg")) "image/jpeg") ((== suffix "gif") "image/gif") ((== suffix "webp") "image/webp") + ((== suffix "pdf") "application/pdf") (else #f) ) ;cond ) ;define +(define (chat-tab-read-binary-file file-url) + (let* ((path (url->string (url-concretize file-url))) + (port (open-input-file path))) + (if (not port) + #u8() + (let* ((bytes (let loop + ((c (read-char port)) (res '())) + (if (eof-object? c) + (reverse res) + (loop (read-char port) (cons (char->integer c) res))))) + (dummy (close-input-port port))) + (apply byte-vector bytes))))) ;define + (define (chat-tab-image-node->pair img-stree) ;; img-stree = (image ...) ;; Returns (mime . base64-data) or #f @@ -312,9 +326,10 @@ ;; Embedded: (tuple (raw-data ) ) ((and (pair? name) (eq? (car name) 'tuple) (>= (length name) 3)) (let ((data-node (cadr name)) (suffix-str (caddr name))) - (let ((suffix (url-suffix suffix-str)) - (mime (chat-tab-suffix->mime (url-suffix suffix-str))) - ) ; + (let* ((raw-suffix (url-suffix suffix-str)) + (suffix (if (== raw-suffix "") suffix-str raw-suffix)) + (mime (chat-tab-suffix->mime suffix)) + ) (if (not mime) #f (cond @@ -322,20 +337,26 @@ ((and (pair? data-node) (>= (length data-node) 2) (eq? (car data-node) 'raw-data) - ) ;and + ) (cons mime (cadr data-node)) - ) ; + ) (else #f) ) ;cond ) ;if ) ;let ) ;let - ) ; - ;; Linked: string path — 需要读文件并 base64 编码 + ) + ;; Linked: string path — 读取文件并 base64 编码 ((string? name) - ;; TODO: 需要加载 (liii base64) 后支持链接图片的 base64 编码 - #f - ) ; + (let* ((path-str (cork->utf8 name)) + (suffix (url-suffix path-str)) + (mime (chat-tab-suffix->mime suffix))) + (if (not mime) + #f + (let ((raw-bytes (chat-tab-read-binary-file (string->url path-str)))) + (if (zero? (bytevector-length raw-bytes)) + #f + (cons mime (utf8->string (bytevector-base64-encode raw-bytes)))))))) (else #f) ) ;cond ) ;let @@ -361,11 +382,12 @@ (define (chat-tab-build-context-input input model thinking) ;; 单轮：只编码当前用户输入 + per-round 参数 ;; 线格式：%chat \n\n - (let* ((content (chat-tab-tree->plain-text input)) + (let* ((stree-input (if (tree? input) (tree->stree input) input)) + (images (chat-tab-collect-images stree-input '())) + (stripped (chat-tab-stree-strip-images stree-input)) + (content (serialize-latex (texmacs->latex stripped '()))) (obj (string->njson "{}")) (params (string->njson "{}")) - (stree-input (if (tree? input) (tree->stree input) input)) - (images (chat-tab-collect-images stree-input '())) ) ; (njson-set! params "model" model) (njson-set! params "thinking" thinking) diff --git a/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm b/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm index f0b159123e..b8f19e7c8e 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm @@ -86,6 +86,14 @@ (serialize-latex (texmacs->latex (tm->stree t) '())) ) ;tm-define +(tm-define (chat-tab-stree-strip-images s) + (cond ((string? s) s) + ((not (pair? s)) s) + ((eq? (car s) 'image) "[image]") + (else (cons (car s) (map chat-tab-stree-strip-images (cdr s)))) + ) +) ;define + (tm-define (chat-tab-tree-has-image? t) (let ((s (if (tree? t) (tree->stree t) t))) (cond ((string? s) #f) diff --git a/devel/0253.md b/devel/0253.md new file mode 100644 index 0000000000..2ce3d5dde1 --- /dev/null +++ b/devel/0253.md @@ -0,0 +1,84 @@ +# [0253] LLM Chat 协议支持图片 + +## 1 相关文档 +- [1078.md](1078.md) - Chat 新会话加载 llm.ts 样式包 +- [0416.md](0416.md) - HTML 导出时嵌入式图片直接编码为 Base64 内联 + +## 2 任务相关的代码文件 +- `TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm` - 添加 `chat-tab-stree-strip-images` +- `TeXmacs/plugins/llm/progs/llm/chat-protocol.scm` - 图片提取、base64 编码、上下文构建 +- `TeXmacs/plugins/llm/progs/llm/chat-persist.scm` - 纯图片标题、加载时 stree→tree 转换 + +## 3 如何测试 + +### 3.1 确定性测试（单元测试） +``` +xmake b stem +``` + +### 3.2 非确定性测试（文档验证） +1. 创建 LLM chat，插入嵌入图片 + 文本，发送后检查 `%chat` JSON： + - `content` 不含 `\includegraphics`，含 `[image]` 占位符 + - `images` 数组含 `{"mime":"image/png","data":"..."}` +2. 插入链接图片（如 `image-1.pdf`），检查 `images` 数组含 `{"mime":"application/pdf","data":"..."}` +3. 纯文本消息不受影响 +4. 纯图片消息的会话标题为 `[image]` +5. 重启后切换到有图片的会话，图片正常渲染（无 `ramdisc://` 错误） + +## 4 如何提交 + +```bash +xmake b stem +gf fmt --changed-since=main +``` + +## 5 What + +LLM Chat 的 `%chat` JSON 协议支持图片输入： + +1. 将嵌入图片的 base64 数据提取到 `images` JSON 数组 +2. 将链接图片读取文件并 base64 编码到 `images` JSON 数组 +3. content 中用 `[image]` 占位符替代 `\includegraphics` LaTeX 代码 +4. 纯图片输入时生成 `[image]` 作为会话标题 +5. 修复持久化加载时 `tmfile-extract` 返回 stree 导致 `raw-data` 丢失的问题 + +## 6 Why + +用户在 LLM chat 输入中插入图片时，`chat-tab-tree->plain-text` 将整个文档树（包括图片节点）转换为 LaTeX，导致 `content` 字段出现 `\includegraphics` 代码。LLM 无法理解 LaTeX 图片命令，需要将图片数据以 base64 格式放入独立的 `images` 数组，同时 content 只保留纯文本和 `[image]` 占位符。 + +## 7 How + +### 数据流 + +`raw-data` 在不同层级有不同表示： +- **TMU 文件**: hex 编码 `<#hex...>` +- **C++ tree**: 原始二进制（`from_tmu.cpp` 的 `from_hex`） +- **stree** (`tree→stree`): base64 字符串（`scheme_ser.cpp` 的 `encode_base64`） +- **stree→tree**: 原始二进制（`scheme_der.cpp` 的 `decode_base64`） + +嵌入图片的 stree 中 `(cadr data-node)` 已经是 base64，直接使用；链接图片需 `bytevector-base64-encode` 编码。 + +### 关键改动 + +- `chat-tab-stree-strip-images`: 递归将 `(image ...)` 替换为 `[image]`，需用 `tm-define` 导出（`define` 无法跨模块访问） +- `chat-tab-build-context-input`: 先收集图片 → 剥离图片 → 转 LaTeX，确保 content 不含 `\includegraphics` +- `chat-tab-suffix->mime`: 新增 PDF MIME 类型，修复 `url-suffix "png"` 返回空的问题（与 `embedded-edit.scm` 的 `embedded-suffix` 同模式） +- `chat-persist-extract-title`: 检测纯图片输入，回退为 `[image]` +- `chat-persist-load-session-content`: `tmfile-extract` 返回 stree 而非 tree，需显式 `stree->tree` 以确保 `RAW_DATA` 经 `decode_base64` 正确还原 + +## 2026/06/10 图片协议支持 + +### What +- `%chat` JSON 协议的 `images` 数组正确提取嵌入和链接图片的 base64 数据 +- `content` 字段用 `[image]` 占位符替代 LaTeX 图片代码 +- 纯图片输入生成 `[image]` 标题 +- 修复持久化加载 ramdisc 渲染错误 + +### Why +LLM 无法理解 `\includegraphics`，需要将图片数据以 base64 格式放入独立字段。 + +### How +- 添加 `chat-tab-stree-strip-images`（`tm-define` 导出）递归替换图片节点 +- `chat-tab-build-context-input` 先收集图片再剥离后转 LaTeX +- 修复 `url-suffix` 对裸扩展名（如 `"png"`）返回空的问题 +- `chat-persist-load-session-content` 显式 `stree->tree` 转换修复 `RAW_DATA` 丢失 From f65ba18cac8c76b167718542d1f7f4d8025f8b75 Mon Sep 17 00:00:00 2001 From: Yuki Date: Wed, 10 Jun 2026 19:22:54 +0800 Subject: [PATCH 2/2] gf-format --- TeXmacs/plugins/llm/progs/llm/chat-loader.scm | 6 +-- .../plugins/llm/progs/llm/chat-persist.scm | 3 +- .../plugins/llm/progs/llm/chat-protocol.scm | 37 +++++++++++++------ .../plugins/llm/progs/llm/chat-tree-ops.scm | 4 +- TeXmacs/progs/generic/search-widgets.scm | 4 +- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/TeXmacs/plugins/llm/progs/llm/chat-loader.scm b/TeXmacs/plugins/llm/progs/llm/chat-loader.scm index d83ff51afb..b9f6b9d0fb 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-loader.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-loader.scm @@ -20,8 +20,4 @@ ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(texmacs-module (llm chat-loader) - (:use (llm chat-protocol) - (llm chat-persist) - ) ;:use -) ;texmacs-module +(texmacs-module (llm chat-loader) (:use (llm chat-protocol) (llm chat-persist))) diff --git a/TeXmacs/plugins/llm/progs/llm/chat-persist.scm b/TeXmacs/plugins/llm/progs/llm/chat-persist.scm index 07af1d2eed..9cdc31d47f 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-persist.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-persist.scm @@ -86,8 +86,7 @@ (verbatim-text (texmacs->verbatim body)) (title (string-replace verbatim-text "\n" " ")) ) ; - (if (and (string-null? (string-trim-spaces title)) - (chat-tab-tree-has-image? body)) + (if (and (string-null? (string-trim-spaces title)) (chat-tab-tree-has-image? body)) "[image]" title ) ;if diff --git a/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm b/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm index 96022c3a46..416c5ddaad 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-protocol.scm @@ -304,17 +304,24 @@ ) ;define (define (chat-tab-read-binary-file file-url) - (let* ((path (url->string (url-concretize file-url))) - (port (open-input-file path))) + (let* ((path (url->string (url-concretize file-url))) (port (open-input-file path))) (if (not port) #u8() (let* ((bytes (let loop ((c (read-char port)) (res '())) (if (eof-object? c) (reverse res) - (loop (read-char port) (cons (char->integer c) res))))) - (dummy (close-input-port port))) - (apply byte-vector bytes))))) ;define + (loop (read-char port) (cons (char->integer c) res)) + ) ;if + ) ;let + ) ;bytes + (dummy (close-input-port port)) + ) ; + (apply byte-vector bytes) + ) ;let* + ) ;if + ) ;let* +) ;define (define (chat-tab-image-node->pair img-stree) ;; img-stree = (image ...) @@ -329,7 +336,7 @@ (let* ((raw-suffix (url-suffix suffix-str)) (suffix (if (== raw-suffix "") suffix-str raw-suffix)) (mime (chat-tab-suffix->mime suffix)) - ) + ) ; (if (not mime) #f (cond @@ -337,26 +344,32 @@ ((and (pair? data-node) (>= (length data-node) 2) (eq? (car data-node) 'raw-data) - ) + ) ;and (cons mime (cadr data-node)) - ) + ) ; (else #f) ) ;cond ) ;if - ) ;let + ) ;let* ) ;let - ) + ) ; ;; Linked: string path — 读取文件并 base64 编码 ((string? name) (let* ((path-str (cork->utf8 name)) (suffix (url-suffix path-str)) - (mime (chat-tab-suffix->mime suffix))) + (mime (chat-tab-suffix->mime suffix)) + ) ; (if (not mime) #f (let ((raw-bytes (chat-tab-read-binary-file (string->url path-str)))) (if (zero? (bytevector-length raw-bytes)) #f - (cons mime (utf8->string (bytevector-base64-encode raw-bytes)))))))) + (cons mime (utf8->string (bytevector-base64-encode raw-bytes))) + ) ;if + ) ;let + ) ;if + ) ;let* + ) ; (else #f) ) ;cond ) ;let diff --git a/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm b/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm index b8f19e7c8e..a86867c79b 100644 --- a/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm +++ b/TeXmacs/plugins/llm/progs/llm/chat-tree-ops.scm @@ -91,8 +91,8 @@ ((not (pair? s)) s) ((eq? (car s) 'image) "[image]") (else (cons (car s) (map chat-tab-stree-strip-images (cdr s)))) - ) -) ;define + ) ;cond +) ;tm-define (tm-define (chat-tab-tree-has-image? t) (let ((s (if (tree? t) (tree->stree t) t))) diff --git a/TeXmacs/progs/generic/search-widgets.scm b/TeXmacs/progs/generic/search-widgets.scm index a4d3d42944..1d5659a1d8 100644 --- a/TeXmacs/progs/generic/search-widgets.scm +++ b/TeXmacs/progs/generic/search-widgets.scm @@ -1293,8 +1293,8 @@ (set-search-window-state #f #f) (let* ((msg-url (url->system floating-search-target)) (in-url (if (chat-message-buffer? floating-search-target) - (url->system (chat-tab-session->input-buffer - (chat-buffer-session-id floating-search-target))) + (url->system (chat-tab-session->input-buffer (chat-buffer-session-id floating-search-target)) + ) ;url->system "" ) ;if ) ;in-url