diff --git a/.eslintrc.json b/.eslintrc.json index 59ca30194..ab408ed02 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -12,7 +12,7 @@ "rules": { "react/react-in-jsx-scope": "off" }, - "ignorePatterns": ["build/**", "build.mjs", "src/utils/is-mobile.mjs"], + "ignorePatterns": ["build/**", "test-results/**", "build.mjs", "src/utils/is-mobile.mjs"], "settings": { "react": { "version": "detect" diff --git a/.gitignore b/.gitignore index 36e152b0d..50d21fa35 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .vscode/ node_modules/ build/ +test-results/ .coverage/ coverage/ .DS_Store diff --git a/.prettierignore b/.prettierignore index 997ea4170..4d2ee8cef 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ build/ +test-results/ src/manifest.json -src/manifest.v2.json \ No newline at end of file +src/manifest.v2.json diff --git a/IMAGE-SUPPORT.zh-CN.md b/IMAGE-SUPPORT.zh-CN.md new file mode 100644 index 000000000..b8f8e3220 --- /dev/null +++ b/IMAGE-SUPPORT.zh-CN.md @@ -0,0 +1,75 @@ +# 侧栏图片适配版 + +基于 ChatGPTBox 2.7.1；图片功能最初开发于 `ab221d2`，现已合并到此 fork 的 `c5ddb90` 之后。这是源码修改版，并非插件商店发布版。 + +## 2026-09-12：Blackboard PDF 侧栏修复（images-2） + +用户实际错误为 `chrome.sidePanel.open requires a valid windowId`，来自上一版的参数校验：PDF 菜单回调缺少有效窗口 ID 时，代码在调用浏览器 API 前就拒绝打开。现在使用回调提供的有效窗口 ID；缺失时同步传 `WINDOW_ID_CURRENT`，由浏览器解析当前窗口，避免异步查询丢失用户手势。保留全局侧栏路径：独立 Edge 测试发现按 PDF 标签页打开可以返回成功却不创建可见侧栏，因此不以 API 成功返回作为验收依据。 + +侧栏与独立窗口共用图片组件，使用标准 `IndependentPanel.html` 入口。历史调试版本曾显示内部版本标记；正式提交已移除该标记、查询参数和 fork 专用版本名。打开侧栏无需读取 PDF 地址、重新下载 PDF 或向 PDF 阅读器注入代码。 + +同时将右键点击监听器的注册移到异步菜单初始化之前，避免扩展后台刚被唤醒时遗漏第一次点击。更新后需在扩展管理页点击“重新加载”，再重新打开侧栏；仅刷新 PDF 页面不会加载新的后台代码。 + +本轮 `npm test` 全部通过，覆盖缺失窗口 ID、无效窗口 ID、缺失整个 tab 对象，确保在用户点击时同步调用 API。格式、lint 和构建通过。原生 Edge 侧栏测试确认 `{}` 和只有 `id` 的回调均通过 `WINDOW_ID_CURRENT` 打开新版侧栏，约 382 像素宽时图片按钮、提示和文件输入可见，选择本地 PNG 后生成正确图片预览，未增加 PDF 请求或额外打开对话标签页。自动化测试使用独立 Edge 环境和本地 PDF，没有访问学校账户；Blackboard 页面由用户手动复测，并在 README 中保留运行截图。 + +## 安装 + +**安装包与源码包不同：** + +- `chromium.zip`：已经编译好的浏览器安装包。包含 `manifest.json`、`content-script.js` 等文件，没有 `src`、`package.json` 和构建环境。这是正常打包结果，请直接加载，不要在其中执行 `npm run build`。 +- `chatgptbox-image-support-source.zip`：完整修改版源码，包含 `src/content-script`、`src/components`、`src/services`、测试、`package.json`、`package-lock.json`、`build.mjs` 和隐藏配置文件。需要自己开发或重新构建时使用此包。 + +1. 在 Edge 打开 `edge://extensions`，或在 Chrome 打开 `chrome://extensions`。 +2. 打开“开发人员模式”，选择“加载解压缩的扩展”。 +3. 选择本项目的 `build/chromium` 文件夹（其中应包含 `manifest.json`）。也可以解压 `build/chromium.zip` 后选择解压目录。 +4. 商店版可以保留；为避免同一网页出现两个入口，使用开发版期间可暂时关闭商店版。 +5. 开发版使用独立的扩展存储，需要重新配置服务商和 API Key。 + +## 使用 + +在侧栏选择 OpenAI 兼容的 Chat Completions API 接入，配置支持图片输入的模型。支持该协议不代表每个模型都能看图；模型和端点需要由服务商提供图片理解能力。 + +- 点击图片按钮选择本地文件。 +- 截图后在输入区域按 Ctrl + V 粘贴。 +- 将图片文件拖入输入区域。 +- 发送前可预览、移除图片。可以只发送图片，也可以同时输入问题。 +- 后续提问会在配置的历史范围内保留此前图片；重试也会带上对应图片。 +- 清空对话会同时清除当前图片草稿。 + +接受 PNG、JPEG、WebP、GIF，每次最多 4 张，每张最多 4 MiB，每次合计最多 12 MiB。较大的截图请先压缩。完整会话还设有图片总量检查，达到限制时请新建或清空对话。 + +本次未实现 ChatGPT / Claude 等网页登录模式的文件上传协议。使用这些模式时，图片入口会提示切换到受支持的 API。旧式文本 Completions 接口也不支持图片输入。 + +## 图片与存储 + +图片作为 base64 数据直接随消息发送给所选 API 服务商，不经过新增中转服务。会话保存沿用插件原有的本地存储机制，因此图片也会随会话保存；导出完整会话 JSON 时会包含图片。Markdown 文本导出仍只包含文字。清除草稿不会删除已经发送并保存在会话中的图片；请使用清空或删除会话。 + +## 重新构建 + +请先解压 **源码包**，安装 Node.js 22 或更新版本。在解压得到的 `chatgptbox-image-support` 文件夹中打开终端，确认该目录下存在 `package.json` 和 `build.mjs`。首次安装依赖需要网络连接。 + +```powershell +npm ci --ignore-scripts +npm test +npm run build +``` + +Windows PowerShell 若提示禁止运行 `npm.ps1`，可使用 `npm.cmd ci --ignore-scripts`、`npm.cmd test` 和 `npm.cmd run build`，无需修改系统执行策略。若提示找不到 `package.json`，通常是在安装包目录或源码包外层目录运行了命令。 + +构建成功后加载生成的 `build/chromium` 目录。源码包没有包含 `node_modules`，因此解压后应先执行安装依赖命令。维护者可运行 `node scripts/package-image-source.mjs` 重新生成源码 ZIP。 + +重新构建后，在扩展管理页点击开发版的“重新加载”，然后重新打开侧栏并刷新需要使用插件的网页。商店自动更新不会更新此开发版。 + +## 验证记录 + +2026-09-10 完成交付检查： + +- 初始图片功能实现阶段的 `npm test`：全部通过，包括图片大小与格式限制、多模态请求、图片历史及重试保留。 +- `npm run lint`：通过。 +- `npm run build`：通过，生成 Chromium / Firefox 构建及 ZIP。构建缓存出现快照警告，不影响生成安装包。 +- 实际加载开发版到独立的无头 Edge 浏览器，在 380 像素宽的 `IndependentPanel.html` 中验证文件选择、删除预览、粘贴事件、拖入事件、纯图片发送、重试、文字追问保留图片、清空草稿与不支持模式的拦截，全部通过。 +- 本地模拟服务确认请求包含 `image_url` 图片数据；未调用真实模型，没有使用个人账户。 + +浏览器脚本位于 `tests/manual/images-smoke.cjs`，运行前将 `CODEX_NODE_MODULES` 指向已安装 Playwright 的依赖目录。该脚本测试侧栏共用页面，未操作浏览器原生侧栏容器；粘贴与拖入通过浏览器事件模拟，未验证操作系统剪贴板。Chrome、Firefox 和真实服务商的图片理解效果未做实测。 + +源码归档脚本必须在 Git checkout 中运行，只收录 Git 已跟踪且通过过滤的文件。解压后的源码仍可正常安装依赖和构建，但不能直接重新运行该 Git 归档脚本。打包前应检查已跟踪源码不含秘密；文件名过滤不能代替内容审查。 diff --git a/README.md b/README.md index 6838625c1..e55ea1d4b 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,37 @@ English   |   [Indonesia](README_IN.md)   | & - You can also use [Ollama](https://github.com/ChatGPTBox-dev/chatGPTBox/issues/616#issuecomment-1975186467) / https://openrouter.ai/docs#models with ChatGPTBox's `Custom Model` mode +## Image support update + +This fork adds image input to ChatGPTBox conversations and improves the Chromium side panel on PDF pages. + +- Attach PNG, JPEG, WebP, or GIF files from the input box, or paste and drop images. +- Preview and remove images before sending; image-only messages are supported. +- Preserve images in conversation history, follow-up requests, and retries. +- Send multimodal `image_url` content through OpenAI-compatible Chat Completions APIs. +- Open the Edge/Chrome side panel when a PDF viewer context-menu callback omits its window ID. +- Keep the native side panel usable at narrow widths without clipping the attachment controls. + +Image input is enabled for OpenAI-compatible API modes. The selected model and API endpoint must support vision. Web-login modes and legacy text Completions endpoints do not support attachments in this fork. Limits are 4 images per message, 4 MiB per image, and 12 MiB total per message. + +To build and load the Chromium extension: + +```powershell +npm ci --ignore-scripts +npm test +npm run build +``` + +Open `edge://extensions` or `chrome://extensions`, enable Developer mode, choose **Load unpacked**, and select `build/chromium`. See [IMAGE-SUPPORT.zh-CN.md](IMAGE-SUPPORT.zh-CN.md) for detailed usage, privacy, troubleshooting, and validation notes. + +**Native side panel on a Blackboard-hosted PDF** + +![Native side panel processing attached PDF screenshots](screenshots/blackboard-pdf-native-side-panel.png) + +**Independent conversation window on the same PDF** + +![Independent conversation window processing attached PDF screenshots](screenshots/blackboard-pdf-independent-window.png) + ## ✨ Features - 🌈 Call up the chat dialog box on any page at any time. (Ctrl+B) diff --git a/README_ZH.md b/README_ZH.md index 8edde2deb..82c73f9c7 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -85,6 +85,37 @@ - 离线/自托管模型 现已支持, 在`自定义模型`模式下使用, 具体查看 [Ollama](https://github.com/ChatGPTBox-dev/chatGPTBox/issues/616#issuecomment-1975186467) / [RWKV-Runner](https://github.com/josStorer/RWKV-Runner), 你还可以部署wenda (https://github.com/wenda-LLM/wenda), 配合自定义模型模式使用, 从而调用各类本地模型, 参考 [#397](https://github.com/ChatGPTBox-dev/chatGPTBox/issues/397) 修改API URL +## 图片支持更新 + +此 fork 为 ChatGPTBox 对话增加了图片输入，并改善了 Chromium 浏览器在 PDF 页面中的原生侧栏行为。 + +- 可从输入框选择 PNG、JPEG、WebP 或 GIF，也支持粘贴截图和拖入图片。 +- 发送前可以预览和删除图片，并支持仅发送图片。 +- 后续提问、历史消息和重试会保留相应图片。 +- 通过 OpenAI 兼容的 Chat Completions API 发送多模态 `image_url` 内容。 +- Edge/Chrome 的 PDF 查看器即使没有向右键菜单回调提供窗口 ID，也可以打开侧栏。 +- 窄侧栏中图片按钮与输入区域保持可见，不会被裁切。 + +图片功能用于 OpenAI 兼容 API 模式，所选模型和 API 服务必须支持视觉输入。此 fork 暂不支持网页登录模式和旧式文本 Completions 接口的图片附件。限制为每次最多 4 张、每张最多 4 MiB、单次合计最多 12 MiB。 + +构建 Chromium 版本： + +```powershell +npm ci --ignore-scripts +npm test +npm run build +``` + +打开 `edge://extensions` 或 `chrome://extensions`，启用开发人员模式，选择“加载解压缩的扩展”，然后选择 `build/chromium`。详细用法、隐私说明、故障排查和验证记录见 [IMAGE-SUPPORT.zh-CN.md](IMAGE-SUPPORT.zh-CN.md)。 + +**Blackboard PDF 中的原生侧栏** + +![原生侧栏正在处理附加的 PDF 截图](screenshots/blackboard-pdf-native-side-panel.png) + +**同一 PDF 上的独立对话窗口** + +![独立对话窗口正在处理附加的 PDF 截图](screenshots/blackboard-pdf-independent-window.png) + ## ✨ Features - 🌈 在任何页面随时呼出聊天对话框 (Ctrl+B) diff --git a/screenshots/blackboard-pdf-independent-window.png b/screenshots/blackboard-pdf-independent-window.png new file mode 100644 index 000000000..9c8ddaa9e Binary files /dev/null and b/screenshots/blackboard-pdf-independent-window.png differ diff --git a/screenshots/blackboard-pdf-native-side-panel.png b/screenshots/blackboard-pdf-native-side-panel.png new file mode 100644 index 000000000..1d1a4361f Binary files /dev/null and b/screenshots/blackboard-pdf-native-side-panel.png differ diff --git a/scripts/package-image-source.mjs b/scripts/package-image-source.mjs new file mode 100644 index 000000000..67755b330 --- /dev/null +++ b/scripts/package-image-source.mjs @@ -0,0 +1,183 @@ +import archiver from 'archiver' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { constants, createWriteStream } from 'node:fs' +import { lstat, mkdir, open } from 'node:fs/promises' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +const defaultRoot = fileURLToPath(new URL('../', import.meta.url)) + +// Source archives are distributable artifacts. Keep their roots explicit so an +// unrelated local file at the repository root can never be packaged by accident. +export const SOURCE_ARCHIVE_ROOTS = Object.freeze([ + '.github', + 'badges', + 'safari', + 'screenshots', + 'scripts', + 'src', + 'tests', + '.eslintrc.json', + '.gitattributes', + '.gitignore', + '.nvmrc', + '.prettierignore', + '.prettierrc', + 'AGENTS.md', + 'build.mjs', + 'CURRENT_CHANGE.md', + 'IMAGE-SUPPORT.zh-CN.md', + 'LICENSE', + 'package-lock.json', + 'package.json', + 'README.md', + 'README_IN.md', + 'README_JA.md', + 'README_TR.md', + 'README_ZH.md', + 'SOURCE_CODE_REVIEW.md', +]) + +const excludedDirectoryNames = new Set([ + '.git', + '.cache', + '.coverage', + 'build', + 'coverage', + 'node_modules', + 'test-results', +]) + +const sensitiveFilePatterns = [ + /^\.env(?:\..*)?$/i, + /^\.npmrc$/i, + /^\.netrc$/i, + /^(?:token|api[-_]?key)(?:\..*)?$/i, + /^config\.json$/i, + /^auth(?:\..*)?$/i, + /^credentials?(?:\..*)?$/i, + /^id_(?:dsa|ecdsa|ed25519|rsa)(?:\..*)?$/i, + /^secrets?(?:\..*)?$/i, + /\.(?:cer|crt|der|jks|key|keystore|p12|pem|pfx)$/i, +] + +export function isSensitiveSourcePath(relativePath) { + const segments = relativePath.split(/[\\/]/).filter(Boolean) + if (segments.some((segment) => excludedDirectoryNames.has(segment))) return true + const basename = segments.at(-1) ?? '' + return sensitiveFilePatterns.some((pattern) => pattern.test(basename)) +} + +const execFileAsync = promisify(execFile) + +export async function collectSourceFiles(root = defaultRoot) { + // Require Git rather than falling back to a recursive walk in extracted ZIPs. + const { stdout } = await execFileAsync('git', ['ls-files', '--cached', '-z'], { + cwd: root, + maxBuffer: 16 * 1024 * 1024, + }) + const files = [] + for (const relativePath of new Set(stdout.split('\0').filter(Boolean))) { + if ( + !SOURCE_ARCHIVE_ROOTS.some( + (allowed) => relativePath === allowed || relativePath.startsWith(`${allowed}/`), + ) + ) + continue + if (isSensitiveSourcePath(relativePath)) continue + const segments = relativePath.split('/') + if (segments.some((part) => part === '..') || path.isAbsolute(relativePath)) continue + files.push(relativePath) + } + return files.sort() +} + +function isSameFileSnapshot(left, right) { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.size === right.size && + left.mtimeMs === right.mtimeMs && + left.ctimeMs === right.ctimeMs + ) +} + +export async function readVerifiedSourceFile(root, relativePath) { + const segments = relativePath.split('/') + if ( + segments.length === 0 || + segments.some((part) => !part || part === '..') || + path.isAbsolute(relativePath) + ) { + throw new Error(`Unsafe source path: ${relativePath}`) + } + + let expectedStat + for (let i = 1; i <= segments.length; i++) { + const candidateStat = await lstat(path.join(root, ...segments.slice(0, i))) + if (candidateStat.isSymbolicLink()) { + throw new Error(`Source archive path contains a symbolic link: ${relativePath}`) + } + if (i === segments.length) expectedStat = candidateStat + } + if (!expectedStat?.isFile()) + throw new Error(`Source archive entry is not a file: ${relativePath}`) + + const absolutePath = path.join(root, ...segments) + const noFollowFlag = constants.O_NOFOLLOW ?? 0 + const fileHandle = await open(absolutePath, constants.O_RDONLY | noFollowFlag) + try { + const openedStat = await fileHandle.stat() + if (!openedStat.isFile() || !isSameFileSnapshot(expectedStat, openedStat)) { + throw new Error(`Source archive entry changed while opening: ${relativePath}`) + } + + const contents = await fileHandle.readFile() + const completedStat = await fileHandle.stat() + if (contents.byteLength !== openedStat.size || !isSameFileSnapshot(openedStat, completedStat)) { + throw new Error(`Source archive entry changed while reading: ${relativePath}`) + } + return contents + } finally { + await fileHandle.close() + } +} + +export async function createImageSourceArchive(root = defaultRoot) { + const files = await collectSourceFiles(root) + const sourceEntries = [] + for (const relativePath of files) { + sourceEntries.push({ + relativePath, + contents: await readVerifiedSourceFile(root, relativePath), + }) + } + const outputDirectory = path.join(root, 'build') + await mkdir(outputDirectory, { recursive: true }) + const outputPath = path.join(outputDirectory, 'chatgptbox-image-support-source.zip') + const output = createWriteStream(outputPath) + const archive = archiver('zip', { zlib: { level: 9 } }) + const completed = new Promise((resolve, reject) => { + output.on('close', resolve) + output.on('error', reject) + archive.on('error', reject) + archive.on('warning', reject) + }) + archive.pipe(output) + + for (const entry of sourceEntries) { + archive.append(entry.contents, { + name: `chatgptbox-image-support/${entry.relativePath.replaceAll('\\', '/')}`, + }) + } + await archive.finalize() + await completed + console.log(`Created ${outputPath} (${archive.pointer()} bytes)`) + return outputPath +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await createImageSourceArchive() +} diff --git a/src/_locales/de/main.json b/src/_locales/de/main.json index 8b0d204bc..cdb9c33f5 100644 --- a/src/_locales/de/main.json +++ b/src/_locales/de/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Nur benutzerdefinierten Website-Regex verwenden, um Website-Übereinstimmungen zu finden und interne Regeln ignorieren", "Input Query": "Eingabeaufforderung", "Resize input box": "Größe des Eingabefelds ändern", + "Images require an OpenAI-compatible vision API.": "Bilder erfordern eine OpenAI-kompatible Vision-API.", + "Choose a model that supports image input.": "Wählen Sie ein Modell, das Bildeingaben unterstützt.", + "Attach images": "Bilder anhängen", + "Drop images here or paste a screenshot": "Bilder hierher ziehen oder einen Screenshot einfügen", + "Attached images": "Angehängte Bilder", + "Image": "Bild", + "Reading image": "Bild wird gelesen", + "Remove image": "Bild entfernen", + "Images must be PNG, JPEG, WEBP, or GIF.": "Bilder müssen PNG, JPEG, WEBP oder GIF sein.", + "Each image must be 4 MiB or smaller.": "Jedes Bild darf höchstens 4 MiB groß sein.", + "You can attach up to 4 images.": "Sie können bis zu 4 Bilder anhängen.", + "Total image size must be 12 MiB or smaller.": "Die Gesamtgröße der Bilder darf höchstens 12 MiB betragen.", + "This file is not a supported image.": "Diese Datei ist kein unterstütztes Bild.", + "Unable to read image.": "Bild kann nicht gelesen werden.", + "Unable to send images.": "Bilder können nicht gesendet werden.", + "Describe these images": "Beschreiben Sie diese Bilder", "Append Query": "Am Ende anhängen", "Prepend Query": "Am Anfang hinzufügen", "Wechat Pay": "WeChat-Pay", diff --git a/src/_locales/en/main.json b/src/_locales/en/main.json index 445148a56..e3e6bde5a 100644 --- a/src/_locales/en/main.json +++ b/src/_locales/en/main.json @@ -26,6 +26,22 @@ "Wechat Pay": "Wechat Pay", "Type your question here\nEnter to send, shift + enter to break line": "Type your question here\nEnter to send\nShift + enter to break line", "Type your question here\nEnter to stop generating\nShift + enter to break line": "Type your question here\nEnter to stop generating\nShift + enter to break line", + "Images require an OpenAI-compatible vision API.": "Images require an OpenAI-compatible vision API.", + "Choose a model that supports image input.": "Choose a model that supports image input.", + "Attach images": "Attach images", + "Drop images here or paste a screenshot": "Drop images here or paste a screenshot", + "Attached images": "Attached images", + "Image": "Image", + "Reading image": "Reading image", + "Remove image": "Remove image", + "Images must be PNG, JPEG, WEBP, or GIF.": "Images must be PNG, JPEG, WEBP, or GIF.", + "Each image must be 4 MiB or smaller.": "Each image must be 4 MiB or smaller.", + "You can attach up to 4 images.": "You can attach up to 4 images.", + "Total image size must be 12 MiB or smaller.": "Total image size must be 12 MiB or smaller.", + "This file is not a supported image.": "This file is not a supported image.", + "Unable to read image.": "Unable to read image.", + "Unable to send images.": "Unable to send images.", + "Describe these images": "Describe these images", "Ask ChatGPT": "Ask ChatGPT", "No Input Found": "No Input Found", "You": "You", diff --git a/src/_locales/es/main.json b/src/_locales/es/main.json index c9f5b5a11..56289833a 100644 --- a/src/_locales/es/main.json +++ b/src/_locales/es/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Utilice exclusivamente expesiones regulares personalizadas para la coincidencia de sitios web, ignorando las reglas integradas", "Input Query": "Consulta de entrada", "Resize input box": "Cambiar el tamaño del cuadro de entrada", + "Images require an OpenAI-compatible vision API.": "Las imágenes requieren una API de visión compatible con OpenAI.", + "Choose a model that supports image input.": "Elige un modelo que admita entrada de imágenes.", + "Attach images": "Adjuntar imágenes", + "Drop images here or paste a screenshot": "Arrastra imágenes aquí o pega una captura de pantalla", + "Attached images": "Imágenes adjuntas", + "Image": "Imagen", + "Reading image": "Leyendo imagen", + "Remove image": "Eliminar imagen", + "Images must be PNG, JPEG, WEBP, or GIF.": "Las imágenes deben ser PNG, JPEG, WEBP o GIF.", + "Each image must be 4 MiB or smaller.": "Cada imagen debe tener un tamaño máximo de 4 MiB.", + "You can attach up to 4 images.": "Puedes adjuntar hasta 4 imágenes.", + "Total image size must be 12 MiB or smaller.": "El tamaño total de las imágenes debe ser de 12 MiB o menos.", + "This file is not a supported image.": "Este archivo no es una imagen compatible.", + "Unable to read image.": "No se puede leer la imagen.", + "Unable to send images.": "No se pueden enviar las imágenes.", + "Describe these images": "Describe estas imágenes", "Append Query": "Añadir consulta", "Prepend Query": "Insertar consulta", "Wechat Pay": "Pago de Wechat", diff --git a/src/_locales/fr/main.json b/src/_locales/fr/main.json index 1fa86e8a3..c9216948a 100644 --- a/src/_locales/fr/main.json +++ b/src/_locales/fr/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Utiliser exclusivement l'expression régulière de site personnalisée pour la correspondance de site Web, en ignorant les règles intégrées", "Input Query": "Sélecteur d'entrée", "Resize input box": "Redimensionner le champ de saisie", + "Images require an OpenAI-compatible vision API.": "Les images nécessitent une API de vision compatible avec OpenAI.", + "Choose a model that supports image input.": "Choisissez un modèle prenant en charge les images.", + "Attach images": "Joindre des images", + "Drop images here or paste a screenshot": "Déposez des images ici ou collez une capture d’écran", + "Attached images": "Images jointes", + "Image": "Image", + "Reading image": "Lecture de l’image", + "Remove image": "Supprimer l’image", + "Images must be PNG, JPEG, WEBP, or GIF.": "Les images doivent être au format PNG, JPEG, WEBP ou GIF.", + "Each image must be 4 MiB or smaller.": "Chaque image doit faire 4 Mio au maximum.", + "You can attach up to 4 images.": "Vous pouvez joindre jusqu’à 4 images.", + "Total image size must be 12 MiB or smaller.": "La taille totale des images doit être de 12 Mio au maximum.", + "This file is not a supported image.": "Ce fichier n’est pas une image prise en charge.", + "Unable to read image.": "Impossible de lire l’image.", + "Unable to send images.": "Impossible d’envoyer les images.", + "Describe these images": "Décrire ces images", "Append Query": "Sélecteur à ajouter", "Prepend Query": "Sélecteur à insérer", "Wechat Pay": "Paiement Wechat", diff --git a/src/_locales/id/main.json b/src/_locales/id/main.json index 5267098d2..79b2a4cd3 100644 --- a/src/_locales/id/main.json +++ b/src/_locales/id/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Gunakan secara eksklusif Regex Situs Kustom untuk pencocokan situs web, mengabaikan aturan bawaan", "Input Query": "Query Masukan", "Resize input box": "Ubah ukuran kotak input", + "Images require an OpenAI-compatible vision API.": "Gambar memerlukan API visi yang kompatibel dengan OpenAI.", + "Choose a model that supports image input.": "Pilih model yang mendukung input gambar.", + "Attach images": "Lampirkan gambar", + "Drop images here or paste a screenshot": "Tarik gambar ke sini atau tempel tangkapan layar", + "Attached images": "Gambar terlampir", + "Image": "Gambar", + "Reading image": "Membaca gambar", + "Remove image": "Hapus gambar", + "Images must be PNG, JPEG, WEBP, or GIF.": "Gambar harus berformat PNG, JPEG, WEBP, atau GIF.", + "Each image must be 4 MiB or smaller.": "Setiap gambar harus berukuran 4 MiB atau kurang.", + "You can attach up to 4 images.": "Anda dapat melampirkan hingga 4 gambar.", + "Total image size must be 12 MiB or smaller.": "Ukuran total gambar harus 12 MiB atau kurang.", + "This file is not a supported image.": "File ini bukan gambar yang didukung.", + "Unable to read image.": "Tidak dapat membaca gambar.", + "Unable to send images.": "Tidak dapat mengirim gambar.", + "Describe these images": "Jelaskan gambar-gambar ini", "Append Query": "Tambahkan Query", "Prepend Query": "Tambahkan Query di Awal", "Wechat Pay": "Pembayaran Wechat", diff --git a/src/_locales/it/main.json b/src/_locales/it/main.json index 4a592ff01..dd2c9e1b7 100644 --- a/src/_locales/it/main.json +++ b/src/_locales/it/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Usa esclusivamente l'espressione regolare del sito personalizzata per la corrispondenza dei siti Web, ignorando le regole incorporate", "Input Query": "Query di input", "Resize input box": "Ridimensiona la casella di input", + "Images require an OpenAI-compatible vision API.": "Le immagini richiedono un’API di visione compatibile con OpenAI.", + "Choose a model that supports image input.": "Scegli un modello che supporti l’input di immagini.", + "Attach images": "Allega immagini", + "Drop images here or paste a screenshot": "Trascina qui le immagini o incolla uno screenshot", + "Attached images": "Immagini allegate", + "Image": "Immagine", + "Reading image": "Lettura dell’immagine", + "Remove image": "Rimuovi immagine", + "Images must be PNG, JPEG, WEBP, or GIF.": "Le immagini devono essere PNG, JPEG, WEBP o GIF.", + "Each image must be 4 MiB or smaller.": "Ogni immagine deve essere di massimo 4 MiB.", + "You can attach up to 4 images.": "Puoi allegare fino a 4 immagini.", + "Total image size must be 12 MiB or smaller.": "La dimensione totale delle immagini deve essere di massimo 12 MiB.", + "This file is not a supported image.": "Questo file non è un’immagine supportata.", + "Unable to read image.": "Impossibile leggere l’immagine.", + "Unable to send images.": "Impossibile inviare le immagini.", + "Describe these images": "Descrivi queste immagini", "Append Query": "Query di appendice", "Prepend Query": "Query di aggiunta in testa", "Wechat Pay": "Paga con Wechat", diff --git a/src/_locales/ja/main.json b/src/_locales/ja/main.json index 91c960949..394337cc2 100644 --- a/src/_locales/ja/main.json +++ b/src/_locales/ja/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "内蔵ルールを無視して、カスタムサイト用の正規表現のみを使用", "Input Query": "入力クエリ", "Resize input box": "入力欄のサイズを変更", + "Images require an OpenAI-compatible vision API.": "画像を使用するには、OpenAI 互換のビジョン API が必要です。", + "Choose a model that supports image input.": "画像入力に対応したモデルを選択してください。", + "Attach images": "画像を添付", + "Drop images here or paste a screenshot": "ここに画像をドロップするか、スクリーンショットを貼り付けてください", + "Attached images": "添付画像", + "Image": "画像", + "Reading image": "画像を読み込み中", + "Remove image": "画像を削除", + "Images must be PNG, JPEG, WEBP, or GIF.": "画像は PNG、JPEG、WEBP、GIF のいずれかである必要があります。", + "Each image must be 4 MiB or smaller.": "各画像は 4 MiB 以下である必要があります。", + "You can attach up to 4 images.": "最大 4 枚の画像を添付できます。", + "Total image size must be 12 MiB or smaller.": "画像の合計サイズは 12 MiB 以下である必要があります。", + "This file is not a supported image.": "このファイルは対応している画像ではありません。", + "Unable to read image.": "画像を読み込めません。", + "Unable to send images.": "画像を送信できません。", + "Describe these images": "これらの画像について説明してください", "Append Query": "末尾に追加するクエリ", "Prepend Query": "先頭に挿入するクエリ", "Wechat Pay": "Wechatペイ", diff --git a/src/_locales/ko/main.json b/src/_locales/ko/main.json index 252ed15f8..00b7917ad 100644 --- a/src/_locales/ko/main.json +++ b/src/_locales/ko/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "사이트 일치에 독점적으로 사용자 정의 사이트 Regex를 사용하며, 내장 규칙을 무시합니다.", "Input Query": "입력 쿼리 선택기", "Resize input box": "입력 상자 크기 조절", + "Images require an OpenAI-compatible vision API.": "이미지에는 OpenAI 호환 비전 API가 필요합니다.", + "Choose a model that supports image input.": "이미지 입력을 지원하는 모델을 선택하세요.", + "Attach images": "이미지 첨부", + "Drop images here or paste a screenshot": "이미지를 여기에 놓거나 스크린샷을 붙여넣으세요", + "Attached images": "첨부된 이미지", + "Image": "이미지", + "Reading image": "이미지 읽는 중", + "Remove image": "이미지 제거", + "Images must be PNG, JPEG, WEBP, or GIF.": "이미지는 PNG, JPEG, WEBP 또는 GIF 형식이어야 합니다.", + "Each image must be 4 MiB or smaller.": "각 이미지는 4 MiB 이하여야 합니다.", + "You can attach up to 4 images.": "이미지는 최대 4개까지 첨부할 수 있습니다.", + "Total image size must be 12 MiB or smaller.": "이미지 전체 크기는 12 MiB 이하여야 합니다.", + "This file is not a supported image.": "이 파일은 지원되는 이미지가 아닙니다.", + "Unable to read image.": "이미지를 읽을 수 없습니다.", + "Unable to send images.": "이미지를 보낼 수 없습니다.", + "Describe these images": "이 이미지들을 설명해 주세요", "Append Query": "끝에 추가할 쿼리 선택기", "Prepend Query": "앞에 추가할 쿼리 선택기", "Wechat Pay": "위챗 페이", diff --git a/src/_locales/pt/main.json b/src/_locales/pt/main.json index b3379c159..2887593b8 100644 --- a/src/_locales/pt/main.json +++ b/src/_locales/pt/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Usar exclusivamente a Expressão Regular do Site Personalizada para combinação de sites, ignorando regras incorporadas", "Input Query": "Consulta de Entrada", "Resize input box": "Redimensionar a caixa de entrada", + "Images require an OpenAI-compatible vision API.": "As imagens requerem uma API de visão compatível com OpenAI.", + "Choose a model that supports image input.": "Escolha um modelo que aceite entrada de imagens.", + "Attach images": "Anexar imagens", + "Drop images here or paste a screenshot": "Arraste imagens para aqui ou cole uma captura de tela", + "Attached images": "Imagens anexadas", + "Image": "Imagem", + "Reading image": "Lendo imagem", + "Remove image": "Remover imagem", + "Images must be PNG, JPEG, WEBP, or GIF.": "As imagens devem ser PNG, JPEG, WEBP ou GIF.", + "Each image must be 4 MiB or smaller.": "Cada imagem deve ter no máximo 4 MiB.", + "You can attach up to 4 images.": "Você pode anexar até 4 imagens.", + "Total image size must be 12 MiB or smaller.": "O tamanho total das imagens deve ser de no máximo 12 MiB.", + "This file is not a supported image.": "Este arquivo não é uma imagem compatível.", + "Unable to read image.": "Não foi possível ler a imagem.", + "Unable to send images.": "Não foi possível enviar as imagens.", + "Describe these images": "Descreva estas imagens", "Append Query": "Consulta Anexada", "Prepend Query": "Consulta Prependida", "Wechat Pay": "Pagamento Wechat", diff --git a/src/_locales/ru/main.json b/src/_locales/ru/main.json index d857f3f62..bf1ce132e 100644 --- a/src/_locales/ru/main.json +++ b/src/_locales/ru/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Использовать только пользовательское регулярное выражение сайта для сопоставления сайта, игнорируя встроенные правила.", "Input Query": "Входной запрос", "Resize input box": "Изменить размер поля ввода", + "Images require an OpenAI-compatible vision API.": "Для изображений требуется API компьютерного зрения, совместимый с OpenAI.", + "Choose a model that supports image input.": "Выберите модель, поддерживающую ввод изображений.", + "Attach images": "Прикрепить изображения", + "Drop images here or paste a screenshot": "Перетащите изображения сюда или вставьте снимок экрана", + "Attached images": "Прикреплённые изображения", + "Image": "Изображение", + "Reading image": "Чтение изображения", + "Remove image": "Удалить изображение", + "Images must be PNG, JPEG, WEBP, or GIF.": "Изображения должны иметь формат PNG, JPEG, WEBP или GIF.", + "Each image must be 4 MiB or smaller.": "Размер каждого изображения не должен превышать 4 МиБ.", + "You can attach up to 4 images.": "Можно прикрепить до 4 изображений.", + "Total image size must be 12 MiB or smaller.": "Общий размер изображений не должен превышать 12 МиБ.", + "This file is not a supported image.": "Этот файл не является поддерживаемым изображением.", + "Unable to read image.": "Не удалось прочитать изображение.", + "Unable to send images.": "Не удалось отправить изображения.", + "Describe these images": "Опишите эти изображения", "Append Query": "Добавить запрос", "Prepend Query": "Вставить запрос", "Wechat Pay": "Wechat-платеж", diff --git a/src/_locales/tr/main.json b/src/_locales/tr/main.json index 1b2391cf4..111807220 100644 --- a/src/_locales/tr/main.json +++ b/src/_locales/tr/main.json @@ -21,6 +21,22 @@ "Exclusively use Custom Site Regex for website matching, ignoring built-in rules": "Yerleşik kuralları yok sayarak web sitesi eşleştirme için yalnızca Özel Site Regex'i kullan", "Input Query": "Girdi Sorgusu", "Resize input box": "Giriş kutusunu yeniden boyutlandır", + "Images require an OpenAI-compatible vision API.": "Görseller için OpenAI uyumlu bir görsel API gerekir.", + "Choose a model that supports image input.": "Görsel girişini destekleyen bir model seçin.", + "Attach images": "Görsel ekle", + "Drop images here or paste a screenshot": "Görselleri buraya sürükleyin veya ekran görüntüsü yapıştırın", + "Attached images": "Eklenen görseller", + "Image": "Görsel", + "Reading image": "Görsel okunuyor", + "Remove image": "Görseli kaldır", + "Images must be PNG, JPEG, WEBP, or GIF.": "Görseller PNG, JPEG, WEBP veya GIF biçiminde olmalıdır.", + "Each image must be 4 MiB or smaller.": "Her görsel en fazla 4 MiB olmalıdır.", + "You can attach up to 4 images.": "En fazla 4 görsel ekleyebilirsiniz.", + "Total image size must be 12 MiB or smaller.": "Görsellerin toplam boyutu en fazla 12 MiB olmalıdır.", + "This file is not a supported image.": "Bu dosya desteklenen bir görsel değil.", + "Unable to read image.": "Görsel okunamadı.", + "Unable to send images.": "Görseller gönderilemedi.", + "Describe these images": "Bu görselleri açıkla", "Append Query": "Sorgu Ekle", "Prepend Query": "Sorgu Ön Eki", "Wechat Pay": "Wechat Pay", diff --git a/src/_locales/zh-hans/main.json b/src/_locales/zh-hans/main.json index cfad6d2d1..5d89b21fe 100644 --- a/src/_locales/zh-hans/main.json +++ b/src/_locales/zh-hans/main.json @@ -26,6 +26,22 @@ "Wechat Pay": "微信打赏", "Type your question here\nEnter to send, shift + enter to break line": "在此输入你的问题\n回车 发送\nshift+回车 换行", "Type your question here\nEnter to stop generating\nShift + enter to break line": "在此输入你的问题\n回车 停止生成\nshift+回车 换行", + "Images require an OpenAI-compatible vision API.": "图片需要兼容 OpenAI 的视觉 API。", + "Choose a model that supports image input.": "请选择支持图像输入的模型。", + "Attach images": "附加图片", + "Drop images here or paste a screenshot": "将图片拖到这里，或粘贴屏幕截图", + "Attached images": "已附加图片", + "Image": "图片", + "Reading image": "正在读取图片", + "Remove image": "移除图片", + "Images must be PNG, JPEG, WEBP, or GIF.": "图片必须是 PNG、JPEG、WEBP 或 GIF 格式。", + "Each image must be 4 MiB or smaller.": "每张图片必须不超过 4 MiB。", + "You can attach up to 4 images.": "最多可附加 4 张图片。", + "Total image size must be 12 MiB or smaller.": "图片总大小必须不超过 12 MiB。", + "This file is not a supported image.": "此文件不是受支持的图片。", + "Unable to read image.": "无法读取图片。", + "Unable to send images.": "无法发送图片。", + "Describe these images": "描述这些图片", "Ask ChatGPT": "询问ChatGPT", "No Input Found": "无输入", "You": "你", diff --git a/src/_locales/zh-hant/main.json b/src/_locales/zh-hant/main.json index 3dd20a229..2e668a37f 100644 --- a/src/_locales/zh-hant/main.json +++ b/src/_locales/zh-hant/main.json @@ -26,6 +26,22 @@ "Wechat Pay": "微信支付贊助", "Type your question here\nEnter to send, shift + enter to break line": "在此輸入你的問題\n按 Enter 傳送，Shift + Enter 換行", "Type your question here\nEnter to stop generating\nShift + enter to break line": "在此輸入你的問題\n按 Enter 停止產生，Shift + Enter 換行", + "Images require an OpenAI-compatible vision API.": "圖片需要相容 OpenAI 的視覺 API。", + "Choose a model that supports image input.": "請選擇支援圖片輸入的模型。", + "Attach images": "附加圖片", + "Drop images here or paste a screenshot": "將圖片拖曳到這裡，或貼上螢幕截圖", + "Attached images": "已附加圖片", + "Image": "圖片", + "Reading image": "正在讀取圖片", + "Remove image": "移除圖片", + "Images must be PNG, JPEG, WEBP, or GIF.": "圖片必須是 PNG、JPEG、WEBP 或 GIF 格式。", + "Each image must be 4 MiB or smaller.": "每張圖片必須不超過 4 MiB。", + "You can attach up to 4 images.": "最多可附加 4 張圖片。", + "Total image size must be 12 MiB or smaller.": "圖片總大小必須不超過 12 MiB。", + "This file is not a supported image.": "此檔案不是支援的圖片。", + "Unable to read image.": "無法讀取圖片。", + "Unable to send images.": "無法傳送圖片。", + "Describe these images": "描述這些圖片", "Ask ChatGPT": "詢問 ChatGPT", "No Input Found": "找不到輸入內容", "You": "你", diff --git a/src/background/index.mjs b/src/background/index.mjs index 79b4f0c0d..6dd4cf181 100644 --- a/src/background/index.mjs +++ b/src/background/index.mjs @@ -945,77 +945,6 @@ try { }, ['requestHeaders', ...(Browser.runtime.getManifest().manifest_version < 3 ? ['blocking'] : [])], ) - - Browser.tabs.onUpdated.addListener(async (tabId, info, tab) => { - const outerTryCatchError = (error) => { - console.error( - '[background] Error in tabs.onUpdated listener callback (outer):', - error, - tabId, - info, - ) - } - try { - if (!tab.url) { - console.debug( - `[background] Skipping side panel update for tabId: ${tabId}. Tab URL: ${tab.url}, Info Status: ${info.status}`, - ) - return - } - console.debug( - `[background] tabs.onUpdated event for tabId: ${tabId}, status: ${info.status}, url: ${tab.url}. Proceeding with side panel update.`, - ) - - let sidePanelSet = false - try { - if (Browser.sidePanel && typeof Browser.sidePanel.setOptions === 'function') { - await Browser.sidePanel.setOptions({ - tabId, - path: 'IndependentPanel.html', - enabled: true, - }) - console.debug( - `[background] Side panel options set for tab ${tabId} using Browser.sidePanel`, - ) - sidePanelSet = true - } - } catch (browserError) { - console.warn('[background] Browser.sidePanel.setOptions failed:', browserError.message) - } - - if (!sidePanelSet) { - console.debug('[background] Attempting chrome.sidePanel.setOptions as fallback.') - const chromeApi = globalThis.chrome - if (chromeApi?.sidePanel && typeof chromeApi.sidePanel.setOptions === 'function') { - try { - await chromeApi.sidePanel.setOptions({ - tabId, - path: 'IndependentPanel.html', - enabled: true, - }) - console.debug( - `[background] Side panel options set for tab ${tabId} using chrome.sidePanel`, - ) - sidePanelSet = true - } catch (chromeError) { - console.error( - '[background] chrome.sidePanel.setOptions also failed:', - chromeError.message, - ) - } - } - } - - if (!sidePanelSet) { - console.warn( - '[background] SidePanel API (Browser.sidePanel or chrome.sidePanel) not available or setOptions failed in this browser. Side panel options not set for tab:', - tabId, - ) - } - } catch (error) { - outerTryCatchError(error) - } - }) } catch (error) { console.error('[background] Error setting up webRequest or tabs listeners:', error) } diff --git a/src/background/menus.mjs b/src/background/menus.mjs index d3a61ee1e..6abaa1d9d 100644 --- a/src/background/menus.mjs +++ b/src/background/menus.mjs @@ -108,45 +108,49 @@ const onClickMenu = (info, tab) => { }) } export function refreshMenu() { - if (Browser.contextMenus.onClicked.hasListener(onClickMenu)) - Browser.contextMenus.onClicked.removeListener(onClickMenu) - Browser.contextMenus.removeAll().then(async () => { - if ((await getUserConfig()).hideContextMenu) return - - await getPreferredLanguageKey().then((lang) => { - changeLanguage(lang) - }) - Browser.contextMenus.create({ - id: menuId, - title: 'ChatGPTBox', - contexts: ['all'], - }) + // A service worker may be starting because this very menu was clicked. + // Subscribe before storage/localization/menu creation can yield, otherwise + // the waking click can be delivered before its listener exists. + if (!Browser.contextMenus.onClicked.hasListener(onClickMenu)) + Browser.contextMenus.onClicked.addListener(onClickMenu) + return Browser.contextMenus + .removeAll() + .then(async () => { + if ((await getUserConfig()).hideContextMenu) return - for (const [k, v] of Object.entries(menuConfig)) { + await getPreferredLanguageKey().then((lang) => { + changeLanguage(lang) + }) Browser.contextMenus.create({ - id: menuId + k, - parentId: menuId, - title: t(v.label), + id: menuId, + title: 'ChatGPTBox', contexts: ['all'], }) - } - Browser.contextMenus.create({ - id: menuId + 'separator1', - parentId: menuId, - contexts: ['selection'], - type: 'separator', - }) - for (const index in defaultConfig.selectionTools) { - const key = defaultConfig.selectionTools[index] - const desc = defaultConfig.selectionToolsDesc[index] + + for (const [k, v] of Object.entries(menuConfig)) { + Browser.contextMenus.create({ + id: menuId + k, + parentId: menuId, + title: t(v.label), + contexts: ['all'], + }) + } Browser.contextMenus.create({ - id: menuId + key, + id: menuId + 'separator1', parentId: menuId, - title: t(desc), contexts: ['selection'], + type: 'separator', }) - } - - Browser.contextMenus.onClicked.addListener(onClickMenu) - }) + for (const index in defaultConfig.selectionTools) { + const key = defaultConfig.selectionTools[index] + const desc = defaultConfig.selectionToolsDesc[index] + Browser.contextMenus.create({ + id: menuId + key, + parentId: menuId, + title: t(desc), + contexts: ['selection'], + }) + } + }) + .catch((error) => console.error('failed to refresh context menus', error)) } diff --git a/src/background/redact.mjs b/src/background/redact.mjs index 3a0d027ce..06a411258 100644 --- a/src/background/redact.mjs +++ b/src/background/redact.mjs @@ -7,6 +7,7 @@ const SENSITIVE_KEYWORDS = [ 'credential', 'jwt', 'session', + 'images', ] export function isPromptOrSelectionLikeKey(lowerKey) { diff --git a/src/components/ConversationCard/index.jsx b/src/components/ConversationCard/index.jsx index 6835faa6a..a181a8b91 100644 --- a/src/components/ConversationCard/index.jsx +++ b/src/components/ConversationCard/index.jsx @@ -41,6 +41,7 @@ import { getConversationAiName, } from '../../popup/sections/api-modes-provider-utils.mjs' import { getDisplayErrorText } from '../../utils/error-text.mjs' +import { canSendImages, validateSessionImages } from '../../services/apis/images.mjs' import { createConversationPortMessage, createRetrySession, @@ -60,11 +61,12 @@ class ConversationItemData extends Object { * @param {string} content * @param {bool} done */ - constructor(type, content, done = false) { + constructor(type, content, done = false, images = []) { super() this.type = type this.content = content this.done = done + this.images = images } } @@ -84,6 +86,7 @@ function ConversationCard(props) { const [completeDraggable, setCompleteDraggable] = useState(false) const useForegroundFetch = isUsingBingWebModel(session) const [apiModes, setApiModes] = useState([]) + const [inputResetKey, setInputResetKey] = useState(0) /** * @type {[ConversationItemData[], (conversationItemData: ConversationItemData[]) => void]} @@ -120,7 +123,7 @@ function ConversationCard(props) { } else { const ret = [] for (const record of session.conversationRecords) { - ret.push(new ConversationItemData('question', record.question, true)) + ret.push(new ConversationItemData('question', record.question, true, record.images)) ret.push(new ConversationItemData('answer', record.answer, true)) } setConversationItemData(ret) @@ -151,7 +154,7 @@ function ConversationCard(props) { useEffect(async () => { // when the page is responsive, session may accumulate redundant data and needs to be cleared after remounting and before making a new request if (props.question && triggered) { - const newSession = initSession({ ...session, question: props.question }) + const newSession = initSession({ ...session, question: props.question, images: [] }) partialAnswerRef.current = '' retryRecordRef.current = null setSession(newSession) @@ -386,6 +389,19 @@ function ConversationCard(props) { }, [port, conversationItemData]) const getRetryFn = (session) => async () => { + try { + validateSessionImages(session) + if ( + (session.images?.length || + session.conversationRecords.some((record) => record.images?.length)) && + !canSendImages(config, session) + ) { + throw new Error(t('Images require an OpenAI-compatible vision API.')) + } + } catch (error) { + updateAnswer(error.message, false, 'error') + return + } updateAnswer(`

${t('Waiting for response...')}

`, false, 'answer') setIsReady(false) @@ -581,9 +597,11 @@ function ConversationCard(props) { }, }) setConversationItemData([]) + setInputResetKey((key) => key + 1) const newSession = initSession({ ...session, question: null, + images: [], conversationRecords: [], }) newSession.sessionId = session.sessionId @@ -661,6 +679,7 @@ function ConversationCard(props) { {conversationItemData.map((data, idx) => ( ) : ( { - const newQuestion = new ConversationItemData('question', question) + onSubmit={async (question, images = []) => { + const newSession = { ...session, question, images, isRetry: false } + validateSessionImages(newSession) + if ( + (images.length || + session.conversationRecords.some((record) => record.images?.length)) && + !canSendImages(config, newSession) + ) { + throw new Error(t('Images require an OpenAI-compatible vision API.')) + } + const newQuestion = new ConversationItemData('question', question, false, images) const newAnswer = new ConversationItemData( 'answer', `

${t('Waiting for response...')}

`, @@ -703,12 +733,13 @@ function ConversationCard(props) { setConversationItemData([...conversationItemData, newQuestion, newAnswer]) setIsReady(false) - const newSession = { ...session, question, isRetry: false } setSession(newSession) try { await postMessage({ session: newSession }) } catch (e) { updateAnswer(e, false, 'error') + setIsReady(true) + throw e } bodyRef.current.scrollTo({ top: bodyRef.current.scrollHeight, diff --git a/src/components/ConversationCard/session.mjs b/src/components/ConversationCard/session.mjs index 40c60f166..4e2cfee9a 100644 --- a/src/components/ConversationCard/session.mjs +++ b/src/components/ConversationCard/session.mjs @@ -56,6 +56,7 @@ export function createRetrySession(session, conversationRecords, retryRecord) { return { ...session, conversationRecords, + ...(retryRecord ? { images: retryRecord.images || [] } : {}), isRetry: retryRecord === null, } } diff --git a/src/components/ConversationItem/index.jsx b/src/components/ConversationItem/index.jsx index d9e9e9f35..e590abc22 100644 --- a/src/components/ConversationItem/index.jsx +++ b/src/components/ConversationItem/index.jsx @@ -16,7 +16,7 @@ AnswerTitle.propTypes = { descName: PropTypes.string, } -export function ConversationItem({ type, content, descName, onRetry }) { +export function ConversationItem({ type, content, descName, onRetry, images = [] }) { const { t } = useTranslation() const [collapsed, setCollapsed] = useState(false) @@ -49,6 +49,21 @@ export function ConversationItem({ type, content, descName, onRetry }) { {!collapsed && {content}} + {!collapsed && + (Array.isArray(images) ? images : []) + .filter( + (image) => + typeof image === 'string' && + /^data:image\/(png|jpeg|webp|gif);base64,[A-Za-z0-9+/]+=*$/.test(image), + ) + .map((image, index) => ( + {`${t('Image')} + ))} ) case 'answer': @@ -130,6 +145,7 @@ ConversationItem.propTypes = { content: PropTypes.string.isRequired, descName: PropTypes.string, onRetry: PropTypes.func, + images: PropTypes.arrayOf(PropTypes.string), } export default memo(ConversationItem) diff --git a/src/components/InputBox/images.mjs b/src/components/InputBox/images.mjs new file mode 100644 index 000000000..4cacd8e20 --- /dev/null +++ b/src/components/InputBox/images.mjs @@ -0,0 +1,124 @@ +import { + IMAGE_MIME_TYPES, + MAX_IMAGE_COUNT, + MAX_IMAGE_BYTES as MAX_IMAGE_SIZE, + MAX_TOTAL_IMAGE_BYTES as MAX_TOTAL_IMAGE_SIZE, +} from '../../utils/image-limits.mjs' +export { IMAGE_MIME_TYPES, MAX_IMAGE_COUNT, MAX_IMAGE_SIZE, MAX_TOTAL_IMAGE_SIZE } +export const IMAGE_ACCEPT = IMAGE_MIME_TYPES.join(',') + +export const IMAGE_FILE_ERROR = Object.freeze({ + INVALID: 'invalid', + TYPE: 'type', + FILE_SIZE: 'file-size', + COUNT: 'count', + TOTAL_SIZE: 'total-size', +}) + +export function hasDraggedFiles(dataTransfer) { + if (Array.from(dataTransfer?.files ?? []).length > 0) return true + return Array.from(dataTransfer?.types ?? []).includes('Files') +} + +export function getDroppedFiles(dataTransfer) { + return Array.from(dataTransfer?.files ?? []) +} + +function getFileSize(file) { + return typeof file?.size === 'number' && Number.isFinite(file.size) && file.size >= 0 + ? file.size + : null +} + +export function validateImageFile(file) { + if (!file || typeof file !== 'object') { + return { valid: false, reason: IMAGE_FILE_ERROR.INVALID } + } + + const type = typeof file.type === 'string' ? file.type.toLowerCase() : '' + const size = getFileSize(file) + if (size === null) return { valid: false, reason: IMAGE_FILE_ERROR.INVALID } + if (size === 0) return { valid: false, reason: IMAGE_FILE_ERROR.INVALID } + if (!IMAGE_MIME_TYPES.includes(type)) return { valid: false, reason: IMAGE_FILE_ERROR.TYPE } + if (size > MAX_IMAGE_SIZE) return { valid: false, reason: IMAGE_FILE_ERROR.FILE_SIZE } + + return { valid: true, type, size } +} + +function getExistingImageSize(files) { + return Array.from(files ?? []).reduce((total, file) => total + (getFileSize(file) ?? 0), 0) +} + +/** + * Select valid image files while enforcing the per-file, count, and total byte limits. + * The returned rejected entries retain their original file and a stable reason code so + * callers can present a localized message without exposing browser-specific errors. + */ +export function validateImageFiles(files, existingFiles = []) { + const accepted = [] + const rejected = [] + let totalSize = getExistingImageSize(existingFiles) + const existingCount = Array.from(existingFiles ?? []).length + + for (const file of Array.from(files ?? [])) { + if (existingCount + accepted.length >= MAX_IMAGE_COUNT) { + rejected.push({ file, reason: IMAGE_FILE_ERROR.COUNT }) + continue + } + + const validation = validateImageFile(file) + if (!validation.valid) { + rejected.push({ file, reason: validation.reason }) + continue + } + + if (totalSize + validation.size > MAX_TOTAL_IMAGE_SIZE) { + rejected.push({ file, reason: IMAGE_FILE_ERROR.TOTAL_SIZE }) + continue + } + + accepted.push(file) + totalSize += validation.size + } + + return { accepted, rejected, totalSize } +} + +function bytesToBase64(bytes) { + let binary = '' + const chunkSize = 0x8000 + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize)) + } + + if (typeof btoa === 'function') return btoa(binary) + if (typeof globalThis.Buffer !== 'undefined') + return globalThis.Buffer.from(bytes).toString('base64') + throw new Error('No base64 encoder is available') +} + +/** Read a validated image file as a data URL for transport and previews. */ +export function readImageAsDataUrl(file) { + if (typeof FileReader !== 'undefined') { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => { + if (typeof reader.result === 'string') resolve(reader.result) + else reject(new Error('Image reader returned an invalid result')) + } + reader.onerror = () => reject(reader.error || new Error('Unable to read image')) + reader.onabort = () => reject(new Error('Image read was aborted')) + reader.readAsDataURL(file) + }) + } + + if (typeof file?.arrayBuffer === 'function') { + return file.arrayBuffer().then((buffer) => { + const type = + typeof file.type === 'string' && file.type ? file.type : 'application/octet-stream' + return `data:${type};base64,${bytesToBase64(new Uint8Array(buffer))}` + }) + } + + return Promise.reject(new Error('Unable to read image')) +} diff --git a/src/components/InputBox/index.jsx b/src/components/InputBox/index.jsx index 97eb1b90a..97decd843 100644 --- a/src/components/InputBox/index.jsx +++ b/src/components/InputBox/index.jsx @@ -12,11 +12,39 @@ import { MIN_INPUT_HEIGHT, } from './resize.mjs' import { shouldHandleInputAction } from './input-action.mjs' +import { + IMAGE_ACCEPT, + IMAGE_FILE_ERROR, + getDroppedFiles, + hasDraggedFiles, + readImageAsDataUrl, + validateImageFiles, +} from './images.mjs' -export function InputBox({ onSubmit, enabled, postMessage, reverseResizeDir }) { +export function InputBox({ + onSubmit, + enabled, + postMessage, + reverseResizeDir, + imagesAllowed = false, + resetKey, +}) { const { t } = useTranslation() const [value, setValue] = useState('') + const [imageEntries, setImageEntries] = useState([]) + const [imageError, setImageError] = useState('') + const [isReadingImages, setIsReadingImages] = useState(false) + const [isSubmitting, setIsSubmitting] = useState(false) const inputRef = useRef(null) + const fileInputRef = useRef(null) + const imageEntriesRef = useRef([]) + const readGenerationRef = useRef(0) + const pendingReadsRef = useRef(0) + const imageIdRef = useRef(0) + const mountedRef = useRef(true) + const draftRevisionRef = useRef(0) + const submittingRef = useRef(false) + const submissionGenerationRef = useRef(0) const resizedRef = useRef(false) const resizeHandleRef = useRef(null) const resizeStartRef = useRef(null) @@ -24,10 +52,230 @@ export function InputBox({ onSubmit, enabled, postMessage, reverseResizeDir }) { const [inputHeight, setInputHeight] = useState(DEFAULT_INPUT_HEIGHT) const [maxInputHeight, setMaxInputHeight] = useState(DEFAULT_INPUT_HEIGHT) + const replaceImageEntries = (nextEntries) => { + imageEntriesRef.current = nextEntries + setImageEntries(nextEntries) + } + + const bumpDraftRevision = () => { + draftRevisionRef.current += 1 + } + + const clearFileInput = () => { + if (fileInputRef.current) fileInputRef.current.value = '' + } + + const invalidateImageReads = () => { + readGenerationRef.current += 1 + pendingReadsRef.current = 0 + if (mountedRef.current) setIsReadingImages(false) + } + + const clearImages = () => { + invalidateImageReads() + replaceImageEntries([]) + bumpDraftRevision() + clearFileInput() + } + + const clearDraft = () => { + submissionGenerationRef.current += 1 + clearImages() + setValue('') + setImageError('') + submittingRef.current = false + if (mountedRef.current) setIsSubmitting(false) + } + + const getImageErrorMessage = (reason) => { + switch (reason) { + case IMAGE_FILE_ERROR.TYPE: + return t('Images must be PNG, JPEG, WEBP, or GIF.') + case IMAGE_FILE_ERROR.FILE_SIZE: + return t('Each image must be 4 MiB or smaller.') + case IMAGE_FILE_ERROR.COUNT: + return t('You can attach up to 4 images.') + case IMAGE_FILE_ERROR.TOTAL_SIZE: + return t('Total image size must be 12 MiB or smaller.') + default: + return t('This file is not a supported image.') + } + } + + const isCurrentRead = (generation) => + mountedRef.current && readGenerationRef.current === generation + + const addImageFiles = (files) => { + if (!files || Array.from(files).length === 0) return + + if (!imagesAllowed) { + setImageError(t('Images require an OpenAI-compatible vision API.')) + return + } + + const currentEntries = imageEntriesRef.current + const result = validateImageFiles( + files, + currentEntries.map((entry) => entry.file), + ) + if (result.rejected.length > 0) { + setImageError(getImageErrorMessage(result.rejected[0].reason)) + } else { + setImageError('') + } + if (result.accepted.length === 0) return + + const generation = readGenerationRef.current + const entries = result.accepted.map((file) => ({ + id: ++imageIdRef.current, + file, + dataUrl: null, + })) + replaceImageEntries([...currentEntries, ...entries]) + bumpDraftRevision() + pendingReadsRef.current += entries.length + setIsReadingImages(true) + + for (const entry of entries) { + Promise.resolve() + .then(() => readImageAsDataUrl(entry.file)) + .then((dataUrl) => { + if (!isCurrentRead(generation)) return + replaceImageEntries( + imageEntriesRef.current.map((currentEntry) => + currentEntry.id === entry.id ? { ...currentEntry, dataUrl } : currentEntry, + ), + ) + }) + .catch(() => { + if (!isCurrentRead(generation)) return + if (!imageEntriesRef.current.some((currentEntry) => currentEntry.id === entry.id)) return + replaceImageEntries( + imageEntriesRef.current.filter((currentEntry) => currentEntry.id !== entry.id), + ) + bumpDraftRevision() + setImageError(t('Unable to read image.')) + }) + .finally(() => { + if (!isCurrentRead(generation)) return + pendingReadsRef.current = Math.max(0, pendingReadsRef.current - 1) + if (pendingReadsRef.current === 0) setIsReadingImages(false) + }) + } + } + + const removeImage = (id) => { + if (!imageEntriesRef.current.some((entry) => entry.id === id)) return + replaceImageEntries(imageEntriesRef.current.filter((entry) => entry.id !== id)) + bumpDraftRevision() + setImageError('') + } + + const handleImageInputChange = (e) => { + addImageFiles(e.currentTarget.files) + e.currentTarget.value = '' + } + + const handlePaste = (e) => { + const items = Array.from(e.clipboardData?.items ?? []) + const files = items + .filter((item) => item.kind === 'file') + .map((item) => item.getAsFile()) + .filter(Boolean) + if (files.length === 0) return + + e.preventDefault() + addImageFiles(files) + } + + const handleDragOver = (e) => { + if (!hasDraggedFiles(e.dataTransfer)) return + e.preventDefault() + if (e.dataTransfer) e.dataTransfer.dropEffect = imagesAllowed ? 'copy' : 'none' + } + + const handleDrop = (e) => { + const files = getDroppedFiles(e.dataTransfer) + if (files.length === 0) return + e.preventDefault() + addImageFiles(files) + } + + const handleKeyDownOrClick = (e) => { + e.stopPropagation() + if (!shouldHandleInputAction(e)) return + + e.preventDefault() + if (!enabled) { + postMessage({ stop: true }) + return + } + + if (isReadingImages || pendingReadsRef.current > 0 || submittingRef.current) return + + const submittedImages = imageEntriesRef.current + .map((entry) => entry.dataUrl) + .filter((dataUrl) => typeof dataUrl === 'string' && dataUrl.length > 0) + const question = value.trim() + ? value + : submittedImages.length > 0 + ? t('Describe these images') + : '' + if (!question) return + + const submittedRevision = draftRevisionRef.current + const submissionGeneration = ++submissionGenerationRef.current + submittingRef.current = true + setIsSubmitting(true) + Promise.resolve() + .then(() => onSubmit(question, submittedImages)) + .then(() => { + if ( + !mountedRef.current || + submissionGenerationRef.current !== submissionGeneration || + draftRevisionRef.current !== submittedRevision + ) + return + clearDraft() + }) + .catch((error) => { + if ( + !mountedRef.current || + submissionGenerationRef.current !== submissionGeneration || + draftRevisionRef.current !== submittedRevision + ) + return + const message = error instanceof Error ? error.message : String(error || '') + setImageError(message || t('Unable to send images.')) + }) + .finally(() => { + if (submissionGenerationRef.current !== submissionGeneration) return + submittingRef.current = false + if (mountedRef.current) setIsSubmitting(false) + }) + } + + useEffect(() => { + inputRef.current?.focus() + }, []) + useEffect(() => { - inputRef.current.focus() + return () => { + mountedRef.current = false + readGenerationRef.current += 1 + } }, []) + useEffect(() => { + clearDraft() + }, [resetKey]) + + useEffect(() => { + if (imagesAllowed || (imageEntriesRef.current.length === 0 && pendingReadsRef.current === 0)) + return + clearImages() + }, [imagesAllowed]) + useEffect(() => { if (hasTopResizeHandle) return @@ -92,20 +340,6 @@ export function InputBox({ onSubmit, enabled, postMessage, reverseResizeDir }) { }) }, [enabled]) - const handleKeyDownOrClick = (e) => { - e.stopPropagation() - if (shouldHandleInputAction(e)) { - e.preventDefault() - if (enabled) { - if (!value) return - onSubmit(value) - setValue('') - } else { - postMessage({ stop: true }) - } - } - } - const handleResizePointerDown = (e) => { if (!e.isPrimary || e.button !== 0) return @@ -154,8 +388,15 @@ export function InputBox({ onSubmit, enabled, postMessage, reverseResizeDir }) { ) } + const sendDisabled = enabled && (isReadingImages || isSubmitting) + return ( -
+
{hasTopResizeHandle && (
)} + + {imagesAllowed ? ( +
+ + + {t('Drop images here or paste a screenshot')} + +
+ ) : ( +
+ {t('Images require an OpenAI-compatible vision API.')} +
+ )} + {imageEntries.length > 0 && ( +
+ {imageEntries.map((entry, index) => ( +
+ {entry.dataUrl ? ( + {`${t('Image')} + ) : ( + + … + + )} + +
+ ))} +
+ )} + {imageError && ( +
+ {imageError} +
+ )}
setValue(e.target.value)} + onChange={(e) => { + setValue(e.target.value) + bumpDraftRevision() + }} onKeyDown={handleKeyDownOrClick} />
', + ) + await writeFile( + join(extension, 'sidebar-launcher.js'), + ` + const tab = JSON.parse(decodeURIComponent(location.hash.slice(1))) + document.getElementById('open').onclick = () => chrome.runtime.sendMessage({ + type: '__testInvokeContextMenu', info: {menuItemId: 'ChatGPTBox-MenuopenSidePanel'}, tab + }).catch(() => {}) + `, + ) + return extension +} + +async function attachTargetEvaluator(cdp, targetId, label) { + const { sessionId } = await cdp.send('Target.attachToTarget', { flatten: false, targetId }) + let commandId = 0 + async function command(method, params = {}) { + const id = ++commandId + return new Promise((resolvePromise, reject) => { + const timer = setTimeout(() => { + cdp.off('Target.receivedMessageFromTarget', onMessage) + reject(new Error(`${label} CDP command timed out: ${method}`)) + }, 10000) + function onMessage(event) { + if (event.sessionId !== sessionId) return + const message = JSON.parse(event.message) + if (message.id !== id) return + clearTimeout(timer) + cdp.off('Target.receivedMessageFromTarget', onMessage) + if (message.error) { + reject(new Error(JSON.stringify(message.error))) + return + } + resolvePromise(message.result) + } + cdp.on('Target.receivedMessageFromTarget', onMessage) + cdp + .send('Target.sendMessageToTarget', { + message: JSON.stringify({ id, method, params }), + sessionId, + }) + .catch((error) => { + clearTimeout(timer) + cdp.off('Target.receivedMessageFromTarget', onMessage) + reject(error) + }) + }) + } + async function evaluate(expression, options = {}) { + const result = await command('Runtime.evaluate', { + awaitPromise: true, + expression, + returnByValue: true, + userGesture: options.userGesture !== false, + }) + if (result?.exceptionDetails) { + throw new Error(JSON.stringify(result.exceptionDetails)) + } + return result?.result?.value + } + return { command, evaluate } +} + +async function makeWorkerEvaluator(context, page) { + const cdp = await context.newCDPSession(page) + const workerTarget = (await cdp.send('Target.getTargets')).targetInfos.find( + (target) => target.type === 'service_worker' && target.url.includes('/background.js'), + ) + assert.ok(workerTarget, 'background service worker CDP target is missing') + const evaluator = await attachTargetEvaluator(cdp, workerTarget.targetId, 'service worker') + return { cdp, evaluate: evaluator.evaluate } +} + +async function waitForPanel(cdp, extensionId, sidePanelPath, timeout = 15000) { + const expectedUrl = new URL(sidePanelPath, `chrome-extension://${extensionId}/`).href + const deadline = Date.now() + timeout + let targets = [] + while (Date.now() < deadline) { + targets = (await cdp.send('Target.getTargets')).targetInfos + const panel = targets.find((target) => target.url === expectedUrl) + if (panel) return { panel, targets } + await sleep(100) + } + const extensionTargets = targets + .filter((target) => target.url.startsWith(`chrome-extension://${extensionId}/`)) + .map((target) => ({ type: target.type, url: target.url })) + throw new Error( + `native side panel target ${expectedUrl} timed out; extension targets: ${JSON.stringify( + extensionTargets, + )}`, + ) +} + +async function readSidePanelPath(extension) { + const manifest = JSON.parse(await readFile(join(extension, 'manifest.json'), 'utf8')) + const sidePanelPath = manifest.side_panel?.default_path + assert.equal(typeof sidePanelPath, 'string', 'extension manifest has no side_panel.default_path') + return sidePanelPath +} + +async function waitForWorkerValue(worker, read, label, timeout = 10000) { + const deadline = Date.now() + timeout + while (Date.now() < deadline) { + const value = await read() + if (value) return value + await sleep(100) + } + throw new Error( + `timed out waiting for ${label}: ${JSON.stringify( + await worker.evaluate(() => ({ + url: location.href, + captured: typeof globalThis.__testContextMenuListener, + errors: globalThis.__testErrors, + })), + )}`, + ) +} + +async function launchFixture(extension, baseUrl) { + const profile = await mkdtemp(resolve('test-results/sidepanel-smoke-profile-')) + const context = await chromium.launchPersistentContext(profile, { + args: [`--disable-extensions-except=${extension}`, `--load-extension=${extension}`], + channel: 'msedge', + headless: true, + viewport: { height: 960, width: 1440 }, + }) + const worker = + context.serviceWorkers()[0] || (await context.waitForEvent('serviceworker', { timeout: 15000 })) + const extensionId = new URL(worker.url()).host + const pdf = await context.newPage() + await pdf.goto(`${baseUrl}/protected.pdf`) + await sleep(500) + await pdf.bringToFront() + const workerEvaluate = async (expression, options) => + worker.evaluate((source) => eval(source), `(${JSON.stringify(expression)})`) + // Keep the regular worker handle available for non-gesture Chrome API queries. + const activeTab = await worker.evaluate( + async () => (await chrome.tabs.query({ active: true, currentWindow: true }))[0], + ) + assert.ok( + activeTab?.id != null && activeTab?.windowId != null, + 'fixture PDF is not an active normal tab', + ) + const cdpEvaluator = await makeWorkerEvaluator(context, pdf) + return { + activeTab, + context, + cdp: cdpEvaluator.cdp, + extensionId, + pdf, + profile, + worker, + workerEvaluate, + evaluate: cdpEvaluator.evaluate, + } +} + +async function directTrial(extension, baseUrl, server, sidePanelPath, includeTabId) { + const fixture = await launchFixture(extension, baseUrl) + const { activeTab, context, cdp, evaluate, extensionId, pdf } = fixture + const baselineTabs = await fixture.worker.evaluate( + async () => (await chrome.tabs.query({})).length, + ) + const baselineRequests = server.pdfRequests + const options = includeTabId + ? { tabId: activeTab.id, windowId: activeTab.windowId } + : { windowId: activeTab.windowId } + let callResult + let callError = null + try { + callResult = await evaluate(`chrome.sidePanel.open(${JSON.stringify(options)})`) + } catch (error) { + callError = String(error) + } + let panel = null + try { + panel = (await waitForPanel(cdp, extensionId, sidePanelPath, 3000)).panel + } catch (error) { + callError ||= String(error) + } + const tabsAfter = await fixture.worker.evaluate(async () => (await chrome.tabs.query({})).length) + const result = { + api: 'chrome.sidePanel.open', + callError, + callResult, + includeTabId, + panelTarget: panel && { type: panel.type, url: panel.url }, + pdfRequestsBefore: baselineRequests, + pdfRequestsAfter: server.pdfRequests, + tabsBefore: baselineTabs, + tabsAfter, + pdfUrl: pdf.url(), + } + await context.close() + return result +} + +async function setImageConfig(worker, baseUrl) { + await worker.evaluate( + async (config) => { + await chrome.storage.local.clear() + await chrome.storage.local.set(config) + }, + { + activeApiModes: ['chatgptApi4o_128k', 'customModel'], + apiKey: 'native-sidepanel-smoke-key', + apiMode: null, + configSchemaVersion: 2, + customApiModes: [], + customOpenAIProviders: [], + customOpenAiApiUrl: baseUrl, + focusAfterAnswer: false, + knownApiModeDefaultIds: ['chatgptApi4o_128k', 'customModel'], + modelName: 'chatgptApi4o_128k', + preferredLanguage: 'en', + providerSecrets: { openai: 'native-sidepanel-smoke-key' }, + sessions: [], + userLanguage: 'en', + }, + ) +} + +async function waitForNativePanel(evaluate, timeout = 15000) { + const deadline = Date.now() + timeout + let state + while (Date.now() < deadline) { + state = await evaluate(`(() => ({ + ready: Boolean( + document.querySelector('.IndependentPanel') && + document.querySelector('.input-box') && + document.querySelector('textarea.interact-input') + ), + attachButtonCount: document.querySelectorAll('.input-image-button').length, + url: location.href, + }))()`) + if (state?.ready && state.attachButtonCount) return state + await sleep(100) + } + throw new Error(`native side panel did not render the image toolbar: ${JSON.stringify(state)}`) +} + +async function inspectNativePanel(evaluate) { + return evaluate(`(() => { + const box = document.querySelector('.input-box') + const button = box?.querySelector('.input-image-button') + const toolbar = button?.parentElement + const hint = toolbar?.querySelector('span') + const chatContainer = document.querySelector('.chat-container') + const fileInputs = box?.querySelectorAll('input[type="file"]') || [] + const rect = (element) => { + if (!element) return null + const bounds = element.getBoundingClientRect() + return { + bottom: Math.round(bounds.bottom * 10) / 10, + height: Math.round(bounds.height * 10) / 10, + left: Math.round(bounds.left * 10) / 10, + right: Math.round(bounds.right * 10) / 10, + top: Math.round(bounds.top * 10) / 10, + width: Math.round(bounds.width * 10) / 10, + } + } + const visible = (element) => { + if (!element) return false + const style = getComputedStyle(element) + const bounds = element.getBoundingClientRect() + return style.display !== 'none' && style.visibility !== 'hidden' && bounds.width > 0 && bounds.height > 0 + } + const boxRect = rect(box) + const toolbarRect = rect(toolbar) + const buttonRect = rect(button) + const hintRect = rect(hint) + const viewport = { width: innerWidth, height: innerHeight } + const escapes = (bounds, container) => Boolean( + bounds && container && ( + bounds.left < container.left || bounds.right > container.right || + bounds.top < container.top || bounds.bottom > container.bottom + ) + ) + return { + url: location.href, + title: document.title, + viewport: { + ...viewport, + documentScrollWidth: document.documentElement.scrollWidth, + documentScrollHeight: document.documentElement.scrollHeight, + }, + layout: { + documentHeightChain: { + html: rect(document.documentElement), + body: rect(document.body), + app: rect(document.querySelector('#app')), + independentPanel: rect(document.querySelector('.IndependentPanel')), + }, + chatContainerBounds: rect(chatContainer), + chatContainerHeightRatio: chatContainer + ? Math.round((chatContainer.getBoundingClientRect().height / innerHeight) * 1000) / 1000 + : 0, + }, + inputBox: { + display: box ? getComputedStyle(box).display : null, + bounds: boxRect, + clientWidth: box?.clientWidth || 0, + scrollWidth: box?.scrollWidth || 0, + overflowX: box ? getComputedStyle(box).overflowX : null, + }, + toolbar: { + bounds: toolbarRect, + clientWidth: toolbar?.clientWidth || 0, + scrollWidth: toolbar?.scrollWidth || 0, + overflowX: toolbar ? getComputedStyle(toolbar).overflowX : null, + buttonCount: box?.querySelectorAll('.input-image-button').length || 0, + buttonText: button?.innerText.trim() || '', + buttonAriaLabel: button?.getAttribute('aria-label') || '', + buttonVisible: visible(button), + buttonBounds: buttonRect, + hintText: hint?.innerText.trim() || '', + hintVisible: visible(hint), + hintBounds: hintRect, + }, + fileInput: { + count: fileInputs.length, + accept: fileInputs[0]?.getAttribute('accept') || '', + multiple: fileInputs[0]?.multiple || false, + ariaLabel: fileInputs[0]?.getAttribute('aria-label') || '', + }, + clipping: { + buttonOutsideToolbar: escapes(buttonRect, toolbarRect), + hintOutsideToolbar: escapes(hintRect, toolbarRect), + toolbarOutsideViewport: Boolean( + (buttonRect && (buttonRect.left < 0 || buttonRect.right > viewport.width || buttonRect.top < 0 || buttonRect.bottom > viewport.height)) || + (hintRect && (hintRect.left < 0 || hintRect.right > viewport.width || hintRect.top < 0 || hintRect.bottom > viewport.height)) + ), + toolbarHorizontalOverflow: Boolean(toolbar && toolbar.scrollWidth > toolbar.clientWidth), + }, + } + })()`) +} + +async function attachLocalImage(panel, filePath) { + const { root } = await panel.command('DOM.getDocument', { depth: -1, pierce: true }) + const { nodeId } = await panel.command('DOM.querySelector', { + nodeId: root.nodeId, + selector: 'input[type="file"][aria-label="Attach images"]', + }) + assert.ok(nodeId, 'native side panel file input is missing from its DOM') + await panel.command('DOM.setFileInputFiles', { files: [filePath], nodeId }) + const deadline = Date.now() + 10000 + let state + while (Date.now() < deadline) { + state = await panel.evaluate(`(() => ({ + previewCount: document.querySelectorAll('.input-box img').length, + previewSource: document.querySelector('.input-box img')?.getAttribute('src') || '', + }))()`) + if (state?.previewCount === 1 && state.previewSource.startsWith('data:image/png;base64,')) { + return state + } + await sleep(100) + } + throw new Error( + `native panel file input did not create an image preview: ${JSON.stringify(state)}`, + ) +} + +async function runContextMenuTrial(extension, baseUrl, server, sidePanelPath, trial) { + const fixture = await launchFixture(extension, baseUrl) + const { activeTab, context, cdp, extensionId, pdf, worker } = fixture + try { + await setImageConfig(worker, baseUrl) + const launcherUrl = `chrome-extension://${extensionId}/sidebar-launcher.html#${encodeURIComponent( + JSON.stringify(activeTab), + )}` + const launcherReady = context.waitForEvent('page') + await worker.evaluate( + async ({ url, windowId }) => chrome.tabs.create({ url, windowId, active: true }), + { url: launcherUrl, windowId: activeTab.windowId }, + ) + const launcher = await launcherReady + await launcher.waitForLoadState() + const activeLauncherTab = await worker.evaluate( + async () => (await chrome.tabs.query({ active: true, currentWindow: true }))[0], + ) + const focusedWindow = await worker.evaluate(async () => chrome.windows.getLastFocused()) + assert.equal( + activeLauncherTab?.windowId, + activeTab.windowId, + 'test launcher is not in the PDF tab window', + ) + assert.equal( + focusedWindow?.id, + activeTab.windowId, + 'PDF tab window is not focused for the current-window fallback trial', + ) + const baselineTabs = await worker.evaluate(async () => (await chrome.tabs.query({})).length) + const baselineRequests = server.pdfRequests + const captured = await waitForWorkerValue( + worker, + () => worker.evaluate(() => typeof globalThis.__testContextMenuListener === 'function'), + 'production context-menu listener', + ) + assert.equal( + captured, + true, + 'instrumentation did not capture the production context-menu listener', + ) + let callbackError = null + try { + await launcher.evaluate((callbackTab) => { + document.getElementById('open').onclick = () => + chrome.runtime + .sendMessage({ + type: '__testInvokeContextMenu', + info: { menuItemId: 'ChatGPTBox-MenuopenSidePanel' }, + tab: callbackTab, + }) + .catch(() => {}) + }, trial.callbackTab(activeTab)) + await launcher.locator('#open').click() + } catch (error) { + callbackError = String(error) + } + // Return focus to the fixture PDF tab in the same window before reading + // the native panel target's DOM. + await pdf.bringToFront() + let panel + try { + panel = (await waitForPanel(cdp, extensionId, sidePanelPath)).panel + } catch (error) { + const diagnostics = await worker.evaluate(async () => ({ + errors: globalThis.__testErrors, + contextMenuInvocations: globalThis.__testContextMenuInvocations, + sidePanelCalls: globalThis.__testSidePanelCalls, + capturedCallback: typeof globalThis.__testContextMenuListener, + activeTabs: await chrome.tabs.query({ active: true, currentWindow: true }), + focusedWindow: await chrome.windows.getLastFocused(), + pdfSidePanelOptions: await chrome.sidePanel + .getOptions({ tabId: globalThis.__testContextMenuInvocations.at(-1)?.tab?.id }) + .catch((error) => ({ error: String(error) })), + })) + throw new Error(`${error.message}; callback diagnostics: ${JSON.stringify(diagnostics)}`) + } + const panelEvaluator = await attachTargetEvaluator(cdp, panel.targetId, 'native side panel') + await waitForNativePanel(panelEvaluator.evaluate) + const sidePanelCall = await worker.evaluate(() => globalThis.__testSidePanelCalls.at(-1)) + assert.deepEqual( + sidePanelCall?.options, + { windowId: -2 }, + 'missing tab.windowId should open the current window panel', + ) + assert.equal(sidePanelCall?.settled, true, 'chrome.sidePanel.open did not settle') + assert.equal(sidePanelCall?.error, undefined, 'chrome.sidePanel.open rejected') + const callbackErrors = await worker.evaluate(() => globalThis.__testErrors) + assert.deepEqual(callbackErrors, [], `background callback errors: ${callbackErrors.join('; ')}`) + const panelDom = await inspectNativePanel(panelEvaluator.evaluate) + console.log(`${trial.name} native panel DOM:`, JSON.stringify(panelDom)) + assert.equal( + panelDom.toolbar.buttonCount, + 1, + 'native side panel Attach images toolbar is missing', + ) + assert.equal(panelDom.toolbar.buttonText, 'Attach images') + assert.equal( + panelDom.toolbar.buttonVisible, + true, + 'native side panel Attach images button is hidden', + ) + assert.equal(panelDom.toolbar.hintText, 'Drop images here or paste a screenshot') + assert.equal(panelDom.toolbar.hintVisible, true, 'native side panel image hint is hidden') + assert.equal(panelDom.fileInput.count, 1, 'native side panel image file input is missing') + assert.equal( + panelDom.fileInput.multiple, + true, + 'native side panel file input does not allow multiple files', + ) + assert.equal( + panelDom.inputBox.display, + 'contents', + 'native input box is expected to use display: contents', + ) + assert.equal( + panelDom.clipping.buttonOutsideToolbar, + false, + 'Attach images button escapes its toolbar', + ) + assert.equal(panelDom.clipping.hintOutsideToolbar, false, 'image hint escapes its toolbar') + assert.equal( + panelDom.clipping.toolbarOutsideViewport, + false, + 'image toolbar is clipped by the panel viewport', + ) + assert.equal( + panelDom.clipping.toolbarHorizontalOverflow, + false, + 'image toolbar overflows horizontally', + ) + assert.ok( + panelDom.layout.chatContainerHeightRatio >= 0.9, + `native side panel chat container uses only ${panelDom.layout.chatContainerHeightRatio} of the viewport height`, + ) + const screenshotPath = resolve('test-results/sidebar2-native.png') + const screenshot = await panelEvaluator.command('Page.captureScreenshot', { + captureBeyondViewport: false, + format: 'png', + fromSurface: true, + }) + await writeFile(screenshotPath, Buffer.from(screenshot.data, 'base64')) + const imagePath = join(fixture.profile, 'native-sidepanel-smoke.png') + await writeFile(imagePath, SMOKE_PNG_BYTES) + const fileState = await attachLocalImage(panelEvaluator, imagePath) + assert.equal(fileState.previewSource, SMOKE_PNG_DATA_URL) + console.log(`${trial.name} native panel file input:`, JSON.stringify(fileState)) + const tabsAfter = await worker.evaluate(async () => (await chrome.tabs.query({})).length) + const result = { + api: 'captured production contextMenus.onClicked callback', + callbackError, + panelTarget: { type: panel.type, url: panel.url }, + callbackTab: trial.callbackTab(activeTab), + sidePanelCall, + pdfRequestsBefore: baselineRequests, + pdfRequestsAfter: server.pdfRequests, + tabsBefore: baselineTabs, + tabsAfter, + pdfUrl: pdf.url(), + } + assert.equal(callbackError, null, 'production context-menu callback threw or rejected') + assert.equal( + result.panelTarget.url, + new URL(sidePanelPath, `chrome-extension://${extensionId}/`).href, + ) + assert.equal(result.tabsAfter, baselineTabs, 'native panel must not create a browser tab') + assert.equal( + result.pdfRequestsAfter, + baselineRequests, + 'opening sidebar must not refetch protected PDF', + ) + assert.equal(result.pdfUrl, `${baseUrl}/protected.pdf`, 'original PDF stays open') + return result + } finally { + await context.close() + } +} + +async function run() { + await mkdir('test-results', { recursive: true }) + const source = resolve(process.env.CODEX_EXTENSION || 'build/chromium') + await stat(source) + const extension = await makeInstrumentedExtension(source) + const sidePanelPath = await readSidePanelPath(extension) + const server = await startServer() + try { + // A real click is required; DevTools userGesture on a service worker does + // not create the extension API gesture token in Edge. Use a regular + // extension tab in the PDF window so the empty-tab fallback tests the + // actual current window that contains the PDF. + const trials = [ + { + name: 'current-window fallback callback', + callbackTab: () => ({}), + }, + { + name: 'tab-id-only current-window fallback callback', + callbackTab: (activeTab) => ({ id: activeTab.id }), + }, + ] + const results = [] + for (const trial of trials) { + const result = await runContextMenuTrial( + extension, + server.baseUrl, + server, + sidePanelPath, + trial, + ) + results.push(result) + console.log('CONTEXT MENU production callback:', JSON.stringify(result)) + } + console.log( + 'PASS: native Edge side panel opened from captured production context-menu callbacks with tab.id-only and empty tab objects; Attach images toolbar, hint, file input preview, viewport clipping, no extra production tab, and no protected PDF refetch verified; screenshot: test-results/sidebar2-native.png.', + ) + } finally { + await new Promise((resolvePromise) => server.server.close(resolvePromise)) + } +} + +run().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/tests/unit/background/menus.test.mjs b/tests/unit/background/menus.test.mjs new file mode 100644 index 000000000..8cf2b9a40 --- /dev/null +++ b/tests/unit/background/menus.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import Browser from 'webextension-polyfill' +import i18next from 'i18next' +import { refreshMenu } from '../../../src/background/menus.mjs' +import { config as menuConfig } from '../../../src/content-script/menu-tools/index.mjs' + +test('handles the waking side-panel click before asynchronous menu setup finishes', async (t) => { + await i18next.init({ lng: 'en', resources: { en: { translation: {} } } }) + const originalMenus = Browser.contextMenus + const originalAction = menuConfig.openSidePanel.action + const listeners = new Set() + let finishRemoval + const removal = new Promise((resolve) => { + finishRemoval = resolve + }) + Browser.contextMenus = { + onClicked: { + addListener: (listener) => listeners.add(listener), + hasListener: (listener) => listeners.has(listener), + }, + removeAll: (callback) => { + return removal.then(() => callback()) + }, + create: () => {}, + } + const action = t.mock.fn(() => Promise.resolve()) + menuConfig.openSidePanel.action = action + t.after(() => { + Browser.contextMenus = originalMenus + menuConfig.openSidePanel.action = originalAction + }) + + const ready = refreshMenu() + assert.equal(listeners.size, 1, 'listener must exist before removeAll resolves') + const tab = { id: 7, windowId: 9 } + for (const listener of listeners) listener({ menuItemId: 'ChatGPTBox-MenuopenSidePanel' }, tab) + assert.deepEqual(action.mock.calls[0].arguments, [true, tab]) + finishRemoval() + await ready + await refreshMenu() + assert.equal(listeners.size, 1, 'refresh must not duplicate the gesture listener') +}) diff --git a/tests/unit/background/redact.test.mjs b/tests/unit/background/redact.test.mjs index b0ad3c20b..85a5ff02f 100644 --- a/tests/unit/background/redact.test.mjs +++ b/tests/unit/background/redact.test.mjs @@ -15,6 +15,7 @@ describe('redactSensitiveFields', () => { credential: 'cred-value', jwt: 'eyJ...', session: 'sess-xyz', + images: ['data:image/png;base64,cHJpdmF0ZQ=='], kimimoonshotrefreshtoken: 'refresh-val', } const result = redactSensitiveFields(input) diff --git a/tests/unit/components/conversation-card-session.test.mjs b/tests/unit/components/conversation-card-session.test.mjs index fdf543c72..1b271f15e 100644 --- a/tests/unit/components/conversation-card-session.test.mjs +++ b/tests/unit/components/conversation-card-session.test.mjs @@ -10,6 +10,17 @@ import { isSupersededRequestMessage, } from '../../../src/components/ConversationCard/session.mjs' +test('retry restores the target images and interruption preserves them in history', () => { + const image = 'data:image/png;base64,aGVsbG8=' + const record = { question: 'Describe', answer: 'Old', images: [image] } + const session = { question: 'Describe', images: [], conversationRecords: [record] } + const retry = createRetrySession(session, [], record) + assert.deepEqual(retry.images, [image]) + const interrupted = finalizeInterruptedSession(retry, 'Partial', record) + assert.deepEqual(interrupted.conversationRecords[0].images, [image]) + assert.deepEqual(session.images, []) +}) + test('finalizeInterruptedSession appends a partial answer without mutating the source session', () => { const session = { question: 'Q1', diff --git a/tests/unit/components/input-box-images.test.mjs b/tests/unit/components/input-box-images.test.mjs new file mode 100644 index 000000000..d2b90b948 --- /dev/null +++ b/tests/unit/components/input-box-images.test.mjs @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import { + IMAGE_FILE_ERROR, + MAX_IMAGE_COUNT, + MAX_IMAGE_SIZE, + MAX_TOTAL_IMAGE_SIZE, + getDroppedFiles, + hasDraggedFiles, + readImageAsDataUrl, + validateImageFile, + validateImageFiles, +} from '../../../src/components/InputBox/images.mjs' + +const image = (size = 1, type = 'image/png') => ({ size, type }) + +describe('input image validation', () => { + test('accepts supported non-empty images and rejects unsupported types or empty files', () => { + assert.equal(validateImageFile(image()).valid, true) + assert.deepEqual(validateImageFile(image(0)), { + valid: false, + reason: IMAGE_FILE_ERROR.INVALID, + }) + assert.deepEqual(validateImageFile(image(1, 'image/svg+xml')), { + valid: false, + reason: IMAGE_FILE_ERROR.TYPE, + }) + }) + + test('enforces per-file, count, and cumulative byte limits', () => { + assert.equal(validateImageFile(image(MAX_IMAGE_SIZE)).valid, true) + assert.equal(validateImageFile(image(MAX_IMAGE_SIZE + 1)).reason, IMAGE_FILE_ERROR.FILE_SIZE) + + const tooMany = validateImageFiles(Array.from({ length: MAX_IMAGE_COUNT + 1 }, () => image())) + assert.equal(tooMany.accepted.length, MAX_IMAGE_COUNT) + assert.equal(tooMany.rejected.at(-1).reason, IMAGE_FILE_ERROR.COUNT) + + const overTotal = validateImageFiles([image(2)], [image(MAX_TOTAL_IMAGE_SIZE - 1)]) + assert.equal(overTotal.accepted.length, 0) + assert.equal(overTotal.rejected[0].reason, IMAGE_FILE_ERROR.TOTAL_SIZE) + }) +}) + +test('only treats file drags and drops as image attachment input', () => { + assert.equal(hasDraggedFiles({ files: [], types: ['text/plain'] }), false) + assert.equal(hasDraggedFiles({ files: [], types: ['text/uri-list'] }), false) + assert.equal(hasDraggedFiles({ files: [], types: ['Files'] }), true) + + const files = [image()] + assert.equal(hasDraggedFiles({ files, types: [] }), true) + assert.deepEqual(getDroppedFiles({ files }), files) + assert.deepEqual(getDroppedFiles({ files: [] }), []) +}) + +test('reads an image as a typed data URL when FileReader is unavailable', async () => { + const file = new File([new Uint8Array([0, 1, 2])], 'pixel.png', { type: 'image/png' }) + assert.equal(await readImageAsDataUrl(file), 'data:image/png;base64,AAEC') +}) diff --git a/tests/unit/content-script/menu-tools.test.mjs b/tests/unit/content-script/menu-tools.test.mjs index 22664a429..610997c64 100644 --- a/tests/unit/content-script/menu-tools.test.mjs +++ b/tests/unit/content-script/menu-tools.test.mjs @@ -103,7 +103,7 @@ describe('closeAllChats', () => { }) describe('openSidePanel', () => { - test('opens the side panel synchronously with the active tab identifiers', async (t) => { + test('opens the global panel synchronously without the PDF tab id', async (t) => { const originalSidePanel = globalThis.chrome.sidePanel let called = false const open = t.mock.fn(() => { @@ -118,8 +118,10 @@ describe('openSidePanel', () => { const result = config.openSidePanel.action(true, { id: 7, windowId: 9 }) assert.equal(called, true) - assert.deepEqual(open.mock.calls[0].arguments, [{ windowId: 9, tabId: 7 }]) + assert.deepEqual(open.mock.calls[0].arguments, [{ windowId: 9 }]) await result + await config.openSidePanel.action(true, { windowId: 9 }) + assert.deepEqual(open.mock.calls[1].arguments, [{ windowId: 9 }]) }) test('rejects when the side-panel API is unavailable', async (t) => { @@ -134,12 +136,12 @@ describe('openSidePanel', () => { }) }) - for (const [name, tab] of [ - ['tab', undefined], - ['tab id', { windowId: 9 }], - ['window id', { id: 7 }], + for (const [name, tab, expected] of [ + ['tab', undefined, { windowId: -2 }], + ['window id', { id: 7 }, { windowId: -2 }], + ['invalid window id', { id: 7, windowId: -1 }, { windowId: -2 }], ]) { - test(`rejects when the ${name} is missing`, async (t) => { + test(`opens synchronously when the ${name} is missing`, async (t) => { const originalSidePanel = globalThis.chrome.sidePanel const open = t.mock.fn(() => Promise.resolve()) globalThis.chrome.sidePanel = { open } @@ -147,10 +149,9 @@ describe('openSidePanel', () => { globalThis.chrome.sidePanel = originalSidePanel }) - await assert.rejects(config.openSidePanel.action(true, tab), { - message: 'chrome.sidePanel.open requires a tab with windowId and id', - }) - assert.equal(open.mock.callCount(), 0) + const result = config.openSidePanel.action(true, tab) + assert.deepEqual(open.mock.calls[0].arguments, [expected]) + await result }) } }) diff --git a/tests/unit/locales/image-translations.test.mjs b/tests/unit/locales/image-translations.test.mjs new file mode 100644 index 000000000..2ba2f6b8c --- /dev/null +++ b/tests/unit/locales/image-translations.test.mjs @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict' +import { readdir, readFile } from 'node:fs/promises' +import test from 'node:test' + +const localeRoot = new URL('../../../src/_locales/', import.meta.url) + +const imageMessageKeys = Object.freeze([ + 'Images require an OpenAI-compatible vision API.', + 'Choose a model that supports image input.', + 'Attach images', + 'Drop images here or paste a screenshot', + 'Attached images', + 'Image', + 'Reading image', + 'Remove image', + 'Images must be PNG, JPEG, WEBP, or GIF.', + 'Each image must be 4 MiB or smaller.', + 'You can attach up to 4 images.', + 'Total image size must be 12 MiB or smaller.', + 'This file is not a supported image.', + 'Unable to read image.', + 'Unable to send images.', + 'Describe these images', +]) + +test('every locale defines the image attachment messages', async () => { + const localeDirectories = (await readdir(localeRoot, { withFileTypes: true })).filter((entry) => + entry.isDirectory(), + ) + + for (const localeDirectory of localeDirectories) { + const messages = JSON.parse( + await readFile(new URL(`${localeDirectory.name}/main.json`, localeRoot), 'utf8'), + ) + + for (const key of imageMessageKeys) { + assert.equal( + typeof messages[key], + 'string', + `${localeDirectory.name} is missing ${JSON.stringify(key)}`, + ) + assert.notEqual(messages[key].trim(), '', `${localeDirectory.name} has an empty ${key}`) + } + } +}) diff --git a/tests/unit/release/package-image-source.test.mjs b/tests/unit/release/package-image-source.test.mjs new file mode 100644 index 000000000..2e7770b29 --- /dev/null +++ b/tests/unit/release/package-image-source.test.mjs @@ -0,0 +1,58 @@ +import { execFileSync } from 'node:child_process' +import assert from 'node:assert/strict' +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { test } from 'node:test' +import { + collectSourceFiles, + isSensitiveSourcePath, + readVerifiedSourceFile, +} from '../../../scripts/package-image-source.mjs' + +test('source package collector only includes allowlisted roots and rejects secrets', async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), 'chatgptbox-source-')) + t.after(() => rm(root, { force: true, recursive: true })) + + await Promise.all([ + mkdir(path.join(root, 'src'), { recursive: true }), + mkdir(path.join(root, 'build'), { recursive: true }), + mkdir(path.join(root, 'notes'), { recursive: true }), + ]) + await Promise.all([ + writeFile(path.join(root, 'README.md'), 'public'), + writeFile(path.join(root, 'package.json'), '{}'), + writeFile(path.join(root, '.env'), 'TOKEN=private'), + writeFile(path.join(root, 'src', 'index.js'), 'export {}'), + writeFile(path.join(root, 'src', '.env.local'), 'TOKEN=private'), + writeFile(path.join(root, 'src', 'credentials.json'), '{}'), + writeFile(path.join(root, 'src', 'private.pem'), 'private'), + writeFile(path.join(root, 'build', 'artifact.zip'), 'generated'), + writeFile(path.join(root, 'notes', 'local.txt'), 'untracked'), + ]) + + execFileSync('git', ['init', '--quiet'], { cwd: root }) + execFileSync( + 'git', + ['add', '--', 'README.md', 'package.json', 'src/index.js', 'src/private.pem'], + { cwd: root }, + ) + await Promise.all( + ['token.json', 'api-key.json', 'config.json', 'arbitrary-private.json'].map((name) => + writeFile(path.join(root, 'src', name), 'private'), + ), + ) + execFileSync('git', ['add', '--', 'src/token.json', 'src/api-key.json', 'src/config.json'], { + cwd: root, + }) + assert.deepEqual(await collectSourceFiles(root), ['README.md', 'package.json', 'src/index.js']) + const stagedSource = await readVerifiedSourceFile(root, 'src/index.js') + await writeFile(path.join(root, 'src', 'index.js'), 'replaced after verified read') + assert.equal(stagedSource.toString('utf8'), 'export {}') + assert.equal(isSensitiveSourcePath('src/.env.production'), true) + assert.equal(isSensitiveSourcePath('.npmrc'), true) + assert.equal(isSensitiveSourcePath('id_ed25519'), true) + assert.equal(isSensitiveSourcePath('src/client.key'), true) + assert.equal(isSensitiveSourcePath('src/credentials.json'), true) + assert.equal(isSensitiveSourcePath('src/components/App.jsx'), false) +}) diff --git a/tests/unit/services/apis/images.test.mjs b/tests/unit/services/apis/images.test.mjs new file mode 100644 index 000000000..5f95ef629 --- /dev/null +++ b/tests/unit/services/apis/images.test.mjs @@ -0,0 +1,236 @@ +import assert from 'node:assert/strict' +import { Buffer } from 'node:buffer' +import { beforeEach, test } from 'node:test' +import { + IMAGE_UNSUPPORTED_ERROR, + MAX_IMAGE_BYTES, + MAX_IMAGE_COUNT, + canSendImages, + validateImageDataUrls, + validateSessionImages, +} from '../../../../src/services/apis/images.mjs' +import { + generateAnswersWithGptCompletionApi, + generateAnswersWithOpenAiApiCompat, +} from '../../../../src/services/apis/openai-api.mjs' +import { createFakePort } from '../../helpers/port.mjs' +import { createMockSseResponse } from '../../helpers/sse-response.mjs' + +const imageSignatures = { + 'image/png': Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + 'image/jpeg': Buffer.from([0xff, 0xd8, 0xff]), + 'image/webp': Buffer.from('RIFF0000WEBP', 'ascii'), + 'image/gif': Buffer.from('GIF89a', 'ascii'), +} + +const imageDataUrl = (mime = 'image/png', bytes = 12) => { + const signature = imageSignatures[mime] ?? Buffer.alloc(0) + const content = Buffer.alloc(Math.max(bytes, signature.length), 0xab) + signature.copy(content) + return `data:${mime};base64,${content.toString('base64')}` +} + +const setStorage = (values) => { + globalThis.__TEST_BROWSER_SHIM__.replaceStorage(values) +} + +beforeEach(() => { + globalThis.__TEST_BROWSER_SHIM__.clearStorage() +}) + +test('validates supported image data URLs and attachment limits', () => { + const images = [ + imageDataUrl('image/png'), + imageDataUrl('image/jpeg'), + imageDataUrl('image/webp'), + imageDataUrl('image/gif'), + ] + + assert.deepEqual(validateImageDataUrls(images), images) + assert.throws( + () => validateImageDataUrls([...images, imageDataUrl()]), + (error) => error.code === 'IMAGE_COUNT_EXCEEDED' && error.message.includes(MAX_IMAGE_COUNT), + ) + assert.throws( + () => validateImageDataUrls([imageDataUrl('image/svg+xml')]), + (error) => error.code === 'INVALID_IMAGE_DATA', + ) + assert.throws( + () => validateImageDataUrls(['data:image/png;base64,not base64']), + (error) => error.code === 'INVALID_IMAGE_DATA', + ) + assert.throws( + () => validateImageDataUrls([imageDataUrl('image/jpeg').replace('image/jpeg', 'image/png')]), + (error) => error.code === 'INVALID_IMAGE_DATA' && error.message.includes('does not match'), + ) +}) + +test('validates per-image size and cumulative session limits', () => { + const maxImage = imageDataUrl('image/png', MAX_IMAGE_BYTES) + assert.equal(validateImageDataUrls([maxImage])[0], maxImage) + + const tooLarge = imageDataUrl('image/png', MAX_IMAGE_BYTES + 1) + assert.throws( + () => validateImageDataUrls([tooLarge]), + (error) => error.code === 'IMAGE_SIZE_EXCEEDED', + ) + + const image = imageDataUrl('image/png', 3 * 1024 * 1024) + const session = { + images: [image], + conversationRecords: Array.from({ length: 8 }, () => ({ + question: 'Q', + answer: 'A', + images: [image], + })), + } + assert.throws( + () => validateSessionImages(session), + (error) => error.code === 'IMAGE_SESSION_SIZE_EXCEEDED', + ) +}) + +test('enables images only for resolved OpenAI-compatible chat endpoints', () => { + const config = { customOpenAiApiUrl: 'https://api.example.com' } + const chatSession = { + modelName: 'chatgptApi4oMini', + apiMode: { groupName: 'chatgptApiModelKeys', itemName: 'gpt-4o-mini' }, + } + const completionSession = { + modelName: 'gptApiInstruct', + apiMode: { groupName: 'gptApiModelKeys', itemName: 'text-davinci-003' }, + } + const webSession = { modelName: 'chatgptWeb' } + + assert.equal(canSendImages(config, chatSession), true) + assert.equal(canSendImages(config, completionSession), false) + assert.equal(canSendImages(config, webSession), false) + assert.equal( + canSendImages( + { + customOpenAIProviders: [ + { + id: 'native', + chatCompletionsUrl: 'http://127.0.0.1:11434/api/chat', + enabled: true, + }, + ], + }, + { + modelName: 'customModel', + apiMode: { + groupName: 'customApiModelKeys', + itemName: 'customModel', + isCustom: true, + customName: 'native-model', + providerId: 'native', + }, + }, + ), + false, + ) +}) + +test('sends current and history images as standard OpenAI chat content and preserves record images', async (t) => { + t.mock.method(console, 'debug', () => {}) + setStorage({ + maxConversationContextLength: 3, + maxResponseTokenLength: 256, + }) + + const previousImage = imageDataUrl('image/jpeg') + const currentImage = imageDataUrl('image/png') + const session = { + modelName: 'chatgptApi4oMini', + conversationRecords: [{ question: 'PrevQ', answer: 'PrevA', images: [previousImage] }], + images: [currentImage], + isRetry: false, + } + const port = createFakePort() + let capturedInit + t.mock.method(globalThis, 'fetch', async (_input, init) => { + capturedInit = init + return createMockSseResponse([ + 'data: {"choices":[{"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\n', + ]) + }) + + await generateAnswersWithOpenAiApiCompat( + 'https://api.example.com/v1', + port, + 'CurrentQ', + session, + 'sk-test', + ) + + const body = JSON.parse(capturedInit.body) + assert.deepEqual(body.messages[0], { + role: 'user', + content: [ + { type: 'text', text: 'PrevQ' }, + { type: 'image_url', image_url: { url: previousImage } }, + ], + }) + assert.deepEqual(body.messages.at(-1), { + role: 'user', + content: [ + { type: 'text', text: 'CurrentQ' }, + { type: 'image_url', image_url: { url: currentImage } }, + ], + }) + assert.deepEqual(session.conversationRecords.at(-1), { + question: 'CurrentQ', + answer: 'OK', + images: [currentImage], + }) +}) + +test('rejects image history before registering completion stream listeners', async () => { + const image = imageDataUrl() + const session = { + modelName: 'gptApiInstruct', + conversationRecords: [{ question: 'OldQ', answer: 'OldA', images: [image] }], + images: [], + isRetry: false, + } + const port = createFakePort() + + await assert.rejects( + generateAnswersWithGptCompletionApi(port, 'CurrentQ', session, 'sk-test'), + (error) => error.message === IMAGE_UNSUPPORTED_ERROR, + ) + assert.deepEqual(port.listenerCounts(), { onMessage: 0, onDisconnect: 0 }) +}) + +test('updates a retried multimodal record without dropping its images', async (t) => { + t.mock.method(console, 'debug', () => {}) + setStorage({ + maxConversationContextLength: 3, + maxResponseTokenLength: 256, + }) + const image = imageDataUrl() + const session = { + modelName: 'chatgptApi4oMini', + conversationRecords: [{ question: 'CurrentQ', answer: 'Old', images: [image] }], + images: [image], + isRetry: true, + } + const port = createFakePort() + t.mock.method(globalThis, 'fetch', async () => + createMockSseResponse([ + 'data: {"choices":[{"delta":{"content":"New"},"finish_reason":"stop"}]}\n\n', + ]), + ) + + await generateAnswersWithOpenAiApiCompat( + 'https://api.example.com/v1', + port, + 'CurrentQ', + session, + 'sk-test', + ) + + assert.deepEqual(session.conversationRecords, [ + { question: 'CurrentQ', answer: 'New', images: [image] }, + ]) +}) diff --git a/tests/unit/services/wrappers-register.test.mjs b/tests/unit/services/wrappers-register.test.mjs index cfee3e4f7..5cf15047b 100644 --- a/tests/unit/services/wrappers-register.test.mjs +++ b/tests/unit/services/wrappers-register.test.mjs @@ -116,6 +116,34 @@ test('registerPortListener calls executor with session, port, and config', async assert.equal(port.postedMessages[0].session, result.session) }) +test('registerPortListener redacts image payloads before debug logging', async (t) => { + const debug = t.mock.fn() + t.mock.method(console, 'debug', debug) + setStorage({ modelName: 'chatgptApi4oMini' }) + + let resolveExec + const execDone = new Promise((resolve) => { + resolveExec = resolve + }) + registerPortListener(async () => resolveExec()) + const port = createFakePort() + triggerConnect(port) + port.emitMessage({ + session: { + conversationRecords: [], + images: ['data:image/png;base64,iVBORw0KGgo='], + }, + }) + await execDone + + const received = debug.mock.calls.find( + ({ arguments: args }) => args[0] === 'received msg (redacted)', + ) + assert.ok(received) + assert.equal(received.arguments[1].session, 'REDACTED') + assert.doesNotMatch(JSON.stringify(received.arguments), /iVBORw0KGgo/) +}) + test('registerPortListener scopes error translations to each request', async (t) => { t.mock.method(console, 'debug', () => {}) t.mock.method(console, 'error', () => {}) diff --git a/tests/unit/utils/basic-guards.test.mjs b/tests/unit/utils/basic-guards.test.mjs index d563cce73..c6d0d8a95 100644 --- a/tests/unit/utils/basic-guards.test.mjs +++ b/tests/unit/utils/basic-guards.test.mjs @@ -102,3 +102,13 @@ test('getConversationPairs keeps empty question and answer strings unchanged', ( { role: 'assistant', content: '' }, ]) }) + +test('generic conversation history keeps image-bearing records as text', () => { + assert.deepEqual( + getConversationPairs([{ question: 'Q', answer: 'A', images: ['data:image/png;base64,AAEC'] }]), + [ + { role: 'user', content: 'Q' }, + { role: 'assistant', content: 'A' }, + ], + ) +})