diff --git a/electron/ipc/ask-code-minimax.test.ts b/electron/ipc/ask-code-minimax.test.ts index 8a73d51a..e3a35df8 100644 --- a/electron/ipc/ask-code-minimax.test.ts +++ b/electron/ipc/ask-code-minimax.test.ts @@ -1,4 +1,7 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; // Mock fetch globally const mockFetch = vi.fn(); @@ -7,7 +10,9 @@ vi.stubGlobal('fetch', mockFetch); import { askAboutCodeMinimax, cancelAskAboutCodeMinimax, + MINIMAX_IMAGE_INPUT_MODEL, MINIMAX_MODEL, + minimaxModelAcceptsImages, setMinimaxApiKey, } from './ask-code-minimax.js'; @@ -261,6 +266,135 @@ describe('askAboutCodeMinimax', () => { }); }); +describe('minimax image input', () => { + const pngPath = path.join(os.tmpdir(), 'parallel-code-ask-code-test.png'); + const pngBytes = Buffer.from('89504e470d0a1a0a', 'hex'); + + beforeAll(() => { + fs.writeFileSync(pngPath, pngBytes); + }); + + afterAll(() => { + fs.rmSync(pngPath, { force: true }); + }); + + beforeEach(() => { + vi.clearAllMocks(); + setMinimaxApiKey('test-key'); + }); + + function requestBody() { + return JSON.parse((mockFetch.mock.calls[0][1] as RequestInit).body as string) as { + model: string; + messages: Array<{ + role: string; + content: string | Array<{ type: string; text?: string; image_url?: { url: string } }>; + }>; + }; + } + + it('reports which models accept image input', () => { + expect(minimaxModelAcceptsImages(MINIMAX_IMAGE_INPUT_MODEL)).toBe(true); + expect(minimaxModelAcceptsImages(MINIMAX_MODEL)).toBe(false); + expect(minimaxModelAcceptsImages('nope')).toBe(false); + }); + + it('keeps the user content a plain string when no image is attached', async () => { + const { win, messages } = makeMockWin(); + + mockFetch.mockResolvedValueOnce(makeStreamResponse('data: [DONE]\n\n')); + + askAboutCodeMinimax(win, { + requestId: 'img-none', + channelId: 'ch-img-none', + prompt: 'Explain this', + }); + + await waitForDone(messages); + + const body = requestBody(); + expect(body.model).toBe(MINIMAX_MODEL); + expect(body.messages.find((m) => m.role === 'user')?.content).toBe('Explain this'); + }); + + it('sends attached images as content parts to an image-capable model', async () => { + const { win, messages } = makeMockWin(); + + mockFetch.mockResolvedValueOnce(makeStreamResponse('data: [DONE]\n\n')); + + askAboutCodeMinimax(win, { + requestId: 'img-one', + channelId: 'ch-img-one', + prompt: 'What does this screenshot show?', + imagePaths: [pngPath], + }); + + await waitForDone(messages); + + const body = requestBody(); + expect(body.model).toBe(MINIMAX_IMAGE_INPUT_MODEL); + + const userContent = body.messages.find((m) => m.role === 'user')?.content; + expect(Array.isArray(userContent)).toBe(true); + const parts = userContent as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(parts[0]).toEqual({ type: 'text', text: 'What does this screenshot show?' }); + expect(parts[1].type).toBe('image_url'); + expect(parts[1].image_url?.url).toBe(`data:image/png;base64,${pngBytes.toString('base64')}`); + + // The system message stays a plain string + expect(typeof body.messages.find((m) => m.role === 'system')?.content).toBe('string'); + }); + + it('rejects image types the chat API cannot accept', () => { + const { win } = makeMockWin(); + + expect(() => + askAboutCodeMinimax(win, { + requestId: 'img-bad-type', + channelId: 'ch-img-bad-type', + prompt: 'Test', + imagePaths: [path.join(os.tmpdir(), 'notes.txt')], + }), + ).toThrow(/Unsupported image type/); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('rejects more images than a single question allows', () => { + const { win } = makeMockWin(); + + expect(() => + askAboutCodeMinimax(win, { + requestId: 'img-too-many', + channelId: 'ch-img-too-many', + prompt: 'Test', + imagePaths: [pngPath, pngPath, pngPath, pngPath, pngPath], + }), + ).toThrow(/Too many images/); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('reports an unreadable image as an error instead of sending a request', async () => { + const { win, messages } = makeMockWin(); + + askAboutCodeMinimax(win, { + requestId: 'img-missing', + channelId: 'ch-img-missing', + prompt: 'Test', + imagePaths: [path.join(os.tmpdir(), 'parallel-code-does-not-exist.png')], + }); + + await waitForDone(messages); + + const errors = messages.filter((m) => (m as Record).type === 'error'); + expect(errors).toHaveLength(1); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); + describe('cancelAskAboutCodeMinimax', () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/electron/ipc/ask-code-minimax.ts b/electron/ipc/ask-code-minimax.ts index 0f262296..21251d44 100644 --- a/electron/ipc/ask-code-minimax.ts +++ b/electron/ipc/ask-code-minimax.ts @@ -1,3 +1,5 @@ +import fs from 'fs'; +import path from 'path'; import type { BrowserWindow } from 'electron'; import { debug as logDebug } from '../log.js'; import { @@ -13,11 +15,106 @@ interface MinimaxAskCodeRequest { requestId: string; channelId: string; prompt: string; + /** + * Absolute paths of images to send alongside the prompt. The app already + * resolves pasted and dropped images to temp files, so a request carries + * those paths rather than the bytes themselves. + */ + imagePaths?: string[]; } const MINIMAX_API_URL = 'https://api.minimax.io/v1/chat/completions'; export const MINIMAX_MODEL = 'MiniMax-M2.7'; +/** Model used when a request carries image input. */ +export const MINIMAX_IMAGE_INPUT_MODEL = 'MiniMax-M3'; + +/** Input modalities a MiniMax model accepts. */ +export type MinimaxInputModality = 'text' | 'image' | 'video'; + +/** Input modalities per model, as published in the provider model catalog. */ +const MINIMAX_INPUT_MODALITIES: Readonly> = { + [MINIMAX_IMAGE_INPUT_MODEL]: ['text', 'image', 'video'], + [MINIMAX_MODEL]: ['text'], +}; + +/** Whether a model accepts image input according to the catalog. */ +export function minimaxModelAcceptsImages(modelId: string): boolean { + return MINIMAX_INPUT_MODALITIES[modelId]?.includes('image') ?? false; +} + +/** Image input is capped separately from the prompt: the bytes never count against it. */ +const MAX_IMAGES_PER_REQUEST = 4; +const MAX_IMAGE_BYTES = 10 * 1024 * 1024; + +const IMAGE_MIME_TYPES: Readonly> = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.gif': 'image/gif', +}; + +/** A chat message content part in the chat completions request schema. */ +type MinimaxContentPart = + | { type: 'text'; text: string } + | { type: 'image_url'; image_url: { url: string } }; + +/** + * Picks the model for a request. Image input requires a model whose catalog + * modalities include images, so image requests fall back to the image-capable + * model whenever the text default cannot accept them. + */ +function resolveModel(hasImages: boolean): string { + if (!hasImages || minimaxModelAcceptsImages(MINIMAX_MODEL)) return MINIMAX_MODEL; + return MINIMAX_IMAGE_INPUT_MODEL; +} + +/** Rejects unsupported or oversized image input before a request is started. */ +function assertImagesSupported(imagePaths: string[]): void { + if (imagePaths.length > MAX_IMAGES_PER_REQUEST) { + throw new Error( + `Too many images (${imagePaths.length}, max ${MAX_IMAGES_PER_REQUEST} per question)`, + ); + } + for (const imagePath of imagePaths) { + if (!IMAGE_MIME_TYPES[path.extname(imagePath).toLowerCase()]) { + throw new Error(`Unsupported image type: ${path.basename(imagePath)}`); + } + } +} + +/** Reads an image from disk into the data URL form the chat API expects. */ +async function imageDataUrl(imagePath: string): Promise { + const mimeType = IMAGE_MIME_TYPES[path.extname(imagePath).toLowerCase()]; + if (!mimeType) throw new Error(`Unsupported image type: ${path.basename(imagePath)}`); + + const bytes = await fs.promises.readFile(imagePath); + if (bytes.byteLength > MAX_IMAGE_BYTES) { + throw new Error( + `Image too large: ${path.basename(imagePath)} (${bytes.byteLength} bytes, max ${MAX_IMAGE_BYTES})`, + ); + } + return `data:${mimeType};base64,${bytes.toString('base64')}`; +} + +/** + * Builds the user message content: a plain string while the question is text + * only, and text plus image parts once images are attached. + */ +async function buildUserContent( + prompt: string, + imagePaths: string[], +): Promise { + if (imagePaths.length === 0) return prompt; + + const parts: MinimaxContentPart[] = [{ type: 'text', text: prompt }]; + for (const imagePath of imagePaths) { + parts.push({ type: 'image_url', image_url: { url: await imageDataUrl(imagePath) } }); + } + return parts; +} + const activeRequests = new RequestRegistry({ maxConcurrent: ASK_CODE_MAX_CONCURRENT, timeoutMs: ASK_CODE_TIMEOUT_MS, @@ -32,6 +129,7 @@ export function setMinimaxApiKey(key: string): void { export function askAboutCodeMinimax(win: BrowserWindow, args: MinimaxAskCodeRequest): void { const { requestId, channelId, prompt } = args; + const imagePaths = args.imagePaths ?? []; const apiKey = storedApiKey; if (!apiKey) { @@ -39,10 +137,13 @@ export function askAboutCodeMinimax(win: BrowserWindow, args: MinimaxAskCodeRequ } assertPromptWithinLimit(prompt); + assertImagesSupported(imagePaths); assertCanStart(activeRequests, requestId); cancelAskAboutCodeMinimax(requestId); + const model = resolveModel(imagePaths.length > 0); + const controller = new AbortController(); const send = (msg: unknown) => { @@ -55,28 +156,31 @@ export function askAboutCodeMinimax(win: BrowserWindow, args: MinimaxAskCodeRequ request.abort(), ); - fetch(MINIMAX_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model: MINIMAX_MODEL, - messages: [ - { - role: 'system', - content: 'Answer concisely about the selected code. Use markdown.', + buildUserContent(prompt, imagePaths) + .then((content) => + fetch(MINIMAX_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, }, - { role: 'user', content: prompt }, - ], - // MiniMax temperature must be in (0.0, 1.0] - temperature: 0.3, - max_tokens: 2048, - stream: true, - }), - signal: controller.signal, - }) + body: JSON.stringify({ + model, + messages: [ + { + role: 'system', + content: 'Answer concisely about the selected code. Use markdown.', + }, + { role: 'user', content }, + ], + // MiniMax temperature must be in (0.0, 1.0] + temperature: 0.3, + max_tokens: 2048, + stream: true, + }), + signal: controller.signal, + }), + ) .then(async (res) => { if (!res.ok || !res.body) { const text = await res.text().catch(() => `HTTP ${res.status}`); diff --git a/electron/ipc/ask-code.ts b/electron/ipc/ask-code.ts index e8bcc568..9c5a952b 100644 --- a/electron/ipc/ask-code.ts +++ b/electron/ipc/ask-code.ts @@ -26,6 +26,8 @@ interface AskCodeRequest { provider?: AskCodeProvider; /** Env file configured for the Claude Code agent, if any. */ envFile?: string; + /** Absolute paths of images attached to the question. */ + imagePaths?: string[]; } const activeRequests = new RequestRegistry({ @@ -34,12 +36,13 @@ const activeRequests = new RequestRegistry({ }); export function askAboutCode(win: BrowserWindow, args: AskCodeRequest): void { - const { requestId, channelId, prompt, cwd, provider, envFile } = args; + const { requestId, channelId, prompt, cwd, provider, envFile, imagePaths } = args; - // Route to MiniMax backend when configured + // Route to MiniMax backend when configured. Image input is wired for this + // provider only; the CLI path below takes a text prompt. if (provider === 'minimax') { activeRequests.cancel(requestId); - askAboutCodeMinimax(win, { requestId, channelId, prompt }); + askAboutCodeMinimax(win, { requestId, channelId, prompt, imagePaths }); return; } diff --git a/electron/ipc/register.ts b/electron/ipc/register.ts index ab8d213f..591421b0 100644 --- a/electron/ipc/register.ts +++ b/electron/ipc/register.ts @@ -900,6 +900,13 @@ export function registerAllHandlers(win: BrowserWindow): void { const provider: string | undefined = typeof args.provider === 'string' ? args.provider : undefined; assertOptionalString(args.envFile, 'envFile'); + const rawImagePaths: unknown = args.imagePaths; + let imagePaths: string[] | undefined; + if (rawImagePaths !== undefined) { + assertStringArray(rawImagePaths, 'imagePaths'); + for (const imagePath of rawImagePaths) validatePath(imagePath, 'imagePath'); + imagePaths = rawImagePaths; + } askAboutCode(win, { requestId: args.requestId, channelId: args.onOutput.__CHANNEL_ID__, @@ -907,6 +914,7 @@ export function registerAllHandlers(win: BrowserWindow): void { cwd: args.cwd, provider: provider === 'minimax' ? 'minimax' : 'claude', envFile: args.envFile, + imagePaths, }); }); diff --git a/src/components/AskCodeCard.tsx b/src/components/AskCodeCard.tsx index 6f2ca764..467c4e6d 100644 --- a/src/components/AskCodeCard.tsx +++ b/src/components/AskCodeCard.tsx @@ -14,6 +14,8 @@ interface AskCodeCardProps { endLine: number; selectedText: string; worktreePath: string; + /** Absolute paths of images attached to the question, if any. */ + imagePaths?: string[]; onDismiss: () => void; } @@ -65,6 +67,7 @@ export function AskCodeCard(props: AskCodeCardProps) { onOutput: channel, provider: store.askCodeProvider, envFile: store.agentEnvFiles['claude-code'], + imagePaths: props.imagePaths?.length ? props.imagePaths : undefined, }).catch((err: unknown) => { setError(errMessage(err)); setLoading(false); diff --git a/src/components/InlineInput.tsx b/src/components/InlineInput.tsx index 84ad8dc0..7d67457d 100644 --- a/src/components/InlineInput.tsx +++ b/src/components/InlineInput.tsx @@ -1,18 +1,32 @@ -import { createSignal, onCleanup, onMount } from 'solid-js'; +import { createSignal, onCleanup, onMount, Show } from 'solid-js'; import { theme } from '../lib/theme'; import { sf } from '../lib/fontScale'; +import { invoke } from '../lib/ipc'; +import { IPC } from '../../electron/ipc/channels'; +import { store } from '../store/store'; +import { warn as logWarn } from '../lib/log'; import type { DiffInteractionMode } from './review-types'; interface InlineInputProps { - onSubmit: (text: string, mode: DiffInteractionMode) => void; + onSubmit: (text: string, mode: DiffInteractionMode, imagePaths?: string[]) => void; onDismiss: () => void; } +/** Shape of the resolved clipboard content returned by the main process. */ +interface ResolvedPaste { + kind: string; + path?: string; +} + export function InlineInput(props: InlineInputProps) { const [text, setText] = createSignal(''); const [mode, setMode] = createSignal('review'); + const [imagePaths, setImagePaths] = createSignal([]); let inputRef: HTMLInputElement | undefined; + /** Images are only sent to a provider whose model accepts image input. */ + const imageInputEnabled = () => mode() === 'ask' && store.askCodeProvider === 'minimax'; + onMount(() => { requestAnimationFrame(() => inputRef?.focus()); const onGlobalKeyDown = (e: KeyboardEvent) => { @@ -32,7 +46,32 @@ export function InlineInput(props: InlineInputProps) { function submit() { const t = text().trim(); - if (t) props.onSubmit(t, mode()); + if (!t) return; + props.onSubmit(t, mode(), imageInputEnabled() ? imagePaths() : undefined); + } + + /** + * Attaches a pasted image to the question. The main process already turns + * clipboard images into temp files, so the same path is reused here. + */ + function handlePaste(e: ClipboardEvent) { + if (!imageInputEnabled()) return; + const hasImage = Array.from(e.clipboardData?.items ?? []).some((item) => + item.type.startsWith('image/'), + ); + if (!hasImage) return; + + e.preventDefault(); + invoke(IPC.ResolveClipboardPaste) + .then((paste) => { + if (paste.kind === 'image' && paste.path) { + const attached = paste.path; + setImagePaths((prev) => (prev.includes(attached) ? prev : [...prev, attached])); + } + }) + .catch((err: unknown) => { + logWarn('askCode.paste', 'ResolveClipboardPaste failed', { err }); + }); } function handleKeyDown(e: KeyboardEvent) { @@ -107,6 +146,7 @@ export function InlineInput(props: InlineInputProps) { value={text()} onInput={(e) => setText(e.currentTarget.value)} onKeyDown={handleKeyDown} + onPaste={handlePaste} style={{ flex: '1', background: theme.bgInput, @@ -120,6 +160,27 @@ export function InlineInput(props: InlineInputProps) { }} /> + {/* Attached images */} + 0}> + + + {/* Submit button */}