Cut streaming render cost

src/components/ConversationCard/answer-buffer.mjs

@@ -0,0 +1,71 @@
+/**
+ * Animation frames are the fastest way to coalesce a burst of updates, but not every host
+ * has them (jsdom and other headless renderers do not). Fall back to a timer there so the
+ * buffer can always be created.
+ * @param {typeof globalThis} [host]
+ * @returns {{requestFrame: (callback: () => void) => unknown, cancelFrame: (handle: unknown) => void}}
+ */
+export function createFrameScheduler(host = globalThis) {
+  if (
+    typeof host.requestAnimationFrame === 'function' &&
+    typeof host.cancelAnimationFrame === 'function'
+  ) {
+    return {
+      requestFrame: (callback) => host.requestAnimationFrame(callback),
+      cancelFrame: (handle) => host.cancelAnimationFrame(handle),
+    }
+  }
+  return {
+    requestFrame: (callback) => setTimeout(callback, 16),
+    cancelFrame: (handle) => clearTimeout(handle),
+  }
+}
+
+/**
+ * Coalesces streamed answer text so a burst of chunks renders once per frame instead of
+ * once per chunk, while never losing the newest text.
+ * @param {object} params
+ * @param {(callback: () => void) => unknown} params.requestFrame
+ * @param {(handle: unknown) => void} params.cancelFrame
+ * @param {(answer: string) => void} params.render
+ */
+export function createAnswerBuffer({ requestFrame, cancelFrame, render }) {
+  let pending = null
+  let frame = null
+
+  const cancelPendingFrame = () => {
+    if (frame === null) return
+    cancelFrame(frame)
+    frame = null
+  }
+
+  const takePending = () => {
+    const answer = pending
+    pending = null
+    return answer
+  }
+
+  return {
+    /** Queue the newest answer, scheduling a render only when none is already scheduled. */
+    push(answer) {
+      pending = answer
+      if (frame !== null) return
+      frame = requestFrame(() => {
+        frame = null
+        const latest = takePending()
+        if (latest !== null) render(latest)
+      })
+    },
+    /** Render the newest answer before the conversation is finalized. */
+    flush() {
+      cancelPendingFrame()
+      const latest = takePending()
+      if (latest !== null) render(latest)
+    },
+    /** Drop the newest answer without rendering it, e.g. when a retry starts. */
+    discard() {
+      cancelPendingFrame()
+      pending = null
+    },
+  }
+}

src/components/ConversationCard/index.jsx

@@ -50,6 +50,7 @@ import {
   isSupersededGenerationMessage,
   isSupersededRequestMessage,
 } from './session.mjs'
+import { createAnswerBuffer, createFrameScheduler } from './answer-buffer.mjs'
 
 const logo = Browser.runtime.getURL('logo.png')
 const UNMATCHED_API_MODE_VALUE = '__current-session-api-mode__'
@@ -187,6 +188,7 @@ function ConversationCard(props) {
       const newSession = initSession({ ...session, question: props.question })
       partialAnswerRef.current = ''
       retryRecordRef.current = null
+      answerBufferRef.current.discard()
       setSession(newSession)
       await postMessage({ session: newSession })
     }
@@ -221,19 +223,33 @@ function ConversationCard(props) {
     })
   }
 
+  const answerBufferRef = useRef(null)
+  if (answerBufferRef.current === null) {
+    answerBufferRef.current = createAnswerBuffer({
+      ...createFrameScheduler(),
+      render: (answer) => updateAnswer(answer, false, 'answer'),
+    })
+  }
+
+  // A buffered frame can outlive a hidden page, so drop it when the card goes away.
+  useEffect(() => {
+    return () => answerBufferRef.current?.discard()
+  }, [])
+
   const portMessageListener = (msg) => {
     if (disposedRef.current) return
     if (isSupersededRequestMessage(msg, requestGenerationIdRef.current)) return
     if (isSupersededGenerationMessage(msg, retryGenerationIdRef.current)) return
 
     if (msg.answer) {
       partialAnswerRef.current = msg.answer
-      updateAnswer(msg.answer, false, 'answer')
+      answerBufferRef.current.push(msg.answer)
     }
     if (msg.session) {
       setSession(msg.done ? { ...msg.session, isRetry: false } : msg.session)
     }
     if (msg.done) {
+      answerBufferRef.current.flush()
       const partialAnswer = partialAnswerRef.current
       const retryRecord = retryRecordRef.current
       const completionState = getInterruptedCompletionState(msg, partialAnswer, retryRecord)
@@ -249,6 +265,7 @@ function ConversationCard(props) {
       setIsReady(true)
     }
     if (msg.error) {
+      answerBufferRef.current.flush()
       const retryRecord = retryRecordRef.current
       setSession((currentSession) => finalizeInterruptedSession(currentSession, '', retryRecord))
       switch (msg.error) {
@@ -431,10 +448,17 @@ function ConversationCard(props) {
         return
       }
       if (disposedRef.current) return
+      // A dropped transport ends the stream without a final message, so flush here: the newest
+      // chunk still renders on a hidden page, where animation frames are paused. A foreground
+      // generation (Bing web) streams through its own transport, though, so this keepalive
+      // port dropping must not unlock sending.
+      if (foregroundPortsRef.current.size === 0) {
+        answerBufferRef.current.flush()
+        setIsReady(true)
+      }
       const nextPort = Browser.runtime.connect()
       portRef.current = nextPort
       setPort(nextPort)
-      setIsReady(true)
     }
 
     const closeChatsMessageListener = (message) => {
@@ -479,6 +503,7 @@ function ConversationCard(props) {
   }, [port, conversationItemData])
 
   const getRetryFn = (session) => async () => {
+    answerBufferRef.current.discard()
     updateAnswer(`<p class="gpt-loading">${t('Waiting for response...')}</p>`, false, 'answer')
     setIsReady(false)
 
@@ -671,6 +696,7 @@ function ConversationCard(props) {
               }
               partialAnswerRef.current = ''
               retryRecordRef.current = null
+              answerBufferRef.current.discard()
               Browser.runtime.sendMessage({
                 type: 'DELETE_CONVERSATION',
                 data: {
@@ -797,6 +823,7 @@ function ConversationCard(props) {
             )
             partialAnswerRef.current = ''
             retryRecordRef.current = null
+            answerBufferRef.current.discard()
             setConversationItemData([...conversationItemData, newQuestion, newAnswer])
             setIsReady(false)
 

src/components/MarkdownRender/highlight-options.mjs

@@ -0,0 +1,52 @@
+/**
+ * Shared rehype-highlight options.
+ *
+ * Auto-detection compiles every registered grammar the first time it runs, which costs
+ * >100ms on a cold page. Scanning a curated subset keeps that near a few milliseconds.
+ * Explicitly labelled code blocks are unaffected: the subset only narrows automatic
+ * detection, they still use every grammar lowlight has registered.
+ *
+ * The subset is lowlight's registered set minus entries that only mislead detection:
+ * `shell` (the Shell Session console-prompt grammar, not a `bash` alias; `bash` aliases
+ * only `sh`/`zsh`), `plaintext`/`python-repl`/`php-template` (not useful for detection),
+ * and `arduino`/`objectivec`/`vbnet`/`wasm` (rare in answers, and they win ambiguous `c`,
+ * `cpp`, `ini` and `sql` matches away from the right language). Keep the
+ * order as lowlight registers them: `highlightAuto` settles equal-relevance candidates by
+ * subset order, so reordering this list changes which language an ambiguous block detects as.
+ */
+export const highlightOptions = {
+  detect: true,
+  subset: [
+    'bash',
+    'c',
+    'cpp',
+    'csharp',
+    'css',
+    'diff',
+    'go',
+    'graphql',
+    'ini',
+    'java',
+    'javascript',
+    'json',
+    'kotlin',
+    'less',
+    'lua',
+    'makefile',
+    'markdown',
+    'perl',
+    'php',
+    'python',
+    'r',
+    'ruby',
+    'rust',
+    'scss',
+    'sql',
+    'swift',
+    'typescript',
+    'xml',
+    'yaml',
+  ],
+  ignoreMissing: true,
+  plainText: ['diagnostic'],
+}

src/components/MarkdownRender/markdown-without-katex.jsx

@@ -5,6 +5,7 @@ import remarkGfm from 'remark-gfm'
 import remarkBreaks from 'remark-breaks'
 import { Pre } from './Pre'
 import { Hyperlink } from './Hyperlink'
+import { highlightOptions } from './highlight-options.mjs'
 import { memo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 
@@ -176,17 +177,7 @@ export function MarkdownRender(props) {
         ]}
         unwrapDisallowed={true}
         remarkPlugins={[remarkGfm, remarkBreaks]}
-        rehypePlugins={[
-          rehypeRaw,
-          [
-            rehypeHighlight,
-            {
-              detect: true,
-              ignoreMissing: true,
-              plainText: ['diagnostic'],
-            },
-          ],
-        ]}
+        rehypePlugins={[rehypeRaw, [rehypeHighlight, highlightOptions]]}
         components={{
           a: Hyperlink,
           pre: Pre,

src/components/MarkdownRender/markdown.jsx

@@ -8,6 +8,7 @@ import remarkGfm from 'remark-gfm'
 import remarkBreaks from 'remark-breaks'
 import { Pre } from './Pre'
 import { Hyperlink } from './Hyperlink'
+import { highlightOptions } from './highlight-options.mjs'
 import { memo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 
@@ -179,18 +180,7 @@ export function MarkdownRender(props) {
         ]}
         unwrapDisallowed={true}
         remarkPlugins={[remarkMath, remarkGfm, remarkBreaks]}
-        rehypePlugins={[
-          rehypeKatex,
-          rehypeRaw,
-          [
-            rehypeHighlight,
-            {
-              detect: true,
-              ignoreMissing: true,
-              plainText: ['diagnostic'],
-            },
-          ],
-        ]}
+        rehypePlugins={[rehypeKatex, rehypeRaw, [rehypeHighlight, highlightOptions]]}
         components={{
           a: Hyperlink,
           pre: Pre,

tests/setup/conversation-card-lifecycle-loader-hooks.mjs

@@ -40,7 +40,14 @@ const sources = {
       return null
     }
   `,
-  'test:conversation-item': 'export default function ConversationItem() { return null }',
+  'test:conversation-item': `
+    export default function ConversationItem(props) {
+      if (props.type === 'answer') {
+        globalThis.__CONVERSATION_LIFECYCLE_TEST__.answerContents.push(props.content)
+      }
+      return null
+    }
+  `,
   'test:conversation-utils': `
     export const apiModeToModelName = () => 'test-model'
     export const createElementAtPosition = () => document.createElement('div')