From 6bf66bef9964e04fdd5e9c44d7d9d8eb3f283153 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 26 Oct 2026 10:57:34 +0100 Subject: [PATCH 1/2] feat(ai-components): add useHeardText for voice-paced text Ports GetStream/stream-chat-react#3343 and GetStream/stream-chat-react#3344 to v15: a standalone hook (React only, no chat SDK imports) that reveals a text at the pace a voice agent speaks it, with its tests, the ai-components export and README docs. --- src/plugins/AIComponents/README.md | 64 ++- .../__tests__/useHeardText.test.ts | 369 ++++++++++++++++++ .../AIComponents/hooks/useHeardText.ts | 261 +++++++++++++ src/plugins/AIComponents/index.ts | 1 + 4 files changed, 692 insertions(+), 3 deletions(-) create mode 100644 src/plugins/AIComponents/__tests__/useHeardText.test.ts create mode 100644 src/plugins/AIComponents/hooks/useHeardText.ts diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index 272f64307..b6b92e634 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -1,6 +1,6 @@ # AI components (`stream-chat-react/ai-components`) -AI-first building blocks for Stream Chat: a message composer with attachments and speech-to-text, a Markdown renderer with syntax highlighting and Chart.js support, a typewriter-style streaming message and an "AI is thinking" indicator. +AI-first building blocks for Stream Chat: a message composer with attachments and speech-to-text, a Markdown renderer with syntax highlighting and Chart.js support, a typewriter-style streaming message, a hook that reveals text at the pace a voice agent speaks it, and an "AI is thinking" indicator. These components used to ship as the separate package `@stream-io/chat-react-ai`. They now live in `stream-chat-react` behind the optional entry point `stream-chat-react/ai-components`, with a stylesheet at `stream-chat-react/css/ai-components.css`. The main entry point (`stream-chat-react`) never imports from this plugin, so apps that do not use it pay nothing for it. @@ -32,8 +32,8 @@ Load `ai-components.css` in addition to `stream-chat-react/css/index.css`. From `stream-chat-react/ai-components`: - Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator` -- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText` -- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions` +- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useHeardText` +- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `HeardTextProgress`, `HeardTextFunction`, `HeardTextSubscribe`, `UseHeardTextParams`, `UseHeardTextResult` `useMessageTextStreaming` is not exported from here; import it from the main entry point (`stream-chat-react`). @@ -543,6 +543,64 @@ function VoiceInput() { } ``` +### `useHeardText` + +Reveals a text at the pace the user hears it spoken, for apps that talk to a voice AI agent: the agent's reply is written into the chat as the model generates it, far ahead of the agent's voice. The hook is standalone (React only, no chat SDK imports) and takes the speech progress from the app as a plain function, so it works with any voice backend. + +```ts +type HeardTextProgress = { chars: number; done: boolean }; +type HeardTextFunction = (now: number) => HeardTextProgress | undefined; +type HeardTextSubscribe = (listener: () => void) => () => void; + +const useHeardText: (params: UseHeardTextParams) => UseHeardTextResult; +``` + +`heard(now)` returns how many characters of the text the user has heard by `now` (a `performance.now()` timestamp) and whether the speech for it has finished. `chars` counts Unicode code points (`Array.from(text).length`), not UTF-16 code units. Return `undefined` when the text is not paced, and `{ chars: 0, done: false }` for a spoken text whose audio has not started yet. + +#### Props + +| Name | Type | Required | Description | +| ------------ | -------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | +| `text` | `string` | yes | The full text to reveal. | +| `heard` | `HeardTextFunction` | yes | The speech progress of `text`. | +| `subscribe` | `HeardTextSubscribe` | no | Change notifications from the speech progress source; each one makes the hook check `heard` again. | +| `generating` | `boolean` | no | `true` while the text is still being written. The full text is revealed once `heard` reports `done` and this is falsy. | +| `live` | `boolean` | no | Defaults to `true`. `false` reveals the full text at once, for example for a message loaded from history. Read at mount. | + +#### Returns + +| Name | Type | Description | +| ----------- | ------------ | ---------------------------------------------------------------------------------- | +| `heardText` | `string` | The part of the text the user has heard (the full text when not paced). | +| `isPaced` | `boolean` | `false` when `heard` returned `undefined`; `heardText` is then the full text. | +| `skip` | `() => void` | Reveal the full text from now on, for example when the user stops the AI response. | + +Behaviour: + +- It reveals the first `floor(chars)` code points of `text` and never moves backwards. Emoji and other astral characters are never split. +- While the text is paced and its speech is not done, it samples `heard` on every animation frame, and stops when the speech is done or the component unmounts. `subscribe` notifications and changes of `text` or `generating` make it check again. +- If `heard` returns `undefined` at first and a value later, the reveal continues from the length already shown. +- `heard` and `subscribe` are read through refs: passing new functions on every render (inline arrow functions) does not restart the pacing. `subscribe` is called once and unsubscribed on unmount. + +#### Example + +```tsx +import { useHeardText } from 'stream-chat-react/ai-components'; + +function SpokenReply({ generating, session, text, turnId }) { + const { heardText } = useHeardText({ + generating, + heard: (now) => session.speech.heard(turnId, now), + subscribe: (listener) => session.speech.subscribe(listener), + text, + }); + + return

{heardText}

; +} +``` + +A drop-in for the `StreamedMessageText` component slot of `stream-chat-react`, built on this hook, is shown in the docs. + ## Examples ### Basic Chat Interface diff --git a/src/plugins/AIComponents/__tests__/useHeardText.test.ts b/src/plugins/AIComponents/__tests__/useHeardText.test.ts new file mode 100644 index 000000000..4c72494f3 --- /dev/null +++ b/src/plugins/AIComponents/__tests__/useHeardText.test.ts @@ -0,0 +1,369 @@ +import { act, renderHook } from '@testing-library/react'; + +import { useHeardText } from '..'; +import type { + HeardTextFunction, + HeardTextProgress, + HeardTextSubscribe, + UseHeardTextParams, +} from '..'; + +const TEXT = 'Hello there, how can I help you today?'; +const FRAME_MS = 16; + +const advance = (ms: number) => + act(() => { + vi.advanceTimersByTime(ms); + }); + +const renderHeardText = (initialProps: UseHeardTextParams) => + renderHook((props: UseHeardTextParams) => useHeardText(props), { initialProps }); + +describe('useHeardText', () => { + beforeEach(() => { + vi.useFakeTimers({ + toFake: [ + 'setTimeout', + 'clearTimeout', + 'setInterval', + 'clearInterval', + 'requestAnimationFrame', + 'cancelAnimationFrame', + 'performance', + 'Date', + ], + }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('reveals the text at the pace it is heard', () => { + const start = performance.now(); + // 20 characters per second + const heard = vi.fn((now) => ({ + chars: ((now - start) / 1000) * 20, + done: false, + })); + const { result } = renderHeardText({ heard, text: TEXT }); + + expect(result.current.heardText).toBe(''); + expect(result.current.isPaced).toBe(true); + + // requestAnimationFrame fires every 16 ms under fake timers + advance(512); + expect(result.current.heardText).toBe(TEXT.slice(0, 10)); + + advance(512); + expect(result.current.heardText).toBe(TEXT.slice(0, 20)); + + advance(5000); + // clamped to the text length, the full text while speech is still in progress + expect(result.current.heardText).toBe(TEXT); + expect(heard).toHaveBeenCalledWith(expect.any(Number)); + }); + + it('floors fractional character counts', () => { + const { result } = renderHeardText({ + heard: () => ({ chars: 4.9, done: false }), + text: TEXT, + }); + expect(result.current.heardText).toBe(TEXT.slice(0, 4)); + }); + + it('treats non-finite and negative counts as nothing heard', () => { + let progress: HeardTextProgress = { chars: Number.NaN, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: TEXT }); + expect(result.current.heardText).toBe(''); + + progress = { chars: -3, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe(''); + }); + + it('never moves the reveal backwards', () => { + let progress: HeardTextProgress = { chars: 12, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: TEXT }); + expect(result.current.heardText).toBe(TEXT.slice(0, 12)); + + progress = { chars: 3, done: false }; + advance(FRAME_MS * 3); + expect(result.current.heardText).toBe(TEXT.slice(0, 12)); + + progress = { chars: 15, done: false }; + advance(FRAME_MS * 2); + expect(result.current.heardText).toBe(TEXT.slice(0, 15)); + }); + + it('reveals the full text once the speech is done and the text is not generating', () => { + let progress: HeardTextProgress = { chars: 5, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: TEXT }); + expect(result.current.heardText).toBe(TEXT.slice(0, 5)); + + progress = { chars: 8, done: true }; + advance(FRAME_MS * 2); + expect(result.current.heardText).toBe(TEXT); + }); + + it('keeps pacing a done text while it is still generating', () => { + const heard: HeardTextFunction = () => ({ chars: 8, done: true }); + const { rerender, result } = renderHeardText({ generating: true, heard, text: TEXT }); + expect(result.current.heardText).toBe(TEXT.slice(0, 8)); + + rerender({ generating: false, heard, text: TEXT }); + expect(result.current.heardText).toBe(TEXT); + }); + + it('stops ticking once the speech is done', () => { + const heard = vi.fn(() => ({ chars: 8, done: true })); + renderHeardText({ generating: true, heard, text: TEXT }); + const callCount = heard.mock.calls.length; + + advance(FRAME_MS * 20); + expect(heard).toHaveBeenCalledTimes(callCount); + }); + + it('stops ticking on unmount', () => { + const heard = vi.fn(() => ({ chars: 1, done: false })); + const { unmount } = renderHeardText({ heard, text: TEXT }); + advance(FRAME_MS * 3); + unmount(); + const callCount = heard.mock.calls.length; + + advance(FRAME_MS * 20); + expect(heard).toHaveBeenCalledTimes(callCount); + }); + + it('falls back to the timeout when requestAnimationFrame is unavailable', () => { + vi.stubGlobal('requestAnimationFrame', undefined); + try { + let progress: HeardTextProgress = { chars: 2, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: TEXT }); + progress = { chars: 6, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe(TEXT.slice(0, 6)); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('wakes up when the subscribed source notifies and unsubscribes on unmount', () => { + let progress: HeardTextProgress = { chars: 2, done: true }; + let notify: (() => void) | undefined; + const unsubscribe = vi.fn(); + const subscribe = vi.fn((listener) => { + notify = listener; + return unsubscribe; + }); + const { result, unmount } = renderHeardText({ + generating: true, + heard: () => progress, + subscribe, + text: TEXT, + }); + expect(result.current.heardText).toBe(TEXT.slice(0, 2)); + + // settled (done), so no ticking until the source notifies + progress = { chars: 7, done: false }; + advance(FRAME_MS * 3); + expect(result.current.heardText).toBe(TEXT.slice(0, 2)); + + act(() => notify?.()); + expect(result.current.heardText).toBe(TEXT.slice(0, 7)); + + // ticking again + progress = { chars: 9, done: false }; + advance(FRAME_MS * 2); + expect(result.current.heardText).toBe(TEXT.slice(0, 9)); + + expect(subscribe).toHaveBeenCalledTimes(1); + unmount(); + expect(unsubscribe).toHaveBeenCalledTimes(1); + }); + + it('is not paced while heard returns undefined', () => { + const heard = vi.fn(() => undefined); + const { rerender, result } = renderHeardText({ heard, text: 'Hello' }); + expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: false }); + + // no ticking while not paced + const callCount = heard.mock.calls.length; + advance(FRAME_MS * 10); + expect(heard).toHaveBeenCalledTimes(callCount); + + // re-checked when the text changes + rerender({ heard, text: 'Hello world' }); + expect(result.current).toMatchObject({ heardText: 'Hello world', isPaced: false }); + expect(heard.mock.calls.length).toBeGreaterThan(callCount); + }); + + it('starts pacing from the length already shown when it becomes paced', () => { + let progress: HeardTextProgress | undefined; + let notify: (() => void) | undefined; + const subscribe: HeardTextSubscribe = (listener) => { + notify = listener; + return () => undefined; + }; + const { rerender, result } = renderHeardText({ + heard: () => progress, + subscribe, + text: 'Hello', + }); + expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: false }); + + progress = { chars: 0, done: false }; + act(() => notify?.()); + // paced now, but does not move back past the 5 characters already shown + expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: true }); + + rerender({ heard: () => progress, subscribe, text: 'Hello world' }); + expect(result.current.heardText).toBe('Hello'); + + progress = { chars: 8, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe('Hello wo'); + }); + + it('reveals the full text at once when not live', () => { + const { result } = renderHeardText({ + generating: true, + heard: () => ({ chars: 0, done: false }), + live: false, + text: TEXT, + }); + expect(result.current).toMatchObject({ heardText: TEXT, isPaced: true }); + }); + + it('reads live only once, at mount', () => { + const heard: HeardTextFunction = () => ({ chars: 3, done: false }); + const { rerender, result } = renderHeardText({ heard, live: true, text: TEXT }); + expect(result.current.heardText).toBe(TEXT.slice(0, 3)); + + rerender({ heard, live: false, text: TEXT }); + advance(FRAME_MS); + expect(result.current.heardText).toBe(TEXT.slice(0, 3)); + }); + + it('does not restart pacing when heard and subscribe get a new identity each render', () => { + let progress: HeardTextProgress = { chars: 4, done: false }; + const unsubscribe = vi.fn(); + const subscribeImpl = vi.fn(() => unsubscribe); + const props = (): UseHeardTextParams => ({ + // new function identities on every render, as with inline arrow functions + heard: () => progress, + subscribe: (listener) => subscribeImpl(listener), + text: TEXT, + }); + const { rerender, result } = renderHeardText(props()); + expect(result.current.heardText).toBe(TEXT.slice(0, 4)); + + for (let i = 0; i < 5; i++) rerender(props()); + expect(subscribeImpl).toHaveBeenCalledTimes(1); + expect(unsubscribe).not.toHaveBeenCalled(); + + // the latest heard is the one being sampled + let latest: HeardTextProgress = { chars: 10, done: false }; + rerender({ ...props(), heard: () => latest }); + advance(FRAME_MS); + expect(result.current.heardText).toBe(TEXT.slice(0, 10)); + + latest = { chars: 12, done: false }; + progress = { chars: 1, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe(TEXT.slice(0, 12)); + }); + + it('skip reveals the full text from then on', () => { + const { result } = renderHeardText({ + heard: () => ({ chars: 3, done: false }), + text: TEXT, + }); + expect(result.current.heardText).toBe(TEXT.slice(0, 3)); + const { skip } = result.current; + + act(() => skip()); + expect(result.current.heardText).toBe(TEXT); + expect(result.current.skip).toBe(skip); + + advance(FRAME_MS * 5); + expect(result.current.heardText).toBe(TEXT); + }); + + describe('with emoji and other astral characters', () => { + // waving hand + skin tone modifier, mathematical script X, grinning face, the + // US flag (two regional indicators) and a BMP character: all but the last are + // surrogate pairs in UTF-16 + const ASTRAL_TEXT = 'Hi \u{1F44B}\u{1F3FD} \u{1D4B3}\u{1F600}!\u{1F1FA}\u{1F1F8}é'; + const CODE_POINTS = Array.from(ASTRAL_TEXT); + const LONE_SURROGATE = + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + expect(CODE_POINTS.length).toBeLessThan(ASTRAL_TEXT.length); + let progress: HeardTextProgress = { chars: 0, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: ASTRAL_TEXT }); + + for (let chars = 1; chars <= CODE_POINTS.length; chars++) { + progress = { chars, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe(CODE_POINTS.slice(0, chars).join('')); + expect(result.current.heardText).not.toMatch(LONE_SURROGATE); + } + // the whole text once chars reaches the code point length, still speaking + expect(result.current.heardText).toBe(ASTRAL_TEXT); + }); + + it('never produces a lone surrogate for fractional or time based progress', () => { + const start = performance.now(); + // 7.3 code points per second + const { result } = renderHeardText({ + heard: (now) => ({ chars: ((now - start) / 1000) * 7.3, done: false }), + text: ASTRAL_TEXT, + }); + + for (let frame = 0; frame < 150; frame++) { + advance(FRAME_MS); + expect(result.current.heardText).not.toMatch(LONE_SURROGATE); + expect(ASTRAL_TEXT.startsWith(result.current.heardText)).toBe(true); + } + expect(result.current.heardText).toBe(ASTRAL_TEXT); + }); + + it('reveals the full text when chars exceeds the code point length', () => { + const { result } = renderHeardText({ + heard: () => ({ chars: ASTRAL_TEXT.length + 10, done: false }), + text: ASTRAL_TEXT, + }); + expect(result.current.heardText).toBe(ASTRAL_TEXT); + }); + + it('never moves the reveal backwards in code points', () => { + let progress: HeardTextProgress = { chars: 6, done: false }; + const { result } = renderHeardText({ heard: () => progress, text: ASTRAL_TEXT }); + expect(result.current.heardText).toBe(CODE_POINTS.slice(0, 6).join('')); + + progress = { chars: 2, done: false }; + advance(FRAME_MS * 3); + expect(result.current.heardText).toBe(CODE_POINTS.slice(0, 6).join('')); + }); + + it('keeps code point boundaries when the text grows', () => { + const heard: HeardTextFunction = () => ({ chars: 5, done: false }); + const { rerender, result } = renderHeardText({ + heard, + text: '\u{1F600}\u{1F600}', + }); + expect(result.current.heardText).toBe('\u{1F600}\u{1F600}'); + + rerender({ + heard, + text: '\u{1F600}\u{1F600}\u{1F44B}\u{1F44B}\u{1F44B}\u{1F44B}', + }); + advance(FRAME_MS); + expect(result.current.heardText).toBe( + '\u{1F600}\u{1F600}\u{1F44B}\u{1F44B}\u{1F44B}', + ); + }); + }); +}); diff --git a/src/plugins/AIComponents/hooks/useHeardText.ts b/src/plugins/AIComponents/hooks/useHeardText.ts new file mode 100644 index 000000000..dcf79fae0 --- /dev/null +++ b/src/plugins/AIComponents/hooks/useHeardText.ts @@ -0,0 +1,261 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +/** How much of a text the user has heard so far. */ +export type HeardTextProgress = { + /** + * Number of characters of the text the user has heard, counted in Unicode code points + * (as `Array.from(text).length` counts them, not UTF-16 code units like `text.length`). + * May be fractional; it is floored before rendering. A value at or above the text's + * code point length reveals the whole text. + */ + chars: number; + /** `true` once the speech for the text has finished (played out, interrupted or skipped). */ + done: boolean; +}; + +/** + * Returns how much of the text the user has heard by `now` (a `performance.now()` + * timestamp). Return `undefined` when the text is not paced (not spoken, or not known + * to the voice source yet); return `{ chars: 0, done: false }` for a spoken text whose + * audio has not started yet. + */ +export type HeardTextFunction = (now: number) => HeardTextProgress | undefined; + +/** + * Registers a listener that is called whenever the speech progress source changes + * (playback started, paused, finished...). Returns an unsubscribe function. + */ +export type HeardTextSubscribe = (listener: () => void) => () => void; + +export type UseHeardTextParams = { + /** The speech progress of `text`, see `HeardTextFunction`. */ + heard: HeardTextFunction; + /** The full text to reveal. */ + text: string; + /** + * `true` while the text is still being written. The full text is only revealed + * automatically once `heard` reports `done` and `generating` is falsy. + */ + generating?: boolean; + /** + * Defaults to `true`. `false` reveals the full text at once, for example for a message + * loaded from history. Read once, when the hook mounts. + */ + live?: boolean; + /** Optional change notifications from the speech progress source. */ + subscribe?: HeardTextSubscribe; +}; + +export type UseHeardTextResult = { + /** The part of the text the user has heard (the full text when not paced). */ + heardText: string; + /** `false` when `heard` returned `undefined`; `heardText` is then the full text. */ + isPaced: boolean; + /** Reveals the full text from now on (e.g. when the user stops the AI response). */ + skip: () => void; +}; + +type PacingState = { + /** Number of code points revealed. Never decreases. */ + count: number; + isPaced: boolean; + /** No more ticking is needed until the inputs change or the source notifies. */ + settled: boolean; +}; + +const now = () => + typeof performance !== 'undefined' && typeof performance.now === 'function' + ? performance.now() + : Date.now(); + +const scheduleFrame = (callback: () => void): (() => void) => { + if (typeof requestAnimationFrame === 'function') { + const id = requestAnimationFrame(callback); + return () => cancelAnimationFrame(id); + } + const id = setTimeout(callback, 16); + return () => clearTimeout(id); +}; + +/** + * UTF-16 offsets of the code point boundaries of `text`: `offsets[n]` is the + * `String.prototype.slice` end index of the first `n` code points, so + * `offsets.length - 1` is the code point length. Slicing at these offsets never splits + * a surrogate pair. + */ +const getCodePointOffsets = (text: string): number[] => { + const offsets = [0]; + let offset = 0; + for (const codePoint of text) { + offset += codePoint.length; + offsets.push(offset); + } + return offsets; +}; + +const sample = ({ + generating, + heard, + live, + previousCount, + skipped, + textLength, +}: { + generating: boolean; + heard: HeardTextFunction; + live: boolean; + previousCount: number; + skipped: boolean; + /** Length of the text in code points. */ + textLength: number; +}): PacingState => { + const progress = heard(now()); + + if (!progress) { + // Not paced: the full text is shown, so pacing that starts later must not move the + // reveal backwards past what has already been shown. + return { count: Math.max(previousCount, textLength), isPaced: false, settled: true }; + } + + if (!live || skipped || (progress.done && !generating)) { + return { count: Math.max(previousCount, textLength), isPaced: true, settled: true }; + } + + const heardChars = Number.isFinite(progress.chars) + ? Math.max(0, Math.floor(progress.chars)) + : 0; + + return { + count: Math.max(previousCount, Math.min(heardChars, textLength)), + isPaced: true, + settled: progress.done, + }; +}; + +const noop = () => undefined; + +/** + * Reveals a text at the pace the user hears it spoken, for example the reply of a voice + * AI agent that is written into the chat faster than the agent speaks it. The speech + * progress comes from the app as a plain function, so the hook works with any voice + * backend and has no dependency on the chat SDK. + * + * - Reveals the first `floor(chars)` Unicode code points of `text` (emoji and other + * astral characters are never split) and never moves backwards. + * - While the text is paced and its speech is not done, samples `heard` on every + * animation frame (a 16 ms timeout without `requestAnimationFrame`). It stops ticking + * when the speech is done or the component unmounts; `subscribe` notifications, and + * changes of `text` or `generating`, make it check again. + * - Reveals the full text once `heard` reports `done` and `generating` is falsy, when + * `live` is `false` at mount, and after `skip()`. + * - When `heard` returns `undefined`, `isPaced` is `false` and `heardText` is the full + * text. If it becomes paced later, the reveal continues from the length already shown. + * + * `heard` and `subscribe` are read through refs, so passing new function identities on + * every render (inline arrow functions) does not restart the pacing. `subscribe` is + * called once on mount (and again only if it goes from absent to present) and its + * unsubscribe function runs on unmount. + * + * @example + * ```tsx + * import { useHeardText } from 'stream-chat-react/ai-components'; + * + * const SpokenReply = ({ generating, session, text, turnId }) => { + * const { heardText } = useHeardText({ + * generating, + * heard: (now) => session.speech.heard(turnId, now), + * subscribe: (listener) => session.speech.subscribe(listener), + * text, + * }); + * return

{heardText}

; + * }; + * ``` + */ +export const useHeardText = ({ + generating = false, + heard, + live = true, + subscribe, + text, +}: UseHeardTextParams): UseHeardTextResult => { + const codePointOffsets = useMemo(() => getCodePointOffsets(text), [text]); + const textLength = codePointOffsets.length - 1; + + const heardRef = useRef(heard); + heardRef.current = heard; + const subscribeRef = useRef(subscribe); + subscribeRef.current = subscribe; + + const [isLive] = useState(live); + const [skipped, setSkipped] = useState(false); + const [pacingState, setPacingState] = useState(() => + sample({ + generating, + heard, + live: isLive, + previousCount: 0, + skipped, + textLength, + }), + ); + const pacingStateRef = useRef(pacingState); + const wakeRef = useRef<() => void>(noop); + + useEffect(() => { + let cancelFrame: (() => void) | undefined; + let disposed = false; + + const tick = () => { + cancelFrame = undefined; + if (disposed) return; + + const previous = pacingStateRef.current; + const next = sample({ + generating, + heard: heardRef.current, + live: isLive, + previousCount: previous.count, + skipped, + textLength, + }); + + if ( + next.count !== previous.count || + next.isPaced !== previous.isPaced || + next.settled !== previous.settled + ) { + pacingStateRef.current = next; + setPacingState(next); + } + + if (!next.settled) cancelFrame = scheduleFrame(tick); + }; + + wakeRef.current = () => { + if (!disposed && !cancelFrame) tick(); + }; + tick(); + + return () => { + disposed = true; + cancelFrame?.(); + wakeRef.current = noop; + }; + }, [generating, isLive, skipped, text, textLength]); + + const hasSubscribe = Boolean(subscribe); + useEffect(() => { + if (!hasSubscribe) return; + return subscribeRef.current?.(() => wakeRef.current()); + }, [hasSubscribe]); + + const skip = useCallback(() => setSkipped(true), []); + + return { + heardText: pacingState.isPaced + ? text.slice(0, codePointOffsets[Math.min(pacingState.count, textLength)]) + : text, + isPaced: pacingState.isPaced, + skip, + }; +}; diff --git a/src/plugins/AIComponents/index.ts b/src/plugins/AIComponents/index.ts index 9a5cc1d2c..852837dfd 100644 --- a/src/plugins/AIComponents/index.ts +++ b/src/plugins/AIComponents/index.ts @@ -1,4 +1,5 @@ export * from './AIMarkdown'; export * from './AIMessageComposer'; export * from './AIStateIndicator/AIStateIndicator'; +export * from './hooks/useHeardText'; export * from './StreamingMessage/StreamingMessage'; From 835c3eb6610b324cc70f21f3db7dd3fbf5791901 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 26 Oct 2026 11:31:11 +0100 Subject: [PATCH 2/2] fix(ai-components): restart useHeardText for a new text and treat Infinity as fully heard - Start the reveal over when `text` is replaced by a different text (the next reply rendered by the same component): the length shown and skip() reset, and `live` is read again. Growing or truncating the text keeps the reveal, capped at the new length. - `chars: Infinity` reveals the whole text, as the type doc says. - Document that `undefined` is only for text that is not spoken, that `subscribe` is bound once (remount to switch sources) and is the only wake-up for a settled hook, and that `heard` runs during render. --- src/plugins/AIComponents/README.md | 23 ++-- .../__tests__/useHeardText.test.ts | 86 +++++++++++- .../AIComponents/hooks/useHeardText.ts | 126 +++++++++++++----- 3 files changed, 192 insertions(+), 43 deletions(-) diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index b6b92e634..03b9148f7 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -555,17 +555,17 @@ type HeardTextSubscribe = (listener: () => void) => () => void; const useHeardText: (params: UseHeardTextParams) => UseHeardTextResult; ``` -`heard(now)` returns how many characters of the text the user has heard by `now` (a `performance.now()` timestamp) and whether the speech for it has finished. `chars` counts Unicode code points (`Array.from(text).length`), not UTF-16 code units. Return `undefined` when the text is not paced, and `{ chars: 0, done: false }` for a spoken text whose audio has not started yet. +`heard(now)` returns how many characters of the text the user has heard by `now` (a `performance.now()` timestamp) and whether the speech for it has finished. `chars` counts Unicode code points (`Array.from(text).length`), not UTF-16 code units; `Infinity` means the whole text. Return `undefined` only for a text that is not spoken at all. A text that will be spoken but whose audio has not started, or that the voice source does not know yet, must return `{ chars: 0, done: false }`: while `heard` returns `undefined` the full text is shown, and the reveal never moves back past what has been shown. `heard` is also called during render, so keep it free of side effects. #### Props -| Name | Type | Required | Description | -| ------------ | -------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -| `text` | `string` | yes | The full text to reveal. | -| `heard` | `HeardTextFunction` | yes | The speech progress of `text`. | -| `subscribe` | `HeardTextSubscribe` | no | Change notifications from the speech progress source; each one makes the hook check `heard` again. | -| `generating` | `boolean` | no | `true` while the text is still being written. The full text is revealed once `heard` reports `done` and this is falsy. | -| `live` | `boolean` | no | Defaults to `true`. `false` reveals the full text at once, for example for a message loaded from history. Read at mount. | +| Name | Type | Required | Description | +| ------------ | -------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------- | +| `text` | `string` | yes | The full text to reveal. | +| `heard` | `HeardTextFunction` | yes | The speech progress of `text`. | +| `subscribe` | `HeardTextSubscribe` | no | Change notifications from the speech progress source; each one makes the hook check `heard` again. See below. | +| `generating` | `boolean` | no | `true` while the text is still being written. The full text is revealed once `heard` reports `done` and this is falsy. | +| `live` | `boolean` | no | Defaults to `true`. `false` reveals the full text at once, for example for a message loaded from history. See below. | #### Returns @@ -577,10 +577,11 @@ const useHeardText: (params: UseHeardTextParams) => UseHeardTextResult; Behaviour: -- It reveals the first `floor(chars)` code points of `text` and never moves backwards. Emoji and other astral characters are never split. -- While the text is paced and its speech is not done, it samples `heard` on every animation frame, and stops when the speech is done or the component unmounts. `subscribe` notifications and changes of `text` or `generating` make it check again. +- It reveals the first `floor(chars)` code points of `text` and never moves backwards while `text` grows or is truncated. Emoji and other astral characters are never split. +- When `text` is replaced by a different text (the next reply rendered by the same component), the reveal starts over: the length shown and `skip()` are reset, and `live` is read again. `live` is otherwise read only at mount. +- While the text is paced and its speech is not done, it samples `heard` on every animation frame, and stops when the speech is done or the component unmounts. `subscribe` notifications and changes of `text` or `generating` make it check again. Without `subscribe`, nothing else does: pass it when `heard` can change after the speech is done, or start returning a value after returning `undefined`. - If `heard` returns `undefined` at first and a value later, the reveal continues from the length already shown. -- `heard` and `subscribe` are read through refs: passing new functions on every render (inline arrow functions) does not restart the pacing. `subscribe` is called once and unsubscribed on unmount. +- `heard` and `subscribe` are read through refs: passing new functions on every render (inline arrow functions) does not restart the pacing. `subscribe` is called once and unsubscribed on unmount; to switch to another speech source, remount the component (for example with a `key`). #### Example diff --git a/src/plugins/AIComponents/__tests__/useHeardText.test.ts b/src/plugins/AIComponents/__tests__/useHeardText.test.ts index 4c72494f3..8d3301964 100644 --- a/src/plugins/AIComponents/__tests__/useHeardText.test.ts +++ b/src/plugins/AIComponents/__tests__/useHeardText.test.ts @@ -72,7 +72,7 @@ describe('useHeardText', () => { expect(result.current.heardText).toBe(TEXT.slice(0, 4)); }); - it('treats non-finite and negative counts as nothing heard', () => { + it('treats NaN, negative and non-numeric counts as nothing heard', () => { let progress: HeardTextProgress = { chars: Number.NaN, done: false }; const { result } = renderHeardText({ heard: () => progress, text: TEXT }); expect(result.current.heardText).toBe(''); @@ -80,6 +80,23 @@ describe('useHeardText', () => { progress = { chars: -3, done: false }; advance(FRAME_MS); expect(result.current.heardText).toBe(''); + + progress = { chars: Number.NEGATIVE_INFINITY, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe(''); + + // a JS caller returning no count at all + progress = { done: false } as unknown as HeardTextProgress; + advance(FRAME_MS); + expect(result.current.heardText).toBe(''); + }); + + it('treats an infinite count as the whole text heard', () => { + const { result } = renderHeardText({ + heard: () => ({ chars: Number.POSITIVE_INFINITY, done: false }), + text: TEXT, + }); + expect(result.current).toMatchObject({ heardText: TEXT, isPaced: true }); }); it('never moves the reveal backwards', () => { @@ -290,6 +307,73 @@ describe('useHeardText', () => { expect(result.current.heardText).toBe(TEXT); }); + describe('when the text is replaced by a different text', () => { + const NEXT_TEXT = 'Sure, here is the forecast for tomorrow.'; + + it('starts a new reveal instead of carrying over the length already shown', () => { + let progress: HeardTextProgress = { chars: 100, done: false }; + const heard: HeardTextFunction = () => progress; + const { rerender, result } = renderHeardText({ heard, text: TEXT }); + expect(result.current.heardText).toBe(TEXT); + + // the next reply, whose audio has not started yet + progress = { chars: 0, done: false }; + rerender({ heard, text: NEXT_TEXT }); + expect(result.current).toMatchObject({ heardText: '', isPaced: true }); + + progress = { chars: 4, done: false }; + advance(FRAME_MS); + expect(result.current.heardText).toBe('Sure'); + }); + + it('starts a new reveal after the previous text was shown unpaced', () => { + let progress: HeardTextProgress | undefined = undefined; + const heard: HeardTextFunction = () => progress; + const { rerender, result } = renderHeardText({ heard, text: TEXT }); + expect(result.current).toMatchObject({ heardText: TEXT, isPaced: false }); + + progress = { chars: 2, done: false }; + rerender({ heard, text: NEXT_TEXT }); + expect(result.current).toMatchObject({ heardText: 'Su', isPaced: true }); + }); + + it('clears a previous skip', () => { + const heard: HeardTextFunction = () => ({ chars: 3, done: false }); + const { rerender, result } = renderHeardText({ heard, text: TEXT }); + act(() => result.current.skip()); + expect(result.current.heardText).toBe(TEXT); + + rerender({ heard, text: NEXT_TEXT }); + expect(result.current.heardText).toBe('Sur'); + }); + + it('reads live again for the new text', () => { + const heard: HeardTextFunction = () => ({ chars: 3, done: false }); + const { rerender, result } = renderHeardText({ heard, live: false, text: TEXT }); + expect(result.current.heardText).toBe(TEXT); + + rerender({ heard, live: true, text: NEXT_TEXT }); + expect(result.current.heardText).toBe('Sur'); + }); + + it('keeps the reveal when the text is truncated, and does not reuse the cut length', () => { + let progress: HeardTextProgress = { chars: 12, done: false }; + const heard: HeardTextFunction = () => progress; + const { rerender, result } = renderHeardText({ heard, text: TEXT }); + expect(result.current.heardText).toBe(TEXT.slice(0, 12)); + + // e.g. the reply was cut short when the user stopped it + rerender({ heard, text: TEXT.slice(0, 8) }); + expect(result.current.heardText).toBe(TEXT.slice(0, 8)); + + // growing again from the truncated text continues from 8, not from 12 + progress = { chars: 3, done: false }; + rerender({ heard, text: `${TEXT.slice(0, 8)} again and again` }); + advance(FRAME_MS); + expect(result.current.heardText).toBe(TEXT.slice(0, 8)); + }); + }); + describe('with emoji and other astral characters', () => { // waving hand + skin tone modifier, mathematical script X, grinning face, the // US flag (two regional indicators) and a BMP character: all but the last are diff --git a/src/plugins/AIComponents/hooks/useHeardText.ts b/src/plugins/AIComponents/hooks/useHeardText.ts index dcf79fae0..29521f6e0 100644 --- a/src/plugins/AIComponents/hooks/useHeardText.ts +++ b/src/plugins/AIComponents/hooks/useHeardText.ts @@ -15,9 +15,14 @@ export type HeardTextProgress = { /** * Returns how much of the text the user has heard by `now` (a `performance.now()` - * timestamp). Return `undefined` when the text is not paced (not spoken, or not known - * to the voice source yet); return `{ chars: 0, done: false }` for a spoken text whose - * audio has not started yet. + * timestamp). + * + * Return `undefined` only for a text that is not spoken at all: the full text is then + * shown, and the reveal never moves back past what has been shown. A text that will be + * spoken but whose audio has not started, or that the voice source does not know yet, + * must return `{ chars: 0, done: false }`, or it is shown in full before it is heard. + * + * Also called during render, so keep it free of side effects. */ export type HeardTextFunction = (now: number) => HeardTextProgress | undefined; @@ -39,10 +44,16 @@ export type UseHeardTextParams = { generating?: boolean; /** * Defaults to `true`. `false` reveals the full text at once, for example for a message - * loaded from history. Read once, when the hook mounts. + * loaded from history. Read when the hook mounts and when `text` is replaced by a + * different text, not on every change. */ live?: boolean; - /** Optional change notifications from the speech progress source. */ + /** + * Optional change notifications from the speech progress source. Without it, `heard` + * is only checked again while the text is paced and its speech is not done, and when + * `text` or `generating` change. Subscribed once; to switch to another source, remount + * the component (for example with a `key`). + */ subscribe?: HeardTextSubscribe; }; @@ -56,13 +67,22 @@ export type UseHeardTextResult = { }; type PacingState = { - /** Number of code points revealed. Never decreases. */ + /** Number of code points revealed. Never decreases while the text continues. */ count: number; isPaced: boolean; /** No more ticking is needed until the inputs change or the source notifies. */ settled: boolean; }; +/** The reveal of one text. Starts over when the text is replaced by a different text. */ +type RevealState = PacingState & { + /** `live` as read when the reveal started. */ + live: boolean; + skipped: boolean; + /** The text `count` refers to. */ + text: string; +}; + const now = () => typeof performance !== 'undefined' && typeof performance.now === 'function' ? performance.now() @@ -121,9 +141,10 @@ const sample = ({ return { count: Math.max(previousCount, textLength), isPaced: true, settled: true }; } - const heardChars = Number.isFinite(progress.chars) - ? Math.max(0, Math.floor(progress.chars)) - : 0; + // `+Infinity` means the whole text has been heard; NaN, negative and non-numeric + // counts mean nothing has + const flooredChars = Math.floor(progress.chars); + const heardChars = flooredChars > 0 ? flooredChars : 0; return { count: Math.max(previousCount, Math.min(heardChars, textLength)), @@ -132,6 +153,39 @@ const sample = ({ }; }; +const startReveal = ({ + generating, + heard, + live, + text, + textLength, +}: { + generating: boolean; + heard: HeardTextFunction; + live: boolean; + text: string; + textLength: number; +}): RevealState => ({ + ...sample({ generating, heard, live, previousCount: 0, skipped: false, textLength }), + live, + skipped: false, + text, +}); + +/** + * Whether `text` continues the reveal of `previous.text`: it extends or truncates the part + * already shown. Anything else (the next reply rendered by the same component) is a new + * text, whose reveal must not start from the length shown for the previous one. + */ +const continuesReveal = (previous: RevealState, text: string) => { + const offsets = getCodePointOffsets(previous.text); + const shown = previous.text.slice( + 0, + offsets[Math.min(previous.count, offsets.length - 1)], + ); + return text.startsWith(shown) || shown.startsWith(text); +}; + const noop = () => undefined; /** @@ -141,20 +195,24 @@ const noop = () => undefined; * backend and has no dependency on the chat SDK. * * - Reveals the first `floor(chars)` Unicode code points of `text` (emoji and other - * astral characters are never split) and never moves backwards. + * astral characters are never split) and never moves backwards while `text` grows or + * is truncated. When `text` is replaced by a different text (the next reply rendered + * by the same component), the reveal starts over: the length shown, `skip()` and + * `live` are reset. * - While the text is paced and its speech is not done, samples `heard` on every * animation frame (a 16 ms timeout without `requestAnimationFrame`). It stops ticking * when the speech is done or the component unmounts; `subscribe` notifications, and * changes of `text` or `generating`, make it check again. * - Reveals the full text once `heard` reports `done` and `generating` is falsy, when - * `live` is `false` at mount, and after `skip()`. + * `live` is `false`, and after `skip()`. * - When `heard` returns `undefined`, `isPaced` is `false` and `heardText` is the full * text. If it becomes paced later, the reveal continues from the length already shown. * * `heard` and `subscribe` are read through refs, so passing new function identities on * every render (inline arrow functions) does not restart the pacing. `subscribe` is * called once on mount (and again only if it goes from absent to present) and its - * unsubscribe function runs on unmount. + * unsubscribe function runs on unmount; to switch to another speech source, remount the + * component. `heard` is also called during render, so keep it free of side effects. * * @example * ```tsx @@ -186,21 +244,23 @@ export const useHeardText = ({ const subscribeRef = useRef(subscribe); subscribeRef.current = subscribe; - const [isLive] = useState(live); - const [skipped, setSkipped] = useState(false); - const [pacingState, setPacingState] = useState(() => - sample({ - generating, - heard, - live: isLive, - previousCount: 0, - skipped, - textLength, - }), + const [revealState, setRevealState] = useState(() => + startReveal({ generating, heard, live, text, textLength }), ); - const pacingStateRef = useRef(pacingState); + let reveal = revealState; + if (revealState.text !== text) { + // Updated during render so that the new text never renders with the previous reveal. + reveal = continuesReveal(revealState, text) + ? { ...revealState, count: Math.min(revealState.count, textLength), text } + : startReveal({ generating, heard, live, text, textLength }); + setRevealState(reveal); + } + const revealRef = useRef(reveal); + revealRef.current = reveal; const wakeRef = useRef<() => void>(noop); + const { live: isLive, skipped } = reveal; + useEffect(() => { let cancelFrame: (() => void) | undefined; let disposed = false; @@ -209,7 +269,7 @@ export const useHeardText = ({ cancelFrame = undefined; if (disposed) return; - const previous = pacingStateRef.current; + const previous = revealRef.current; const next = sample({ generating, heard: heardRef.current, @@ -224,8 +284,9 @@ export const useHeardText = ({ next.isPaced !== previous.isPaced || next.settled !== previous.settled ) { - pacingStateRef.current = next; - setPacingState(next); + const updated = { ...previous, ...next }; + revealRef.current = updated; + setRevealState(updated); } if (!next.settled) cancelFrame = scheduleFrame(tick); @@ -249,13 +310,16 @@ export const useHeardText = ({ return subscribeRef.current?.(() => wakeRef.current()); }, [hasSubscribe]); - const skip = useCallback(() => setSkipped(true), []); + const skip = useCallback( + () => setRevealState((previous) => ({ ...previous, skipped: true })), + [], + ); return { - heardText: pacingState.isPaced - ? text.slice(0, codePointOffsets[Math.min(pacingState.count, textLength)]) + heardText: reveal.isPaced + ? text.slice(0, codePointOffsets[Math.min(reveal.count, textLength)]) : text, - isPaced: pacingState.isPaced, + isPaced: reveal.isPaced, skip, }; };